update proxy

Fixes an issue where things become unhappy when the room profile for a user is missing.

.ci/patch_for_twisted_trunk.sh

@@ -1,8 +0,0 @@
-#!/bin/sh
-
-# replaces the dependency on Twisted in `python_dependencies` with trunk.
-
-set -e
-cd "$(dirname "$0")"/..
-
-sed -i -e 's#"Twisted.*"#"Twisted @ git+https://github.com/twisted/twisted"#' synapse/python_dependencies.py

.ci/postgres-config.yaml

@@ -1,19 +0,0 @@
-# Configuration file used for testing the 'synapse_port_db' script.
-# Tells the script to connect to the postgresql database that will be available in the
-# CI's Docker setup at the point where this file is considered.
-server_name: "localhost:8800"
-
-signing_key_path: ".ci/test.signing.key"
-
-report_stats: false
-
-database:
-  name: "psycopg2"
-  args:
-    user: postgres
-    host: localhost
-    password: postgres
-    database: synapse
-
-# Suppress the key server warning.
-trusted_key_servers: []

.ci/scripts/postgres_exec.py

@@ -1,31 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import sys
-
-import psycopg2
-
-# a very simple replacment for `psql`, to make up for the lack of the postgres client
-# libraries in the synapse docker image.
-
-# We use "postgres" as a database because it's bound to exist and the "synapse" one
-# doesn't exist yet.
-db_conn = psycopg2.connect(
-    user="postgres", host="localhost", password="postgres", dbname="postgres"
-)
-db_conn.autocommit = True
-cur = db_conn.cursor()
-for c in sys.argv[1:]:
-    cur.execute(c)

.ci/scripts/test_export_data_command.sh

@@ -1,57 +0,0 @@
-#!/usr/bin/env bash
-
-# Test for the export-data admin command against sqlite and postgres
-
-set -xe
-cd "$(dirname "$0")/../.."
-
-echo "--- Install dependencies"
-
-# Install dependencies for this test.
-pip install psycopg2
-
-# Install Synapse itself. This won't update any libraries.
-pip install -e .
-
-echo "--- Generate the signing key"
-
-# Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
-
-echo "--- Prepare test database"
-
-# Make sure the SQLite3 database is using the latest schema and has no pending background update.
-scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
-
-# Run the export-data command on the sqlite test database
-python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
---output-directory /tmp/export_data
-
-# Test that the output directory exists and contains the rooms directory
-dir="/tmp/export_data/rooms"
-if [ -d "$dir" ]; then
-  echo "Command successful, this test passes"
-else
-  echo "No output directories found, the command fails against a sqlite database."
-  exit 1
-fi
-
-# Create the PostgreSQL database.
-.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
-
-# Port the SQLite databse to postgres so we can check command works against postgres
-echo "+++ Port SQLite3 databse to postgres"
-scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
-
-# Run the export-data command on postgres database
-python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
---output-directory /tmp/export_data2
-
-# Test that the output directory exists and contains the rooms directory
-dir2="/tmp/export_data2/rooms"
-if [ -d "$dir2" ]; then
-  echo "Command successful, this test passes"
-else
-  echo "No output directories found, the command fails against a postgres database."
-  exit 1
-fi

.ci/scripts/test_old_deps.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-# this script is run by GitHub Actions in a plain `bionic` container; it installs the
-# minimal requirements for tox and hands over to the py3-old tox environment.
-
-set -ex
-
-apt-get update
-apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox
-
-export LANG="C.UTF-8"
-
-# Prevent virtualenv from auto-updating pip to an incompatible version
-export VIRTUALENV_NO_DOWNLOAD=1
-
-exec tox -e py3-old,combine

.ci/scripts/test_synapse_port_db.sh

@@ -1,57 +0,0 @@
-#!/usr/bin/env bash
-#
-# Test script for 'synapse_port_db'.
-#   - sets up synapse and deps
-#   - runs the port script on a prepopulated test sqlite db
-#   - also runs it against an new sqlite db
-
-
-set -xe
-cd "$(dirname "$0")/../.."
-
-echo "--- Install dependencies"
-
-# Install dependencies for this test.
-pip install psycopg2 coverage coverage-enable-subprocess
-
-# Install Synapse itself. This won't update any libraries.
-pip install -e .
-
-echo "--- Generate the signing key"
-
-# Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
-
-echo "--- Prepare test database"
-
-# Make sure the SQLite3 database is using the latest schema and has no pending background update.
-scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
-
-# Create the PostgreSQL database.
-.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
-
-echo "+++ Run synapse_port_db against test database"
-coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
-
-# We should be able to run twice against the same database.
-echo "+++ Run synapse_port_db a second time"
-coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
-
-#####
-
-# Now do the same again, on an empty database.
-
-echo "--- Prepare empty SQLite database"
-
-# we do this by deleting the sqlite db, and then doing the same again.
-rm .ci/test_db.db
-
-scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
-
-# re-create the PostgreSQL database.
-.ci/scripts/postgres_exec.py \
-  "DROP DATABASE synapse" \
-  "CREATE DATABASE synapse"
-
-echo "+++ Run synapse_port_db against empty database"
-coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

.ci/sqlite-config.yaml

@@ -1,16 +0,0 @@
-# Configuration file used for testing the 'synapse_port_db' script.
-# Tells the 'update_database' script to connect to the test SQLite database to upgrade its
-# schema and run background updates on it.
-server_name: "localhost:8800"
-
-signing_key_path: ".ci/test.signing.key"
-
-report_stats: false
-
-database:
-  name: "sqlite3"
-  args:
-    database: ".ci/test_db.db"
-
-# Suppress the key server warning.
-trusted_key_servers: []

.ci/twisted_trunk_build_failed_issue_template.md

@@ -1,4 +0,0 @@
----
-title: CI run against Twisted trunk is failing
----
-See https://github.com/{{env.GITHUB_REPOSITORY}}/actions/runs/{{env.GITHUB_RUN_ID}}

.ci/worker-blacklist

@@ -1,2 +0,0 @@
-# This file serves as a blacklist for SyTest tests that we expect will fail in
-# Synapse when run under worker mode. For more details, see sytest-blacklist.

.circleci/config.yml

@@ -0,0 +1,172 @@
+version: 2
+jobs:
+  dockerhubuploadrelease:
+    machine: true
+    steps:
+      - checkout
+      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_TAG} .
+      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 .
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
+  dockerhubuploadlatest:
+    machine: true
+    steps:
+      - checkout
+      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_SHA1} .
+      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_SHA1}-py3 --build-arg PYTHON_VERSION=3.6 .
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      - run: docker tag matrixdotorg/synapse:${CIRCLE_SHA1} matrixdotorg/synapse:latest
+      - run: docker tag matrixdotorg/synapse:${CIRCLE_SHA1}-py3 matrixdotorg/synapse:latest-py3
+      - run: docker push matrixdotorg/synapse:${CIRCLE_SHA1}
+      - run: docker push matrixdotorg/synapse:${CIRCLE_SHA1}-py3
+      - run: docker push matrixdotorg/synapse:latest
+      - run: docker push matrixdotorg/synapse:latest-py3
+  sytestpy2:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
+    steps:
+      - checkout
+      - run: /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy2postgres:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
+    steps:
+      - checkout
+      - run: POSTGRES=1 /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy2merged:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy2postgresmerged:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: POSTGRES=1 /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+
+  sytestpy3:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
+    steps:
+      - checkout
+      - run: /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy3postgres:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
+    steps:
+      - checkout
+      - run: POSTGRES=1 /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy3merged:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy3postgresmerged:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: POSTGRES=1 /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - sytestpy2:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy2postgres:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy3:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy3postgres:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy2merged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
+      - sytestpy2postgresmerged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
+      - sytestpy3merged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
+      - sytestpy3postgresmerged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
+      - dockerhubuploadrelease:
+          filters:
+            tags:
+              only: /v[0-9].[0-9]+.[0-9]+.*/
+            branches:
+              ignore: /.*/
+      - dockerhubuploadlatest:
+          filters:
+            branches:
+              only: master

.circleci/merge_base_branch.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -e
+
+# CircleCI doesn't give CIRCLE_PR_NUMBER in the environment for non-forked PRs. Wonderful.
+# In this case, we just need to do some ~shell magic~ to strip it out of the PULL_REQUEST URL.
+echo 'export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}"' >> $BASH_ENV
+source $BASH_ENV
+
+if [[ -z "${CIRCLE_PR_NUMBER}" ]]
+then
+    echo "Can't figure out what the PR number is! Assuming merge target is develop."
+
+    # It probably hasn't had a PR opened yet. Since all PRs land on develop, we
+    # can probably assume it's based on it and will be merged into it.
+    GITBASE="develop"
+else
+    # Get the reference, using the GitHub API
+    GITBASE=`wget -O- https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
+fi
+
+# Show what we are before
+git show -s
+
+# Set up username so it can do a merge
+git config --global user.email bot@matrix.org
+git config --global user.name "A robot"
+
+# Fetch and merge. If it doesn't work, it will raise due to set -e.
+git fetch -u origin $GITBASE
+git merge --no-edit origin/$GITBASE
+
+# Show what we are after.
+git show -s

.codecov.yml

@@ -1,14 +0,0 @@
-comment: off
-
-coverage:
-  status:
-    project:
-      default:
-        target: 0  # Target % coverage, can be auto. Turned off for now
-        threshold: null
-        base: auto
-    patch:
-      default:
-        target: 0
-        threshold: null
-        base: auto

.coveragerc

@@ -1,8 +1,12 @@
 [run]
 branch = True
 parallel = True
-include=$TOP/synapse/*
-data_file = $TOP/.coverage
+source = synapse
+
+[paths]
+source=
+   coverage
 
 [report]
 precision = 2
+ignore_errors = True

.dockerignore

@@ -1,13 +1,7 @@
-# ignore everything by default
-*
-
-# things to include
-!docker
-!scripts
-!synapse
-!MANIFEST.in
-!README.rst
-!setup.py
-!synctl
-
-**/__pycache__
+Dockerfile
+.travis.yml
+.gitignore
+demo/etc
+tox.ini
+.git/*
+.tox/*

.editorconfig

@@ -1,9 +0,0 @@
-# EditorConfig https://EditorConfig.org
-
-# top-most EditorConfig file
-root = true
-
-# 4 space indentation
-[*.py]
-indent_style = space
-indent_size = 4

.git-blame-ignore-revs

@@ -1,8 +0,0 @@
-# Black reformatting (#5482).
-32e7c9e7f20b57dd081023ac42d6931a8da9b3a3
-
-# Target Python 3.5 with black (#8664).
-aff1eb7c671b0a3813407321d2702ec46c71fa56
-
-# Update black to 20.8b1 (#9381).
-0a00b7ff14890987f09112a2ae696c61001e6cf1

.github/CODEOWNERS

@@ -1,2 +0,0 @@
-# Automatically request reviews from the synapse-core team when a pull request comes in.
-* @matrix-org/synapse-core

.github/FUNDING.yml

@@ -1,4 +0,0 @@
-# One username per supported platform and one custom link
-patreon: matrixdotorg
-liberapay: matrixdotorg
-custom: https://paypal.me/matrixdotorg

.github/ISSUE_TEMPLATE.md

@@ -1,5 +0,0 @@
-**If you are looking for support** please ask in **#synapse:matrix.org**
-(using a matrix.org account if necessary). We do not use GitHub issues for
-support.
-
-**If you want to report a security issue** please see https://matrix.org/security-disclosure-policy/

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -4,13 +4,11 @@ about: Create a report to help us improve
 
 ---
 
-<!--
+<!-- 
 
-**THIS IS NOT A SUPPORT CHANNEL!**
-**IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**,
-please ask in **#synapse:matrix.org** (using a matrix.org account if necessary)
+**IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**: 
+You will likely get better support more quickly if you ask in ** #matrix:matrix.org ** ;)
 
-If you want to report a security issue, please see https://matrix.org/security-disclosure-policy/
 
 This is a bug report template. By following the instructions below and
 filling out the sections with your information, you will help the us to get all
@@ -19,7 +17,7 @@ the necessary data to fix your issue.
 You can also preview your report before submitting it. You may remove sections
 that aren't relevant to your particular case.
 
-Text between <!-- and --​> marks will be invisible in the report.
+Text between <!-- and --> marks will be invisible in the report.
 
 -->
 
@@ -33,7 +31,7 @@ Text between <!-- and --​> marks will be invisible in the report.
 - that reproduce the bug
 - using hyphens as bullet points
 
-<!--
+<!-- 
 Describe how what happens differs from what you expected.
 
 If you can identify any relevant log snippets from _homeserver.log_, please include
@@ -46,26 +44,22 @@ those (please be careful to remove any personal or private data). Please surroun
 <!-- IMPORTANT: please answer the following questions, to help us narrow down the problem -->
 
 <!-- Was this issue identified on matrix.org or another homeserver? -->
-- **Homeserver**:
+- **Homeserver**: 
 
 If not matrix.org:
 
-<!--
- What version of Synapse is running?
-
-You can find the Synapse version with this command:
-
-$ curl http://localhost:8008/_synapse/admin/v1/server_version
-
-(You may need to replace `localhost:8008` if Synapse is not configured to
-listen on that port.)
+<!-- 
+What version of Synapse is running? 
+You can find the Synapse version by inspecting the server headers (replace matrix.org with
+your own homeserver domain):
+$ curl -v https://matrix.org/_matrix/client/versions 2>&1 | grep "Server:"
 -->
-- **Version**:
+- **Version**: 
 
-- **Install method**:
+- **Install method**: 
 <!-- examples: package manager/git clone/pip  -->
 
-- **Platform**:
+- **Platform**: 
 <!--
 Tell us about the environment in which your homeserver is operating
 distro, hardware, if it's running in a vm/container, etc.

.github/ISSUE_TEMPLATE/SUPPORT_REQUEST.md

@@ -4,7 +4,6 @@ about: I need support for Synapse
 
 ---
 
-Please don't file github issues asking for support.
+# Please ask for support in [**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org)
 
-Instead, please join [`#synapse:matrix.org`](https://matrix.to/#/#synapse:matrix.org)
-(from a matrix.org account if necessary), and ask there.
+## Don't file an issue as a support request.

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,14 +1,7 @@
 ### Pull Request Checklist
 
-<!-- Please read https://matrix-org.github.io/synapse/latest/development/contributing_guide.html before submitting your pull request -->
+<!-- Please read CONTRIBUTING.rst before submitting your pull request -->
 
 * [ ] Pull request is based on the develop branch
-* [ ] Pull request includes a [changelog file](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should:
-  - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.".
-  - Use markdown where necessary, mostly for `code blocks`.
-  - End with either a period (.) or an exclamation mark (!).
-  - Start with a capital letter.
-  - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry.
-* [ ] Pull request includes a [sign off](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#sign-off)
-* [ ] [Code style](https://matrix-org.github.io/synapse/latest/code_style.html) is correct
-  (run the [linters](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#run-the-linters))
+* [ ] Pull request includes a [changelog file](CONTRIBUTING.rst#changelog)
+* [ ] Pull request includes a [sign off](CONTRIBUTING.rst#sign-off)

.github/SUPPORT.md

@@ -1,3 +1,3 @@
-[**#synapse:matrix.org**](https://matrix.to/#/#synapse:matrix.org) is the official support room for
-Synapse, and can be accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html.
-Please ask for support there, rather than filing github issues.
+[**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org) is the official support room for Matrix, and can be accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html 
+
+It can also be access via IRC bridge at irc://irc.freenode.net/matrix or on the web here: https://webchat.freenode.net/?channels=matrix

.github/workflows/docker.yml

@@ -1,75 +0,0 @@
-# GitHub actions workflow which builds and publishes the docker images.
-
-name: Build docker images
-
-on:
-  push:
-    tags: ["v*"]
-    branches: [ master, main, develop ]
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up QEMU
-        id: qemu
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: arm64
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Inspect builder
-        run: docker buildx inspect
-          
-      - name: Log in to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Calculate docker image tag
-        id: set-tag
-        run: |
-          case "${GITHUB_REF}" in
-              refs/heads/develop)
-                  tag=develop
-                  ;;
-              refs/heads/master|refs/heads/main)
-                  tag=latest
-                  ;;
-              refs/tags/*)
-                  tag=${GITHUB_REF#refs/tags/}
-                  ;;
-              *)
-                  tag=${GITHUB_SHA}
-                  ;;
-          esac
-          echo "::set-output name=tag::$tag"
-
-        # for release builds, we want to get the amd64 image out asap, so first
-        # we do an amd64-only build, before following up with a multiarch build.
-      - name: Build and push amd64
-        uses: docker/build-push-action@v2
-        if: "${{ startsWith(github.ref, 'refs/tags/v') }}"
-        with:
-          push: true
-          labels: "gitsha1=${{ github.sha }}"
-          tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
-          file: "docker/Dockerfile"
-          platforms: linux/amd64
-
-      - name: Build and push all platforms
-        uses: docker/build-push-action@v2
-        with:
-          push: true
-          labels: "gitsha1=${{ github.sha }}"
-          tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
-          file: "docker/Dockerfile"
-          platforms: linux/amd64,linux/arm64

.github/workflows/docs.yaml

@@ -1,65 +0,0 @@
-name: Deploy the documentation
-
-on:
-  push:
-    branches:
-      # For bleeding-edge documentation
-      - develop
-      # For documentation specific to a release
-      - 'release-v*'
-      # stable docs
-      - master
-
-  workflow_dispatch:
-
-jobs:
-  pages:
-    name: GitHub Pages
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Setup mdbook
-        uses: peaceiris/actions-mdbook@4b5ef36b314c2599664ca107bb8c02412548d79d # v1.1.14
-        with:
-          mdbook-version: '0.4.9'
-
-      - name: Build the documentation
-        # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
-        # However, we're using docs/README.md for other purposes and need to pick a new page
-        # as the default. Let's opt for the welcome page instead.
-        run: |
-          mdbook build
-          cp book/welcome_and_overview.html book/index.html
-
-      # Figure out the target directory.
-      #
-      # The target directory depends on the name of the branch
-      #
-      - name: Get the target directory name
-        id: vars
-        run: |
-          # first strip the 'refs/heads/' prefix with some shell foo
-          branch="${GITHUB_REF#refs/heads/}"
-
-          case $branch in
-              release-*)
-                  # strip 'release-' from the name for release branches.
-                  branch="${branch#release-}"
-                  ;;
-              master)
-                  # deploy to "latest" for the master branch.
-                  branch="latest"
-                  ;;
-          esac
-
-          # finally, set the 'branch-version' var.
-          echo "::set-output name=branch-version::$branch"
-          
-      # Deploy to the target directory.
-      - name: Deploy to gh pages
-        uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./book
-          destination_dir: ./${{ steps.vars.outputs.branch-version }}

.github/workflows/release-artifacts.yml

@@ -1,130 +0,0 @@
-# GitHub actions workflow which builds the release artifacts.
-
-name: Build release artifacts
-
-on:
-  # we build on PRs and develop to (hopefully) get early warning
-  # of things breaking (but only build one set of debs)
-  pull_request:
-  push:
-    branches: ["develop"]
-
-    # we do the full build on tags.
-    tags: ["v*"]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-  
-permissions:
-  contents: write
-
-jobs:
-  get-distros:
-    name: "Calculate list of debian distros"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - id: set-distros
-        run: |
-          # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
-          dists='["debian:sid"]'
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-              dists=$(scripts-dev/build_debian_packages --show-dists-json)
-          fi
-          echo "::set-output name=distros::$dists"
-    # map the step outputs to job outputs
-    outputs:
-      distros: ${{ steps.set-distros.outputs.distros }}
-
-  # now build the packages with a matrix build.
-  build-debs:
-    needs: get-distros
-    name: "Build .deb packages"
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        distro: ${{ fromJson(needs.get-distros.outputs.distros) }}
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          path: src
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          install: true
-
-      - name: Set up docker layer caching
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
-
-      - name: Set up python
-        uses: actions/setup-python@v2
-
-      - name: Build the packages
-        # see https://github.com/docker/build-push-action/issues/252
-        # for the cache magic here
-        run: |
-          ./src/scripts-dev/build_debian_packages \
-            --docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \
-            --docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
-            --docker-build-arg=--progress=plain \
-            --docker-build-arg=--load \
-            "${{ matrix.distro }}"
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
-
-      - name: Upload debs as artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: debs
-          path: debs/*
-
-  build-sdist:
-    name: "Build pypi distribution files"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install wheel
-      - run: |
-          python setup.py sdist bdist_wheel
-      - uses: actions/upload-artifact@v2
-        with:
-          name: python-dist
-          path: dist/*
-
-  # if it's a tag, create a release and attach the artifacts to it
-  attach-assets:
-    name: "Attach assets to release"
-    if: ${{ !failure() && !cancelled() && startsWith(github.ref, 'refs/tags/') }}
-    needs:
-      - build-debs
-      - build-sdist
-    runs-on: ubuntu-latest
-    steps:
-      - name: Download all workflow run artifacts
-        uses: actions/download-artifact@v2
-      - name: Build a tarball for the debs
-        run: tar -cvJf debs.tar.xz debs
-      - name: Attach to release
-        uses: softprops/action-gh-release@a929a66f232c1b11af63782948aa2210f981808a  # PR#109
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          files: |
-            python-dist/*
-            debs.tar.xz
-          # if it's not already published, keep the release as a draft.
-          draft: true
-          # mark it as a prerelease if the tag contains 'rc'.
-          prerelease: ${{ contains(github.ref, 'rc') }}

.github/workflows/tests.yml

@@ -1,420 +0,0 @@
-name: Tests
-
-on:
-  push:
-    branches: ["develop", "release-*"]
-  pull_request:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        toxenv:
-          - "check-sampleconfig"
-          - "check_codestyle"
-          - "check_isort"
-          - "mypy"
-          - "packaging"
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install tox
-      - run: tox -e ${{ matrix.toxenv }}
-
-  lint-crlf:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Check line endings
-        run: scripts-dev/check_line_terminators.sh
-
-  lint-newsfile:
-    if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
-      - uses: actions/setup-python@v2
-      - run: pip install tox
-      - run: scripts-dev/check-newsfragment
-        env:
-          PULL_REQUEST_NUMBER: ${{ github.event.number }}
-
-  lint-sdist:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: "3.x"
-      - run: pip install wheel
-      - run: python setup.py sdist bdist_wheel
-      - uses: actions/upload-artifact@v2
-        with:
-          name: Python Distributions
-          path: dist/*
-
-  # Dummy step to gate other tests on without repeating the whole list
-  linting-done:
-    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    needs: [lint, lint-crlf, lint-newsfile, lint-sdist]
-    runs-on: ubuntu-latest
-    steps:
-      - run: "true"
-
-  trial:
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
-        database: ["sqlite"]
-        toxenv: ["py"]
-        include:
-          # Newest Python without optional deps
-          - python-version: "3.10"
-            toxenv: "py-noextras"
-
-          # Oldest Python with PostgreSQL
-          - python-version: "3.7"
-            database: "postgres"
-            postgres-version: "10"
-            toxenv: "py"
-
-          # Newest Python with newest PostgreSQL
-          - python-version: "3.10"
-            database: "postgres"
-            postgres-version: "14"
-            toxenv: "py"
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
-        if: ${{ matrix.postgres-version }}
-        run: |
-          docker run -d -p 5432:5432 \
-            -e POSTGRES_PASSWORD=postgres \
-            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
-            postgres:${{ matrix.postgres-version }}
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - run: pip install tox
-      - name: Await PostgreSQL
-        if: ${{ matrix.postgres-version }}
-        timeout-minutes: 2
-        run: until pg_isready -h localhost; do sleep 1; done
-      - run: tox -e ${{ matrix.toxenv }}
-        env:
-          TRIAL_FLAGS: "--jobs=2"
-          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
-          SYNAPSE_POSTGRES_HOST: localhost
-          SYNAPSE_POSTGRES_USER: postgres
-          SYNAPSE_POSTGRES_PASSWORD: postgres
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  trial-olddeps:
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Test with old deps
-        uses: docker://ubuntu:bionic # For old python and sqlite
-        with:
-          workdir: /github/workspace
-          entrypoint: .ci/scripts/test_old_deps.sh
-        env:
-          TRIAL_FLAGS: "--jobs=2"
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  trial-pypy:
-    # Very slow; only run if the branch name includes 'pypy'
-    if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["pypy-3.7"]
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - run: pip install tox
-      - run: tox -e py
-        env:
-          TRIAL_FLAGS: "--jobs=2"
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  sytest:
-    if: ${{ !failure() && !cancelled() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    container:
-      image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
-      volumes:
-        - ${{ github.workspace }}:/src
-      env:
-        SYTEST_BRANCH: ${{ github.head_ref }}
-        POSTGRES: ${{ matrix.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
-        WORKERS: ${{ matrix.workers && 1 }}
-        REDIS: ${{ matrix.redis && 1 }}
-        BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
-        TOP: ${{ github.workspace }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - sytest-tag: bionic
-
-          - sytest-tag: bionic
-            postgres: postgres
-
-          - sytest-tag: testing
-            postgres: postgres
-
-          - sytest-tag: bionic
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: postgres
-            workers: workers
-            redis: redis
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Prepare test blacklist
-        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
-      - name: Run SyTest
-        run: /bootstrap.sh synapse
-        working-directory: /src
-      - name: Summarise results.tap
-        if: ${{ always() }}
-        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
-      - name: Upload SyTest logs
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
-        with:
-          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
-          path: |
-            /logs/results.tap
-            /logs/**/*.log*
-
-  export-data:
-    if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
-    needs: [linting-done, portdb]
-    runs-on: ubuntu-latest
-    env:
-      TOP: ${{ github.workspace }}
-
-    services:
-      postgres:
-        image: postgres
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_PASSWORD: "postgres"
-          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
-        with:
-          python-version: "3.9"
-      - run: .ci/scripts/test_export_data_command.sh
-
-  portdb:
-    if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    env:
-      TOP: ${{ github.workspace }}
-    strategy:
-      matrix:
-        include:
-          - python-version: "3.7"
-            postgres-version: "10"
-
-          - python-version: "3.10"
-            postgres-version: "14"
-
-    services:
-      postgres:
-        image: postgres:${{ matrix.postgres-version }}
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_PASSWORD: "postgres"
-          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - run: .ci/scripts/test_synapse_port_db.sh
-
-  complement:
-    if: ${{ !failure() && !cancelled() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    container:
-      # https://github.com/matrix-org/complement/blob/master/dockerfiles/ComplementCIBuildkite.Dockerfile
-      image: matrixdotorg/complement:latest
-      env:
-        CI: true
-      ports:
-        - 8448:8448
-      volumes:
-        - /var/run/docker.sock:/var/run/docker.sock
-
-    steps:
-      - name: Run actions/checkout@v2 for synapse
-        uses: actions/checkout@v2
-        with:
-          path: synapse
-
-      # Attempt to check out the same branch of Complement as the PR. If it
-      # doesn't exist, fallback to master.
-      - name: Checkout complement
-        shell: bash
-        run: |
-          mkdir -p complement
-          # Attempt to use the version of complement which best matches the current
-          # build. Depending on whether this is a PR or release, etc. we need to
-          # use different fallbacks.
-          #
-          # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
-          #    for pull requests, otherwise GITHUB_REF).
-          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
-          #    (GITHUB_BASE_REF for pull requests).
-          # 3. Use the default complement branch ("master").
-          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
-            # Skip empty branch names and merge commits.
-            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
-              continue
-            fi
-
-            (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
-          done
-
-      # Build initial Synapse image
-      - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
-        working-directory: synapse
-        env:
-          DOCKER_BUILDKIT: 1
-
-      # Build a ready-to-run Synapse image based on the initial image above.
-      # This new image includes a config file, keys for signing and TLS, and
-      # other settings to make it suitable for testing under Complement.
-      - run: docker build -t complement-synapse -f Synapse.Dockerfile .
-        working-directory: complement/dockerfiles
-
-      # Run Complement
-      - run: set -o pipefail && go test -v -json -tags synapse_blacklist,msc2403 ./tests/... 2>&1 | gotestfmt
-        shell: bash
-        env:
-          COMPLEMENT_BASE_IMAGE: complement-synapse:latest
-        working-directory: complement
-
-  # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
-  tests-done:
-    if: ${{ always() }}
-    needs:
-      - lint
-      - lint-crlf
-      - lint-newsfile
-      - lint-sdist
-      - trial
-      - trial-olddeps
-      - sytest
-      - portdb
-      - complement
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set build result
-        env:
-          NEEDS_CONTEXT: ${{ toJSON(needs) }}
-        # the `jq` incantation dumps out a series of "<job> <result>" lines.
-        # we set it to an intermediate variable to avoid a pipe, which makes it
-        # hard to set $rc.
-        run: |
-          rc=0
-          results=$(jq -r 'to_entries[] | [.key,.value.result] | join(" ")' <<< $NEEDS_CONTEXT)
-          while read job result ; do
-              # The newsfile lint may be skipped on non PR builds
-              if [ $result == "skipped" ] && [ $job == "lint-newsfile" ]; then
-                continue
-              fi
-
-              if [ "$result" != "success" ]; then
-                  echo "::set-failed ::Job $job returned $result"
-                  rc=1
-              fi
-          done <<< $results
-          exit $rc

.github/workflows/twisted_trunk.yml

@@ -1,92 +0,0 @@
-name: Twisted Trunk
-
-on:
-  schedule:
-    - cron: 0 8 * * *
-
-  workflow_dispatch:
-
-jobs:
-  mypy:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e mypy
-
-  trial:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.6
-      - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e py
-        env:
-          TRIAL_FLAGS: "--jobs=2"
-
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  sytest:
-    runs-on: ubuntu-latest
-    container:
-      image: matrixdotorg/sytest-synapse:buster
-      volumes:
-        - ${{ github.workspace }}:/src
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Patch dependencies
-        run: .ci/patch_for_twisted_trunk.sh
-        working-directory: /src
-      - name: Run SyTest
-        run: /bootstrap.sh synapse
-        working-directory: /src
-      - name: Summarise results.tap
-        if: ${{ always() }}
-        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
-      - name: Upload SyTest logs
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
-        with:
-          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
-          path: |
-            /logs/results.tap
-            /logs/**/*.log*
-
-  # open an issue if the build fails, so we know about it.
-  open-issue:
-    if: failure()
-    needs:
-      - mypy
-      - trial
-      - sytest
-
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          update_existing: true
-          filename: .ci/twisted_trunk_build_failed_issue_template.md

.gitignore

@@ -1,56 +1,59 @@
-# filename patterns
-*~
+*.pyc
 .*.swp
-.#*
-*.deb
-*.egg
-*.egg-info
+*~
 *.lock
-*.py[cod]
-*.snap
-*.tac
+
+.DS_Store
 _trial_temp/
 _trial_temp*/
-/out
-.DS_Store
-__pycache__/
+logs/
+dbs/
+*.egg
+dist/
+docs/build/
+*.egg-info
 
-# stuff that is likely to exist when you run a server locally
-/*.db
-/*.log
-/*.log.*
-/*.log.config
-/*.pid
-/.python-version
-/*.signing.key
-/env/
-/.venv*/
-/homeserver*.yaml
-/logs
-/media_store/
-/uploads
+cmdclient_config.json
+homeserver*.db
+homeserver*.log
+homeserver*.log.*
+homeserver*.pid
+homeserver*.yaml
 
-# IDEs
-/.idea/
-/.ropeproject/
-/.vscode/
+*.signing.key
+*.tls.crt
+*.tls.dh
+*.tls.key
 
-# build products
-!/.coveragerc
-/.coverage*
-/.mypy_cache/
-/.tox
-/.tox-pg-container
-/build/
-/coverage.*
-/dist/
-/docs/build/
-/htmlcov
-/pip-wheel-metadata/
+.coverage
+htmlcov
 
-# docs
-book/
+demo/*/*.db
+demo/*/*.log
+demo/*/*.log.*
+demo/*/*.pid
+demo/media_store.*
+demo/etc
 
-# complement
-/complement-master
-/master.tar.gz
+uploads
+cache
+
+.idea/
+media_store/
+
+*.tac
+
+build/
+venv/
+venv*/
+*venv/
+
+localhost-800*/
+static/client/register/register_config.js
+.tox
+
+env/
+*.config
+
+.vscode/
+.ropeproject/

.travis.yml

@@ -0,0 +1,73 @@
+sudo: false
+language: python
+
+cache:
+  directories:
+    # we only bother to cache the wheels; parts of the http cache get
+    # invalidated every build (because they get served with a max-age of 600
+    # seconds), which means that we end up re-uploading the whole cache for
+    # every build, which is time-consuming In any case, it's not obvious that
+    # downloading the cache from S3 would be much faster than downloading the
+    # originals from pypi.
+    #
+    - $HOME/.cache/pip/wheels
+
+# don't clone the whole repo history, one commit will do
+git:
+  depth: 1
+
+# only build branches we care about (PRs are built seperately)
+branches:
+  only:
+    - master
+    - develop
+    - /^release-v/
+
+# When running the tox environments that call Twisted Trial, we can pass the -j
+# flag to run the tests concurrently. We set this to 2 for CPU bound tests
+# (SQLite) and 4 for I/O bound tests (PostgreSQL).
+matrix:
+  fast_finish: true
+  include:
+  - python: 2.7
+    env: TOX_ENV=packaging
+
+  - python: 3.6
+    env: TOX_ENV="pep8,check_isort"
+
+  - python: 2.7
+    env: TOX_ENV=py27 TRIAL_FLAGS="-j 2"
+
+  - python: 2.7
+    env: TOX_ENV=py27-old TRIAL_FLAGS="-j 2"
+
+  - python: 2.7
+    env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
+    services:
+      - postgresql
+
+  - python: 3.5
+    env: TOX_ENV=py35 TRIAL_FLAGS="-j 2"
+
+  - python: 3.6
+    env: TOX_ENV=py36 TRIAL_FLAGS="-j 2"
+
+  - python: 3.6
+    env: TOX_ENV=py36-postgres TRIAL_FLAGS="-j 4"
+    services:
+      - postgresql
+
+  - # we only need to check for the newsfragment if it's a PR build
+    if: type = pull_request
+    python: 3.6
+    env: TOX_ENV=check-newsfragment
+    script:
+      - git remote set-branches --add origin develop
+      - git fetch origin develop
+      - tox -e $TOX_ENV
+
+install:
+  - pip install tox
+
+script:
+  - tox -e $TOX_ENV

AUTHORS.rst

@@ -1,8 +1,34 @@
-The following is an incomplete list of people outside the core team who have
-contributed to Synapse. It is no longer maintained: more recent contributions
-are listed in the `changelog <CHANGES.md>`_.
+Erik Johnston <erik at matrix.org>
+ * HS core
+ * Federation API impl
 
-----
+Mark Haines <mark at matrix.org>
+ * HS core
+ * Crypto
+ * Content repository
+ * CS v2 API impl
+
+Kegan Dougal <kegan at matrix.org>
+ * HS core
+ * CS v1 API impl
+ * AS API impl
+
+Paul "LeoNerd" Evans <paul at matrix.org>
+ * HS core
+ * Presence
+ * Typing Notifications
+ * Performance metrics and caching layer
+
+Dave Baker <dave at matrix.org>
+ * Push notifications
+ * Auth CS v2 impl
+
+Matthew Hodgson <matthew at matrix.org>
+ * General doc & housekeeping
+ * Vertobot/vertobridge matrix<->verto PoC
+
+Emmanuel Rohee <manu at matrix.org>
+ * Supporting iOS clients (testability and fallback registration)
 
 Turned to Dust <dwinslow86 at gmail.com>
  * ArchLinux installation instructions
@@ -36,16 +62,7 @@ Christoph Witzany <christoph at web.crofting.com>
  * Add LDAP support for authentication
 
 Pierre Jaury <pierre at jaury.eu>
- * Docker packaging
+* Docker packaging
 
 Serban Constantin <serban.constantin at gmail dot com>
- * Small bug fix
-
-Joseph Weston <joseph at weston.cloud>
- * Add admin API for querying HS version
-
-Benjamin Saunders <ben.e.saunders at gmail dot com>
- * Documentation improvements
-
-Werner Sembach <werner.sembach at fau dot de>
- * Automatically remove a group/community when it is empty
+ * Small bug fix

CONTRIBUTING.md

@@ -1,3 +0,0 @@
-# Welcome to Synapse
-
-Please see the [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html) in our rendered documentation.

CONTRIBUTING.rst

@@ -0,0 +1,169 @@
+Contributing code to Matrix
+===========================
+
+Everyone is welcome to contribute code to Matrix
+(https://github.com/matrix-org), provided that they are willing to license
+their contributions under the same license as the project itself. We follow a
+simple 'inbound=outbound' model for contributions: the act of submitting an
+'inbound' contribution means that the contributor agrees to license the code
+under the same terms as the project's overall 'outbound' license - in our
+case, this is almost always Apache Software License v2 (see LICENSE).
+
+How to contribute
+~~~~~~~~~~~~~~~~~
+
+The preferred and easiest way to contribute changes to Matrix is to fork the
+relevant project on github, and then create a pull request to ask us to pull
+your changes into our repo
+(https://help.github.com/articles/using-pull-requests/)
+
+**The single biggest thing you need to know is: please base your changes on
+the develop branch - /not/ master.**
+
+We use the master branch to track the most recent release, so that folks who
+blindly clone the repo and automatically check out master get something that
+works. Develop is the unstable branch where all the development actually
+happens: the workflow is that contributors should fork the develop branch to
+make a 'feature' branch for a particular contribution, and then make a pull
+request to merge this back into the matrix.org 'official' develop branch. We
+use github's pull request workflow to review the contribution, and either ask
+you to make any refinements needed or merge it and make them ourselves. The
+changes will then land on master when we next do a release.
+
+We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Travis CI 
+<https://travis-ci.org/matrix-org/synapse>`_ for continuous integration. All
+pull requests to synapse get automatically tested by Travis and CircleCI.
+If your change breaks the build, this will be shown in GitHub, so please
+keep an eye on the pull request for feedback.
+
+To run unit tests in a local development environment, you can use:
+
+- ``tox -e py27`` (requires tox to be installed by ``pip install tox``) for
+  SQLite-backed Synapse on Python 2.7.
+- ``tox -e py35`` for SQLite-backed Synapse on Python 3.5.
+- ``tox -e py36`` for SQLite-backed Synapse on Python 3.6.
+- ``tox -e py27-postgres`` for PostgreSQL-backed Synapse on Python 2.7
+  (requires a running local PostgreSQL with access to create databases).
+- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 2.7
+  (requires Docker). Entirely self-contained, recommended if you don't want to
+  set up PostgreSQL yourself.
+
+Docker images are available for running the integration tests (SyTest) locally,
+see the `documentation in the SyTest repo
+<https://github.com/matrix-org/sytest/blob/develop/docker/README.md>`_ for more
+information.
+
+Code style
+~~~~~~~~~~
+
+All Matrix projects have a well-defined code-style - and sometimes we've even
+got as far as documenting it... For instance, synapse's code style doc lives
+at https://github.com/matrix-org/synapse/tree/master/docs/code_style.rst.
+
+Please ensure your changes match the cosmetic style of the existing project,
+and **never** mix cosmetic and functional changes in the same commit, as it
+makes it horribly hard to review otherwise.
+
+Changelog
+~~~~~~~~~
+
+All changes, even minor ones, need a corresponding changelog / newsfragment
+entry. These are managed by Towncrier
+(https://github.com/hawkowl/towncrier).
+
+To create a changelog entry, make a new file in the ``changelog.d``
+file named in the format of ``PRnumber.type``. The type can be
+one of ``feature``, ``bugfix``, ``removal`` (also used for
+deprecations), or ``misc`` (for internal-only changes). The content of
+the file is your changelog entry, which can contain Markdown
+formatting. Adding credits to the changelog is encouraged, we value
+your contributions and would like to have you shouted out in the
+release notes!
+
+For example, a fix in PR #1234 would have its changelog entry in
+``changelog.d/1234.bugfix``, and contain content like "The security levels of
+Florbs are now validated when recieved over federation. Contributed by Jane
+Matrix".
+
+Attribution
+~~~~~~~~~~~
+
+Everyone who contributes anything to Matrix is welcome to be listed in the
+AUTHORS.rst file for the project in question. Please feel free to include a
+change to AUTHORS.rst in your pull request to list yourself and a short
+description of the area(s) you've worked on. Also, we sometimes have swag to
+give away to contributors - if you feel that Matrix-branded apparel is missing
+from your life, please mail us your shipping address to matrix at matrix.org and
+we'll try to fix it :)
+
+Sign off
+~~~~~~~~
+
+In order to have a concrete record that your contribution is intentional
+and you agree to license it under the same terms as the project's license, we've adopted the
+same lightweight approach that the Linux Kernel
+(https://www.kernel.org/doc/Documentation/SubmittingPatches), Docker
+(https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other
+projects use: the DCO (Developer Certificate of Origin:
+http://developercertificate.org/). This is a simple declaration that you wrote
+the contribution or otherwise have the right to contribute it to Matrix::
+
+    Developer Certificate of Origin
+    Version 1.1
+
+    Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
+    660 York Street, Suite 102,
+    San Francisco, CA 94110 USA
+
+    Everyone is permitted to copy and distribute verbatim copies of this
+    license document, but changing it is not allowed.
+
+    Developer's Certificate of Origin 1.1
+
+    By making a contribution to this project, I certify that:
+
+    (a) The contribution was created in whole or in part by me and I
+        have the right to submit it under the open source license
+        indicated in the file; or
+
+    (b) The contribution is based upon previous work that, to the best
+        of my knowledge, is covered under an appropriate open source
+        license and I have the right under that license to submit that
+        work with modifications, whether created in whole or in part
+        by me, under the same open source license (unless I am
+        permitted to submit under a different license), as indicated
+        in the file; or
+
+    (c) The contribution was provided directly to me by some other
+        person who certified (a), (b) or (c) and I have not modified
+        it.
+
+    (d) I understand and agree that this project and the contribution
+        are public and that a record of the contribution (including all
+        personal information I submit with it, including my sign-off) is
+        maintained indefinitely and may be redistributed consistent with
+        this project or the open source license(s) involved.
+
+If you agree to this for your contribution, then all that's needed is to
+include the line in your commit or pull request comment::
+
+    Signed-off-by: Your Name <your@email.example.org>
+
+We accept contributions under a legally identifiable name, such as
+your name on government documentation or common-law names (names
+claimed by legitimate usage or repute). Unfortunately, we cannot
+accept anonymous contributions at this time.
+
+Git allows you to add this signoff automatically when using the ``-s``
+flag to ``git commit``, which uses the name and email set in your
+``user.name`` and ``user.email`` git configs.
+
+Conclusion
+~~~~~~~~~~
+
+That's it!  Matrix is a very open and collaborative project as you might expect
+given our obsession with open communication.  If we're going to successfully
+matrix together all the fragmented communication technologies out there we are
+reliant on contributions and collaboration from the community to do so.  So
+please get involved - and we hope you have as much fun hacking on Matrix as we
+do!

INSTALL.md

@@ -1,7 +0,0 @@
-# Installation Instructions
-
-This document has moved to the
-[Synapse documentation website](https://matrix-org.github.io/synapse/latest/setup/installation.html).
-Please update your links.
-
-The markdown source is available in [docs/setup/installation.md](docs/setup/installation.md).

MANIFEST.in

@@ -8,23 +8,14 @@ include demo/demo.tls.dh
 include demo/*.py
 include demo/*.sh
 
-include synapse/py.typed
-recursive-include synapse/storage *.sql
-recursive-include synapse/storage *.sql.postgres
-recursive-include synapse/storage *.sql.sqlite
-recursive-include synapse/storage *.py
-recursive-include synapse/storage *.txt
-recursive-include synapse/storage *.md
+recursive-include synapse/storage/schema *.sql
+recursive-include synapse/storage/schema *.py
 
 recursive-include docs *
 recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
 recursive-include tests *.py
-recursive-include tests *.pem
-recursive-include tests *.p8
-recursive-include tests *.crt
-recursive-include tests *.key
 
 recursive-include synapse/res *
 recursive-include synapse/static *.css
@@ -32,25 +23,18 @@ recursive-include synapse/static *.gif
 recursive-include synapse/static *.html
 recursive-include synapse/static *.js
 
-exclude .codecov.yml
-exclude .coveragerc
-exclude .dockerignore
-exclude .editorconfig
 exclude Dockerfile
-exclude mypy.ini
-exclude sytest-blacklist
+exclude .dockerignore
 exclude test_postgresql.sh
 
-include book.toml
 include pyproject.toml
 recursive-include changelog.d *
 
-prune .circleci
 prune .github
-prune .ci
-prune contrib
-prune debian
 prune demo/etc
 prune docker
-prune snap
-prune stubs
+prune .circleci
+prune .coveragerc
+
+exclude jenkins*
+recursive-exclude jenkins *.sh

UPGRADE.rst

@@ -1,7 +1,319 @@
 Upgrading Synapse
 =================
 
-This document has moved to the `Synapse documentation website <https://matrix-org.github.io/synapse/latest/upgrade>`_.
-Please update your links.
+Before upgrading check if any special steps are required to upgrade from the
+what you currently have installed to current version of synapse. The extra
+instructions that may be required are listed later in this document.
 
-The markdown source is available in `docs/upgrade.md <docs/upgrade.md>`_.
+1. If synapse was installed in a virtualenv then active that virtualenv before
+   upgrading. If synapse is installed in a virtualenv in ``~/.synapse/`` then
+   run:
+
+   .. code:: bash
+
+       source ~/.synapse/bin/activate
+
+2. If synapse was installed using pip then upgrade to the latest version by
+   running:
+
+   .. code:: bash
+
+       pip install --upgrade --process-dependency-links matrix-synapse
+
+       # restart synapse
+       synctl restart
+
+
+   If synapse was installed using git then upgrade to the latest version by
+   running:
+
+   .. code:: bash
+
+       # Pull the latest version of the master branch.
+       git pull
+       # Update the versions of synapse's python dependencies.
+       python synapse/python_dependencies.py | xargs pip install --upgrade
+
+       # restart synapse
+       ./synctl restart
+
+
+To check whether your update was sucessful, you can check the Server header
+returned by the Client-Server API:
+
+.. code:: bash
+
+    # replace <host.name> with the hostname of your synapse homeserver.
+    # You may need to specify a port (eg, :8448) if your server is not
+    # configured on port 443.
+    curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"
+
+Upgrading to v0.33.7
+====================
+
+This release removes the example email notification templates from
+``res/templates`` (they are now internal to the python package). This should
+only affect you if you (a) deploy your Synapse instance from a git checkout or
+a github snapshot URL, and (b) have email notifications enabled.
+
+If you have email notifications enabled, you should ensure that
+``email.template_dir`` is either configured to point at a directory where you
+have installed customised templates, or leave it unset to use the default
+templates.
+
+Upgrading to v0.27.3
+====================
+
+This release expands the anonymous usage stats sent if the opt-in
+``report_stats`` configuration is set to ``true``. We now capture RSS memory
+and cpu use at a very coarse level. This requires administrators to install
+the optional ``psutil`` python module.
+
+We would appreciate it if you could assist by ensuring this module is available
+and ``report_stats`` is enabled. This will let us see if performance changes to
+synapse are having an impact to the general community.
+
+Upgrading to v0.15.0
+====================
+
+If you want to use the new URL previewing API (/_matrix/media/r0/preview_url)
+then you have to explicitly enable it in the config and update your dependencies
+dependencies.  See README.rst for details.
+
+
+Upgrading to v0.11.0
+====================
+
+This release includes the option to send anonymous usage stats to matrix.org,
+and requires that administrators explictly opt in or out by setting the
+``report_stats`` option to either ``true`` or ``false``.
+
+We would really appreciate it if you could help our project out by reporting
+anonymized usage statistics from your homeserver. Only very basic aggregate
+data (e.g. number of users) will be reported, but it helps us to track the
+growth of the Matrix community, and helps us to make Matrix a success, as well
+as to convince other networks that they should peer with us.
+
+
+Upgrading to v0.9.0
+===================
+
+Application services have had a breaking API change in this version.
+
+They can no longer register themselves with a home server using the AS HTTP API. This
+decision was made because a compromised application service with free reign to register
+any regex in effect grants full read/write access to the home server if a regex of ``.*``
+is used. An attack where a compromised AS re-registers itself with ``.*`` was deemed too
+big of a security risk to ignore, and so the ability to register with the HS remotely has
+been removed.
+
+It has been replaced by specifying a list of application service registrations in
+``homeserver.yaml``::
+
+  app_service_config_files: ["registration-01.yaml", "registration-02.yaml"]
+
+Where ``registration-01.yaml`` looks like::
+
+  url: <String>  # e.g. "https://my.application.service.com"
+  as_token: <String>
+  hs_token: <String>
+  sender_localpart: <String>  # This is a new field which denotes the user_id localpart when using the AS token
+  namespaces:
+    users:
+      - exclusive: <Boolean>
+        regex: <String>  # e.g. "@prefix_.*"
+    aliases:
+      - exclusive: <Boolean>
+        regex: <String>
+    rooms:
+      - exclusive: <Boolean>
+        regex: <String>
+
+Upgrading to v0.8.0
+===================
+
+Servers which use captchas will need to add their public key to::
+
+  static/client/register/register_config.js
+
+    window.matrixRegistrationConfig = {
+        recaptcha_public_key: "YOUR_PUBLIC_KEY"
+    };
+
+This is required in order to support registration fallback (typically used on
+mobile devices).
+
+
+Upgrading to v0.7.0
+===================
+
+New dependencies are:
+
+- pydenticon
+- simplejson
+- syutil
+- matrix-angular-sdk
+
+To pull in these dependencies in a virtual env, run::
+
+    python synapse/python_dependencies.py | xargs -n 1 pip install
+
+Upgrading to v0.6.0
+===================
+
+To pull in new dependencies, run::
+
+    python setup.py develop --user
+
+This update includes a change to the database schema. To upgrade you first need
+to upgrade the database by running::
+
+    python scripts/upgrade_db_to_v0.6.0.py <db> <server_name> <signing_key>
+
+Where `<db>` is the location of the database, `<server_name>` is the
+server name as specified in the synapse configuration, and `<signing_key>` is
+the location of the signing key as specified in the synapse configuration.
+
+This may take some time to complete. Failures of signatures and content hashes
+can safely be ignored.
+
+
+Upgrading to v0.5.1
+===================
+
+Depending on precisely when you installed v0.5.0 you may have ended up with
+a stale release of the reference matrix webclient installed as a python module.
+To uninstall it and ensure you are depending on the latest module, please run::
+
+    $ pip uninstall syweb
+
+Upgrading to v0.5.0
+===================
+
+The webclient has been split out into a seperate repository/pacakage in this
+release. Before you restart your homeserver you will need to pull in the
+webclient package by running::
+
+  python setup.py develop --user
+
+This release completely changes the database schema and so requires upgrading
+it before starting the new version of the homeserver.
+
+The script "database-prepare-for-0.5.0.sh" should be used to upgrade the
+database. This will save all user information, such as logins and profiles,
+but will otherwise purge the database. This includes messages, which
+rooms the home server was a member of and room alias mappings.
+
+If you would like to keep your history, please take a copy of your database
+file and ask for help in #matrix:matrix.org. The upgrade process is,
+unfortunately, non trivial and requires human intervention to resolve any
+resulting conflicts during the upgrade process.
+
+Before running the command the homeserver should be first completely
+shutdown. To run it, simply specify the location of the database, e.g.:
+
+  ./scripts/database-prepare-for-0.5.0.sh "homeserver.db"
+
+Once this has successfully completed it will be safe to restart the
+homeserver. You may notice that the homeserver takes a few seconds longer to
+restart than usual as it reinitializes the database.
+
+On startup of the new version, users can either rejoin remote rooms using room
+aliases or by being reinvited. Alternatively, if any other homeserver sends a
+message to a room that the homeserver was previously in the local HS will
+automatically rejoin the room.
+
+Upgrading to v0.4.0
+===================
+
+This release needs an updated syutil version. Run::
+
+    python setup.py develop
+
+You will also need to upgrade your configuration as the signing key format has
+changed. Run::
+
+    python -m synapse.app.homeserver --config-path <CONFIG> --generate-config
+
+
+Upgrading to v0.3.0
+===================
+
+This registration API now closely matches the login API. This introduces a bit
+more backwards and forwards between the HS and the client, but this improves
+the overall flexibility of the API. You can now GET on /register to retrieve a list
+of valid registration flows. Upon choosing one, they are submitted in the same
+way as login, e.g::
+
+  {
+    type: m.login.password,
+    user: foo,
+    password: bar
+  }
+
+The default HS supports 2 flows, with and without Identity Server email
+authentication. Enabling captcha on the HS will add in an extra step to all
+flows: ``m.login.recaptcha`` which must be completed before you can transition
+to the next stage. There is a new login type: ``m.login.email.identity`` which
+contains the ``threepidCreds`` key which were previously sent in the original
+register request. For more information on this, see the specification.
+
+Web Client
+----------
+
+The VoIP specification has changed between v0.2.0 and v0.3.0. Users should
+refresh any browser tabs to get the latest web client code. Users on
+v0.2.0 of the web client will not be able to call those on v0.3.0 and
+vice versa.
+
+
+Upgrading to v0.2.0
+===================
+
+The home server now requires setting up of SSL config before it can run. To
+automatically generate default config use::
+
+    $ python synapse/app/homeserver.py \
+        --server-name machine.my.domain.name \
+        --bind-port 8448 \
+        --config-path homeserver.config \
+        --generate-config
+
+This config can be edited if desired, for example to specify a different SSL
+certificate to use. Once done you can run the home server using::
+
+    $ python synapse/app/homeserver.py --config-path homeserver.config
+
+See the README.rst for more information.
+
+Also note that some config options have been renamed, including:
+
+- "host" to "server-name"
+- "database" to "database-path"
+- "port" to "bind-port" and "unsecure-port"
+
+
+Upgrading to v0.0.1
+===================
+
+This release completely changes the database schema and so requires upgrading
+it before starting the new version of the homeserver.
+
+The script "database-prepare-for-0.0.1.sh" should be used to upgrade the
+database. This will save all user information, such as logins and profiles,
+but will otherwise purge the database. This includes messages, which
+rooms the home server was a member of and room alias mappings.
+
+Before running the command the homeserver should be first completely
+shutdown. To run it, simply specify the location of the database, e.g.:
+
+  ./scripts/database-prepare-for-0.0.1.sh "homeserver.db"
+
+Once this has successfully completed it will be safe to restart the
+homeserver. You may notice that the homeserver takes a few seconds longer to
+restart than usual as it reinitializes the database.
+
+On startup of the new version, users can either rejoin remote rooms using room
+aliases or by being reinvited. Alternatively, if any other homeserver sends a
+message to a room that the homeserver was previously in the local HS will
+automatically rejoin the room.

book.toml

@@ -1,39 +0,0 @@
-# Documentation for possible options in this file is at
-# https://rust-lang.github.io/mdBook/format/config.html
-[book]
-title = "Synapse"
-authors = ["The Matrix.org Foundation C.I.C."]
-language = "en"
-multilingual = false
-
-# The directory that documentation files are stored in
-src = "docs"
-
-[build]
-# Prevent markdown pages from being automatically generated when they're
-# linked to in SUMMARY.md
-create-missing = false
-
-[output.html]
-# The URL visitors will be directed to when they try to edit a page
-edit-url-template = "https://github.com/matrix-org/synapse/edit/develop/{path}"
-
-# Remove the numbers that appear before each item in the sidebar, as they can
-# get quite messy as we nest deeper
-no-section-label = true
-
-# The source code URL of the repository
-git-repository-url = "https://github.com/matrix-org/synapse"
-
-# The path that the docs are hosted on
-site-url = "/synapse/"
-
-# Additional HTML, JS, CSS that's injected into each page of the book.
-# More information available in docs/website_files/README.md
-additional-css = [
-    "docs/website_files/table-of-contents.css",
-    "docs/website_files/remove-nav-buttons.css",
-    "docs/website_files/indent-section-headers.css",
-]
-additional-js = ["docs/website_files/table-of-contents.js"]
-theme = "docs/website_files/theme"

changelog.d/11530.bugfix

@@ -1,2 +0,0 @@
-Fix a long-standing issue which could cause Synapse to incorrectly accept data in the unsigned field of events
-received over federation.

changelog.d/11561.feature

@@ -1 +0,0 @@
-Add `track_puppeted_user_ips` config flag to record client IP addresses against puppeted users, and include the puppeted users in monthly active user counts.

changelog.d/11576.feature

@@ -1 +0,0 @@
-Remove the `"password_hash"` field from the response dictionaries of the [Users Admin API](https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html).

changelog.d/11577.feature

@@ -1 +0,0 @@
-Include whether the requesting user has participated in a thread when generating a summary for [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440).

changelog.d/11583.bugfix

@@ -1 +0,0 @@
-Fix a performance regression in `/sync` handling, introduced in 1.49.0.

changelog.d/11587.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug where Synapse wouldn't cache a response indicating that a remote user has no devices.

changelog.d/11593.bugfix

@@ -1 +0,0 @@
-Fix an error in to get federation status of a destination server even if no error has occurred. This admin API was new introduced in Synapse 1.49.0.

changelog.d/11612.misc

@@ -1 +0,0 @@
-Avoid database access in the JSON serialization process.

changelog.d/11659.bugfix

@@ -1 +0,0 @@
-Include the bundled aggregations in the `/sync` response, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675).

changelog.d/11667.bugfix

@@ -1 +0,0 @@
-Fix `/_matrix/client/v1/room/{roomId}/hierarchy` endpoint returning incorrect fields which have been present since Synapse 1.49.0.

changelog.d/11669.bugfix

@@ -1 +0,0 @@
-Fix preview of some gif URLs (like tenor.com). Contributed by Philippe Daouadi.

changelog.d/11672.feature

@@ -1 +0,0 @@
-Return an `M_FORBIDDEN` error code instead of `M_UNKNOWN` when a spam checker module prevents a user from creating a room.

changelog.d/11675.feature

@@ -1 +0,0 @@
-Add a flag to the `synapse_review_recent_signups` script to ignore and filter appservice users.

changelog.d/11682.removal

@@ -1 +0,0 @@
-Remove the unstable `/send_relation` endpoint.

changelog.d/11685.misc

@@ -1 +0,0 @@
-Run `pyupgrade --py37-plus --keep-percent-format` on Synapse.

changelog.d/11686.doc

@@ -1 +0,0 @@
-Warn against using a Let's Encrypt certificate for TLS/DTLS TURN server client connections, and suggest using ZeroSSL certificate instead. This bypasses client-side connectivity errors caused by WebRTC libraries that reject Let's Encrypt certificates. Contibuted by @AndrewFerr.

changelog.d/11691.misc

@@ -1 +0,0 @@
-Use buildkit's cache feature to speed up docker builds.

changelog.d/11692.misc

@@ -1 +0,0 @@
-Use `auto_attribs` and native type hints for attrs classes.

changelog.d/11693.misc

@@ -1 +0,0 @@
-Remove debug logging for #4422, which has been closed since Synapse 0.99.

changelog.d/11695.bugfix

@@ -1 +0,0 @@
-Fix a bug where the only the first 50 rooms from a space were returned from the `/hierarchy` API. This has existed since the introduction of the API in Synapse v1.41.0.

changelog.d/11699.misc

@@ -1 +0,0 @@
-Remove fallback code for Python 2.

changelog.d/11701.misc

@@ -1 +0,0 @@
-Add a test for [an edge case](https://github.com/matrix-org/synapse/pull/11532#discussion_r769104461) in the `/sync` logic.

changelog.d/11702.misc

@@ -1 +0,0 @@
-Add the option to write sqlite test dbs to disk when running tests.

changelog.d/11707.misc

@@ -1 +0,0 @@
-Improve Complement test output for Gitub Actions.

changelog.d/11710.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in Synapse v1.18.0 where password reset and address validation emails would not be sent if their subject was configured to use the 'app' template variable. Contributed by @br4nnigan.

changelog.d/11714.misc

@@ -1 +0,0 @@
-Fix a typechecker problem related to our (ab)use of `nacl.signing.SigningKey`s.

changelog.d/11715.doc

@@ -1 +0,0 @@
-Document the new `SYNAPSE_TEST_PERSIST_SQLITE_DB` environment variable in the contributing guide.

changelog.d/11716.misc

@@ -1 +0,0 @@
-Fix docstring on `add_account_data_for_user`.

changelog.d/11718.misc

@@ -1 +0,0 @@
-Complement environment variable name change and update `.gitignore`.

changelog.d/11723.misc

@@ -1 +0,0 @@
-Simplify calculation of prometheus metrics for garbage collection.

changelog.d/11724.misc

@@ -1 +0,0 @@
-Improve accuracy of `python_twisted_reactor_tick_time` prometheus metric.

changelog.d/11724.removal

@@ -1 +0,0 @@
-Remove `python_twisted_reactor_pending_calls` prometheus metric.

changelog.d/11725.doc

@@ -1 +0,0 @@
-Document that now the minimum supported PostgreSQL version is 10.

changelog.d/11735.doc

@@ -1 +0,0 @@
-Fix typo in demo docs: differnt.

changelog.d/11737.bugfix

@@ -1 +0,0 @@
-Make the list rooms admin api sort stable. Contributed by Daniël Sonck.

changelog.d/11739.doc

@@ -1 +0,0 @@
-Update room spec url in config files.

changelog.d/11740.doc

@@ -1 +0,0 @@
-Mention python3-venv and libpq-dev dependencies in contribution guide.

changelog.d/11742.misc

@@ -1 +0,0 @@
-Minor efficiency improvements when inserting many values into the database.

changelog.d/11744.misc

@@ -1 +0,0 @@
-Invite PR authors to give themselves credit in the changelog.

changelog.d/11745.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in Synapse v1.18.0 where password reset and address validation emails would not be sent if their subject was configured to use the 'app' template variable. Contributed by @br4nnigan.

changelog.d/11749.feature

@@ -1 +0,0 @@
-Add `track_puppeted_user_ips` config flag to record client IP addresses against puppeted users, and include the puppeted users in monthly active user counts.

changelog.d/11755.doc

@@ -1 +0,0 @@
-Update documentation for configuring login with facebook.

changelog.d/11757.feature

@@ -1 +0,0 @@
-Add `track_puppeted_user_ips` config flag to record client IP addresses against puppeted users, and include the puppeted users in monthly active user counts.

changelog.d/11761.misc

@@ -1 +0,0 @@
-Remove `log_function` utility function and its uses.

changelog.d/11768.misc

@@ -1 +0,0 @@
-Use `auto_attribs` and native type hints for attrs classes.

changelog.d/3830.feature

@@ -0,0 +1 @@
+Add option to track MAU stats (but not limit people)

changelog.d/4176.bugfix

@@ -0,0 +1 @@
+The media repository now no longer fails to decode UTF-8 filenames when downloading remote media.

changelog.d/4180.misc

@@ -0,0 +1 @@
+A coveragerc file, as well as the py36-coverage tox target, have been added.

changelog.d/4182.misc

@@ -0,0 +1 @@
+Add a GitHub pull request template and add multiple issue templates

changelog.d/4183.bugfix

@@ -0,0 +1 @@
+URL previews now correctly decode non-UTF-8 text if the header contains a `<meta http-equiv="Content-Type"` header.

changelog.d/4188.misc

@@ -0,0 +1 @@
+Update README to reflect the fact that #1491 is fixed

changelog.d/4192.bugfix

@@ -0,0 +1 @@
+Fix an issue where public consent URLs had two slashes.

changelog.d/4193.misc

@@ -0,0 +1 @@
+Add missing `jpeg` package prerequisite for OpenBSD in README.

changelog.d/4197.bugfix

@@ -0,0 +1 @@
+Fallback auth now accepts the session parameter on Python 3.

changelog.d/4200.misc

@@ -0,0 +1 @@
+Add a note saying you need to manually reclaim disk space after using the Purge History API

changelog.d/4204.misc

@@ -0,0 +1 @@
+Fix logcontext leaks in EmailPusher and in tests

changelog.d/4207.bugfix

@@ -0,0 +1 @@
+Remove riot.im from the list of trusted Identity Servers in the default configuration

contrib/cmdclient/console.py

@@ -15,6 +15,10 @@
 # limitations under the License.
 
 """ Starts a synapse client console. """
+
+from twisted.internet import reactor, defer, threads
+from http import TwistedHttpClient
+
 import argparse
 import cmd
 import getpass
@@ -23,22 +27,18 @@ import shlex
 import sys
 import time
 import urllib
-from http import TwistedHttpClient
-from typing import Optional
-
-import nacl.encoding
-import nacl.signing
 import urlparse
-from signedjson.sign import SignatureVerifyException, verify_signed_json
 
-from twisted.internet import defer, reactor, threads
+import nacl.signing
+import nacl.encoding
+
+from signedjson.sign import verify_signed_json, SignatureVerifyException
 
 CONFIG_JSON = "cmdclient_config.json"
 
-# TODO: The concept of trusted identity servers has been deprecated. This option and checks
-#  should be removed
-TRUSTED_ID_SERVERS = ["localhost:8001"]
-
+TRUSTED_ID_SERVERS = [
+    'localhost:8001'
+]
 
 class SynapseCmd(cmd.Cmd):
 
@@ -58,7 +58,7 @@ class SynapseCmd(cmd.Cmd):
             "token": token,
             "verbose": "on",
             "complete_usernames": "on",
-            "send_delivery_receipts": "on",
+            "send_delivery_receipts": "on"
         }
         self.path_prefix = "/_matrix/client/api/v1"
         self.event_stream_token = "END"
@@ -93,7 +93,7 @@ class SynapseCmd(cmd.Cmd):
         return self.config["user"].split(":")[1]
 
     def do_config(self, line):
-        """Show the config for this client: "config"
+        """ Show the config for this client: "config"
         Edit a key value mapping: "config key value" e.g. "config token 1234"
         Config variables:
             user: The username to auth with.
@@ -109,7 +109,7 @@ class SynapseCmd(cmd.Cmd):
         by using $. E.g. 'config roomid room1' then 'raw get /rooms/$roomid'.
         """
         if len(line) == 0:
-            print(json.dumps(self.config, indent=4))
+            print json.dumps(self.config, indent=4)
             return
 
         try:
@@ -119,11 +119,12 @@ class SynapseCmd(cmd.Cmd):
             config_rules = [  # key, valid_values
                 ("verbose", ["on", "off"]),
                 ("complete_usernames", ["on", "off"]),
-                ("send_delivery_receipts", ["on", "off"]),
+                ("send_delivery_receipts", ["on", "off"])
             ]
             for key, valid_vals in config_rules:
                 if key == args["key"] and args["val"] not in valid_vals:
-                    print("%s value must be one of %s" % (args["key"], valid_vals))
+                    print "%s value must be one of %s" % (args["key"],
+                                                          valid_vals)
                     return
 
             # toggle the http client verbosity
@@ -132,11 +133,11 @@ class SynapseCmd(cmd.Cmd):
 
             # assign the new config
             self.config[args["key"]] = args["val"]
-            print(json.dumps(self.config, indent=4))
+            print json.dumps(self.config, indent=4)
 
             save_config(self.config)
         except Exception as e:
-            print(e)
+            print e
 
     def do_register(self, line):
         """Registers for a new account: "register <userid> <noupdate>"
@@ -152,32 +153,33 @@ class SynapseCmd(cmd.Cmd):
             pwd = getpass.getpass("Type a password for this user: ")
             pwd2 = getpass.getpass("Retype the password: ")
             if pwd != pwd2 or len(pwd) == 0:
-                print("Password mismatch.")
+                print "Password mismatch."
                 pwd = None
             else:
                 password = pwd
 
-        body = {"type": "m.login.password"}
+        body = {
+            "type": "m.login.password"
+        }
         if "userid" in args:
             body["user"] = args["userid"]
         if password:
             body["password"] = password
 
-        reactor.callFromThread(self._do_register, body, "noupdate" not in args)
+        reactor.callFromThread(self._do_register, body,
+                               "noupdate" not in args)
 
     @defer.inlineCallbacks
     def _do_register(self, data, update_config):
         # check the registration flows
         url = self._url() + "/register"
         json_res = yield self.http_client.do_request("GET", url)
-        print(json.dumps(json_res, indent=4))
+        print json.dumps(json_res, indent=4)
 
         passwordFlow = None
         for flow in json_res["flows"]:
-            if flow["type"] == "m.login.recaptcha" or (
-                "stages" in flow and "m.login.recaptcha" in flow["stages"]
-            ):
-                print("Unable to register: Home server requires captcha.")
+            if flow["type"] == "m.login.recaptcha" or ("stages" in flow and "m.login.recaptcha" in flow["stages"]):
+                print "Unable to register: Home server requires captcha."
                 return
             if flow["type"] == "m.login.password" and "stages" not in flow:
                 passwordFlow = flow
@@ -187,7 +189,7 @@ class SynapseCmd(cmd.Cmd):
             return
 
         json_res = yield self.http_client.do_request("POST", url, data=data)
-        print(json.dumps(json_res, indent=4))
+        print json.dumps(json_res, indent=4)
         if update_config and "user_id" in json_res:
             self.config["user"] = json_res["user_id"]
             self.config["token"] = json_res["access_token"]
@@ -199,7 +201,9 @@ class SynapseCmd(cmd.Cmd):
         """
         try:
             args = self._parse(line, ["user_id"], force_keys=True)
-            can_login = threads.blockingCallFromThread(reactor, self._check_can_login)
+            can_login = threads.blockingCallFromThread(
+                reactor,
+                self._check_can_login)
             if can_login:
                 p = getpass.getpass("Enter your password: ")
                 user = args["user_id"]
@@ -207,25 +211,29 @@ class SynapseCmd(cmd.Cmd):
                     domain = self._domain()
                     if domain:
                         user = "@" + user + ":" + domain
-
+                
                 reactor.callFromThread(self._do_login, user, p)
-                # print " got %s " % p
+                #print " got %s " % p
         except Exception as e:
-            print(e)
+            print e
 
     @defer.inlineCallbacks
     def _do_login(self, user, password):
         path = "/login"
-        data = {"user": user, "password": password, "type": "m.login.password"}
+        data = {
+            "user": user,
+            "password": password,
+            "type": "m.login.password"
+        }
         url = self._url() + path
         json_res = yield self.http_client.do_request("POST", url, data=data)
-        print(json_res)
+        print json_res
 
         if "access_token" in json_res:
             self.config["user"] = user
             self.config["token"] = json_res["access_token"]
             save_config(self.config)
-            print("Login successful.")
+            print "Login successful."
 
     @defer.inlineCallbacks
     def _check_can_login(self):
@@ -234,19 +242,18 @@ class SynapseCmd(cmd.Cmd):
         # submitting!
         url = self._url() + path
         json_res = yield self.http_client.do_request("GET", url)
-        print(json_res)
+        print json_res
 
         if "flows" not in json_res:
-            print("Failed to find any login flows.")
+            print "Failed to find any login flows."
             defer.returnValue(False)
 
-        flow = json_res["flows"][0]  # assume first is the one we want.
-        if "type" not in flow or "m.login.password" != flow["type"] or "stages" in flow:
+        flow = json_res["flows"][0] # assume first is the one we want.
+        if ("type" not in flow or "m.login.password" != flow["type"] or
+                "stages" in flow):
             fallback_url = self._url() + "/login/fallback"
-            print(
-                "Unable to login via the command line client. Please visit "
-                "%s to login." % fallback_url
-            )
+            print ("Unable to login via the command line client. Please visit "
+                "%s to login." % fallback_url)
             defer.returnValue(False)
         defer.returnValue(True)
 
@@ -256,34 +263,21 @@ class SynapseCmd(cmd.Cmd):
         <clientSecret> A string of characters generated when requesting an email that you'll supply in subsequent calls to identify yourself
         <sendAttempt> The number of times the user has requested an email. Leave this the same between requests to retry the request at the transport level. Increment it to request that the email be sent again.
         """
-        args = self._parse(line, ["address", "clientSecret", "sendAttempt"])
+        args = self._parse(line, ['address', 'clientSecret', 'sendAttempt'])
 
-        postArgs = {
-            "email": args["address"],
-            "clientSecret": args["clientSecret"],
-            "sendAttempt": args["sendAttempt"],
-        }
+        postArgs = {'email': args['address'], 'clientSecret': args['clientSecret'], 'sendAttempt': args['sendAttempt']}
 
         reactor.callFromThread(self._do_emailrequest, postArgs)
 
     @defer.inlineCallbacks
     def _do_emailrequest(self, args):
-        # TODO: Update to use v2 Identity Service API endpoint
-        url = (
-            self._identityServerUrl()
-            + "/_matrix/identity/api/v1/validate/email/requestToken"
-        )
+        url = self._identityServerUrl()+"/_matrix/identity/api/v1/validate/email/requestToken"
 
-        json_res = yield self.http_client.do_request(
-            "POST",
-            url,
-            data=urllib.urlencode(args),
-            jsonreq=False,
-            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
-        )
-        print(json_res)
-        if "sid" in json_res:
-            print("Token sent. Your session ID is %s" % (json_res["sid"]))
+        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
+                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
+        print json_res
+        if 'sid' in json_res:
+            print "Token sent. Your session ID is %s" % (json_res['sid'])
 
     def do_emailvalidate(self, line):
         """Validate and associate a third party ID
@@ -291,58 +285,39 @@ class SynapseCmd(cmd.Cmd):
         <token> The token sent to your third party identifier address
         <clientSecret> The same clientSecret you supplied in requestToken
         """
-        args = self._parse(line, ["sid", "token", "clientSecret"])
+        args = self._parse(line, ['sid', 'token', 'clientSecret'])
 
-        postArgs = {
-            "sid": args["sid"],
-            "token": args["token"],
-            "clientSecret": args["clientSecret"],
-        }
+        postArgs = { 'sid' : args['sid'], 'token' : args['token'], 'clientSecret': args['clientSecret'] }
 
         reactor.callFromThread(self._do_emailvalidate, postArgs)
 
     @defer.inlineCallbacks
     def _do_emailvalidate(self, args):
-        # TODO: Update to use v2 Identity Service API endpoint
-        url = (
-            self._identityServerUrl()
-            + "/_matrix/identity/api/v1/validate/email/submitToken"
-        )
+        url = self._identityServerUrl()+"/_matrix/identity/api/v1/validate/email/submitToken"
 
-        json_res = yield self.http_client.do_request(
-            "POST",
-            url,
-            data=urllib.urlencode(args),
-            jsonreq=False,
-            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
-        )
-        print(json_res)
+        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
+                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
+        print json_res
 
     def do_3pidbind(self, line):
         """Validate and associate a third party ID
         <sid> The session ID (sid) given to you in the response to requestToken
         <clientSecret> The same clientSecret you supplied in requestToken
         """
-        args = self._parse(line, ["sid", "clientSecret"])
+        args = self._parse(line, ['sid', 'clientSecret'])
 
-        postArgs = {"sid": args["sid"], "clientSecret": args["clientSecret"]}
-        postArgs["mxid"] = self.config["user"]
+        postArgs = { 'sid' : args['sid'], 'clientSecret': args['clientSecret'] }
+        postArgs['mxid'] = self.config["user"]
 
         reactor.callFromThread(self._do_3pidbind, postArgs)
 
     @defer.inlineCallbacks
     def _do_3pidbind(self, args):
-        # TODO: Update to use v2 Identity Service API endpoint
-        url = self._identityServerUrl() + "/_matrix/identity/api/v1/3pid/bind"
+        url = self._identityServerUrl()+"/_matrix/identity/api/v1/3pid/bind"
 
-        json_res = yield self.http_client.do_request(
-            "POST",
-            url,
-            data=urllib.urlencode(args),
-            jsonreq=False,
-            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
-        )
-        print(json_res)
+        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
+                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
+        print json_res
 
     def do_join(self, line):
         """Joins a room: "join <roomid>" """
@@ -350,7 +325,7 @@ class SynapseCmd(cmd.Cmd):
             args = self._parse(line, ["roomid"], force_keys=True)
             self._do_membership_change(args["roomid"], "join", self._usr())
         except Exception as e:
-            print(e)
+            print e
 
     def do_joinalias(self, line):
         try:
@@ -358,34 +333,36 @@ class SynapseCmd(cmd.Cmd):
             path = "/join/%s" % urllib.quote(args["roomname"])
             reactor.callFromThread(self._run_and_pprint, "POST", path, {})
         except Exception as e:
-            print(e)
+            print e
 
     def do_topic(self, line):
-        """ "topic [set|get] <roomid> [<newtopic>]"
+        """"topic [set|get] <roomid> [<newtopic>]"
         Set the topic for a room: topic set <roomid> <newtopic>
         Get the topic for a room: topic get <roomid>
         """
         try:
             args = self._parse(line, ["action", "roomid", "topic"])
             if "action" not in args or "roomid" not in args:
-                print("Must specify set|get and a room ID.")
+                print "Must specify set|get and a room ID."
                 return
             if args["action"].lower() not in ["set", "get"]:
-                print("Must specify set|get, not %s" % args["action"])
+                print "Must specify set|get, not %s" % args["action"]
                 return
 
             path = "/rooms/%s/topic" % urllib.quote(args["roomid"])
 
             if args["action"].lower() == "set":
                 if "topic" not in args:
-                    print("Must specify a new topic.")
+                    print "Must specify a new topic."
                     return
-                body = {"topic": args["topic"]}
+                body = {
+                    "topic": args["topic"]
+                }
                 reactor.callFromThread(self._run_and_pprint, "PUT", path, body)
             elif args["action"].lower() == "get":
                 reactor.callFromThread(self._run_and_pprint, "GET", path)
         except Exception as e:
-            print(e)
+            print e
 
     def do_invite(self, line):
         """Invite a user to a room: "invite <userid> <roomid>" """
@@ -396,66 +373,49 @@ class SynapseCmd(cmd.Cmd):
 
             reactor.callFromThread(self._do_invite, args["roomid"], user_id)
         except Exception as e:
-            print(e)
+            print e
 
     @defer.inlineCallbacks
     def _do_invite(self, roomid, userstring):
-        if not userstring.startswith("@") and self._is_on("complete_usernames"):
-            # TODO: Update to use v2 Identity Service API endpoint
-            url = self._identityServerUrl() + "/_matrix/identity/api/v1/lookup"
+        if (not userstring.startswith('@') and
+                    self._is_on("complete_usernames")):
+            url = self._identityServerUrl()+"/_matrix/identity/api/v1/lookup"
 
-            json_res = yield self.http_client.do_request(
-                "GET", url, qparams={"medium": "email", "address": userstring}
-            )
+            json_res = yield self.http_client.do_request("GET", url, qparams={'medium':'email','address':userstring})
 
             mxid = None
 
-            if "mxid" in json_res and "signatures" in json_res:
-                # TODO: Update to use v2 Identity Service API endpoint
-                url = (
-                    self._identityServerUrl()
-                    + "/_matrix/identity/api/v1/pubkey/ed25519"
-                )
+            if 'mxid' in json_res and 'signatures' in json_res:
+                url = self._identityServerUrl()+"/_matrix/identity/api/v1/pubkey/ed25519"
 
                 pubKey = None
                 pubKeyObj = yield self.http_client.do_request("GET", url)
-                if "public_key" in pubKeyObj:
-                    pubKey = nacl.signing.VerifyKey(
-                        pubKeyObj["public_key"], encoder=nacl.encoding.HexEncoder
-                    )
+                if 'public_key' in pubKeyObj:
+                    pubKey = nacl.signing.VerifyKey(pubKeyObj['public_key'], encoder=nacl.encoding.HexEncoder)
                 else:
-                    print("No public key found in pubkey response!")
+                    print "No public key found in pubkey response!"
 
                 sigValid = False
 
                 if pubKey:
-                    for signame in json_res["signatures"]:
+                    for signame in json_res['signatures']:
                         if signame not in TRUSTED_ID_SERVERS:
-                            print(
-                                "Ignoring signature from untrusted server %s"
-                                % (signame)
-                            )
+                            print "Ignoring signature from untrusted server %s" % (signame)
                         else:
                             try:
                                 verify_signed_json(json_res, signame, pubKey)
                                 sigValid = True
-                                print(
-                                    "Mapping %s -> %s correctly signed by %s"
-                                    % (userstring, json_res["mxid"], signame)
-                                )
+                                print "Mapping %s -> %s correctly signed by %s" % (userstring, json_res['mxid'], signame)
                                 break
                             except SignatureVerifyException as e:
-                                print("Invalid signature from %s" % (signame))
-                                print(e)
+                                print "Invalid signature from %s" % (signame)
+                                print e
 
                 if sigValid:
-                    print("Resolved 3pid %s to %s" % (userstring, json_res["mxid"]))
-                    mxid = json_res["mxid"]
+                    print "Resolved 3pid %s to %s" % (userstring, json_res['mxid'])
+                    mxid = json_res['mxid']
                 else:
-                    print(
-                        "Got association for %s but couldn't verify signature"
-                        % (userstring)
-                    )
+                    print "Got association for %s but couldn't verify signature" % (userstring)
 
             if not mxid:
                 mxid = "@" + userstring + ":" + self._domain()
@@ -468,17 +428,18 @@ class SynapseCmd(cmd.Cmd):
             args = self._parse(line, ["roomid"], force_keys=True)
             self._do_membership_change(args["roomid"], "leave", self._usr())
         except Exception as e:
-            print(e)
+            print e
 
     def do_send(self, line):
         """Sends a message. "send <roomid> <body>" """
         args = self._parse(line, ["roomid", "body"])
         txn_id = "txn%s" % int(time.time())
-        path = "/rooms/%s/send/m.room.message/%s" % (
-            urllib.quote(args["roomid"]),
-            txn_id,
-        )
-        body_json = {"msgtype": "m.text", "body": args["body"]}
+        path = "/rooms/%s/send/m.room.message/%s" % (urllib.quote(args["roomid"]),
+                                             txn_id)
+        body_json = {
+            "msgtype": "m.text",
+            "body": args["body"]
+        }
         reactor.callFromThread(self._run_and_pprint, "PUT", path, body_json)
 
     def do_list(self, line):
@@ -491,11 +452,11 @@ class SynapseCmd(cmd.Cmd):
         "list messages <roomid> from=END&to=START&limit=3"
         """
         args = self._parse(line, ["type", "roomid", "qp"])
-        if "type" not in args or "roomid" not in args:
-            print("Must specify type and room ID.")
+        if not "type" in args or not "roomid" in args:
+            print "Must specify type and room ID."
             return
         if args["type"] not in ["members", "messages"]:
-            print("Unrecognised type: %s" % args["type"])
+            print "Unrecognised type: %s" % args["type"]
             return
         room_id = args["roomid"]
         path = "/rooms/%s/%s" % (urllib.quote(room_id), args["type"])
@@ -506,11 +467,12 @@ class SynapseCmd(cmd.Cmd):
                 try:
                     key_value = key_value_str.split("=")
                     qp[key_value[0]] = key_value[1]
-                except Exception:
-                    print("Bad query param: %s" % key_value)
+                except:
+                    print "Bad query param: %s" % key_value
                     return
 
-        reactor.callFromThread(self._run_and_pprint, "GET", path, query_params=qp)
+        reactor.callFromThread(self._run_and_pprint, "GET", path,
+                               query_params=qp)
 
     def do_create(self, line):
         """Creates a room.
@@ -546,22 +508,14 @@ class SynapseCmd(cmd.Cmd):
         args = self._parse(line, ["method", "path", "data"])
         # sanity check
         if "method" not in args or "path" not in args:
-            print("Must specify path and method.")
+            print "Must specify path and method."
             return
 
         args["method"] = args["method"].upper()
-        valid_methods = [
-            "PUT",
-            "GET",
-            "POST",
-            "DELETE",
-            "XPUT",
-            "XGET",
-            "XPOST",
-            "XDELETE",
-        ]
+        valid_methods = ["PUT", "GET", "POST", "DELETE",
+                         "XPUT", "XGET", "XPOST", "XDELETE"]
         if args["method"] not in valid_methods:
-            print("Unsupported method: %s" % args["method"])
+            print "Unsupported method: %s" % args["method"]
             return
 
         if "data" not in args:
@@ -570,7 +524,7 @@ class SynapseCmd(cmd.Cmd):
             try:
                 args["data"] = json.loads(args["data"])
             except Exception as e:
-                print("Data is not valid JSON. %s" % e)
+                print "Data is not valid JSON. %s" % e
                 return
 
         qp = {"access_token": self._tok()}
@@ -583,16 +537,13 @@ class SynapseCmd(cmd.Cmd):
                 parsed_url = urlparse.urlparse(args["path"])
                 qp.update(urlparse.parse_qs(parsed_url.query))
                 args["path"] = parsed_url.path
-            except Exception:
+            except:
                 pass
 
-        reactor.callFromThread(
-            self._run_and_pprint,
-            args["method"],
-            args["path"],
-            args["data"],
-            query_params=qp,
-        )
+        reactor.callFromThread(self._run_and_pprint, args["method"],
+                                                     args["path"],
+                                                     args["data"],
+                                                     query_params=qp)
 
     def do_stream(self, line):
         """Stream data from the server: "stream <longpoll timeout ms>" """
@@ -602,31 +553,26 @@ class SynapseCmd(cmd.Cmd):
             try:
                 timeout = int(args["timeout"])
             except ValueError:
-                print("Timeout must be in milliseconds.")
+                print "Timeout must be in milliseconds."
                 return
         reactor.callFromThread(self._do_event_stream, timeout)
 
     @defer.inlineCallbacks
     def _do_event_stream(self, timeout):
-        res = yield defer.ensureDeferred(
-            self.http_client.get_json(
+        res = yield self.http_client.get_json(
                 self._url() + "/events",
                 {
                     "access_token": self._tok(),
                     "timeout": str(timeout),
-                    "from": self.event_stream_token,
-                },
-            )
-        )
-        print(json.dumps(res, indent=4))
+                    "from": self.event_stream_token
+                })
+        print json.dumps(res, indent=4)
 
         if "chunk" in res:
             for event in res["chunk"]:
-                if (
-                    event["type"] == "m.room.message"
-                    and self._is_on("send_delivery_receipts")
-                    and event["user_id"] != self._usr()
-                ):  # not sent by us
+                if (event["type"] == "m.room.message" and
+                        self._is_on("send_delivery_receipts") and
+                        event["user_id"] != self._usr()):  # not sent by us
                     self._send_receipt(event, "d")
 
         # update the position in the stram
@@ -634,28 +580,18 @@ class SynapseCmd(cmd.Cmd):
             self.event_stream_token = res["end"]
 
     def _send_receipt(self, event, feedback_type):
-        path = "/rooms/%s/messages/%s/%s/feedback/%s/%s" % (
-            urllib.quote(event["room_id"]),
-            event["user_id"],
-            event["msg_id"],
-            self._usr(),
-            feedback_type,
-        )
+        path = ("/rooms/%s/messages/%s/%s/feedback/%s/%s" %
+               (urllib.quote(event["room_id"]), event["user_id"], event["msg_id"],
+                self._usr(), feedback_type))
         data = {}
-        reactor.callFromThread(
-            self._run_and_pprint,
-            "PUT",
-            path,
-            data=data,
-            alt_text="Sent receipt for %s" % event["msg_id"],
-        )
+        reactor.callFromThread(self._run_and_pprint, "PUT", path, data=data,
+                               alt_text="Sent receipt for %s" % event["msg_id"])
 
     def _do_membership_change(self, roomid, membership, userid):
-        path = "/rooms/%s/state/m.room.member/%s" % (
-            urllib.quote(roomid),
-            urllib.quote(userid),
-        )
-        data = {"membership": membership}
+        path = "/rooms/%s/state/m.room.member/%s" % (urllib.quote(roomid), urllib.quote(userid))
+        data = {
+            "membership": membership
+        }
         reactor.callFromThread(self._run_and_pprint, "PUT", path, data=data)
 
     def do_displayname(self, line):
@@ -691,7 +627,7 @@ class SynapseCmd(cmd.Cmd):
         self._do_presence_state(2, line)
 
     def _parse(self, line, keys, force_keys=False):
-        """Parses the given line.
+        """ Parses the given line.
 
         Args:
             line : The line to parse
@@ -708,21 +644,16 @@ class SynapseCmd(cmd.Cmd):
         for i, arg in enumerate(line_args):
             for config_key in self.config:
                 if ("$" + config_key) in arg:
-                    arg = arg.replace("$" + config_key, self.config[config_key])
+                    arg = arg.replace("$" + config_key,
+                                      self.config[config_key])
             line_args[i] = arg
 
         return dict(zip(keys, line_args))
 
     @defer.inlineCallbacks
-    def _run_and_pprint(
-        self,
-        method,
-        path,
-        data=None,
-        query_params: Optional[dict] = None,
-        alt_text=None,
-    ):
-        """Runs an HTTP request and pretty prints the output.
+    def _run_and_pprint(self, method, path, data=None,
+                        query_params={"access_token": None}, alt_text=None):
+        """ Runs an HTTP request and pretty prints the output.
 
         Args:
             method: HTTP method
@@ -730,37 +661,35 @@ class SynapseCmd(cmd.Cmd):
             data: Raw JSON data if any
             query_params: dict of query parameters to add to the url
         """
-        query_params = query_params or {"access_token": None}
-
         url = self._url() + path
         if "access_token" in query_params:
             query_params["access_token"] = self._tok()
 
-        json_res = yield self.http_client.do_request(
-            method, url, data=data, qparams=query_params
-        )
+        json_res = yield self.http_client.do_request(method, url,
+                                                    data=data,
+                                                    qparams=query_params)
         if alt_text:
-            print(alt_text)
+            print alt_text
         else:
-            print(json.dumps(json_res, indent=4))
+            print json.dumps(json_res, indent=4)
 
 
 def save_config(config):
-    with open(CONFIG_JSON, "w") as out:
+    with open(CONFIG_JSON, 'w') as out:
         json.dump(config, out)
 
 
 def main(server_url, identity_server_url, username, token, config_path):
-    print("Synapse command line client")
-    print("===========================")
-    print("Server: %s" % server_url)
-    print("Type 'help' to get started.")
-    print("Close this console with CTRL+C then CTRL+D.")
+    print "Synapse command line client"
+    print "==========================="
+    print "Server: %s" % server_url
+    print "Type 'help' to get started."
+    print "Close this console with CTRL+C then CTRL+D."
     if not username or not token:
-        print("-  'register <username>' - Register an account")
-        print("-  'stream' - Connect to the event stream")
-        print("-  'create <roomid>' - Create a room")
-        print("-  'send <roomid> <message>' - Send a message")
+        print "-  'register <username>' - Register an account"
+        print "-  'stream' - Connect to the event stream"
+        print "-  'create <roomid>' - Create a room"
+        print "-  'send <roomid> <message>' - Send a message"
     http_client = TwistedHttpClient()
 
     # the command line client
@@ -770,14 +699,14 @@ def main(server_url, identity_server_url, username, token, config_path):
     global CONFIG_JSON
     CONFIG_JSON = config_path  # bit cheeky, but just overwrite the global
     try:
-        with open(config_path, "r") as config:
+        with open(config_path, 'r') as config:
             syn_cmd.config = json.load(config)
             try:
                 http_client.verbose = "on" == syn_cmd.config["verbose"]
-            except Exception:
+            except:
                 pass
-            print("Loaded config from %s" % config_path)
-    except Exception:
+            print "Loaded config from %s" % config_path
+    except:
         pass
 
     # Twisted-specific: Runs the command processor in Twisted's event loop
@@ -787,37 +716,27 @@ def main(server_url, identity_server_url, username, token, config_path):
     reactor.run()
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     parser = argparse.ArgumentParser("Starts a synapse client.")
     parser.add_argument(
-        "-s",
-        "--server",
-        dest="server",
-        default="http://localhost:8008",
-        help="The URL of the home server to talk to.",
-    )
+        "-s", "--server", dest="server", default="http://localhost:8008",
+        help="The URL of the home server to talk to.")
     parser.add_argument(
-        "-i",
-        "--identity-server",
-        dest="identityserver",
-        default="http://localhost:8090",
-        help="The URL of the identity server to talk to.",
-    )
+        "-i", "--identity-server", dest="identityserver", default="http://localhost:8090",
+        help="The URL of the identity server to talk to.")
     parser.add_argument(
-        "-u", "--username", dest="username", help="Your username on the server."
-    )
-    parser.add_argument("-t", "--token", dest="token", help="Your access token.")
+        "-u", "--username", dest="username",
+        help="Your username on the server.")
     parser.add_argument(
-        "-c",
-        "--config",
-        dest="config",
-        default=CONFIG_JSON,
-        help="The location of the config.json file to read from.",
-    )
+        "-t", "--token", dest="token",
+        help="Your access token.")
+    parser.add_argument(
+        "-c", "--config", dest="config", default=CONFIG_JSON,
+        help="The location of the config.json file to read from.")
     args = parser.parse_args()
 
     if not args.server:
-        print("You must supply a server URL to communicate with.")
+        print "You must supply a server URL to communicate with."
         parser.print_help()
         sys.exit(1)
 

contrib/cmdclient/http.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,21 +13,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
-import urllib
-from pprint import pformat
-from typing import Optional
-
-from twisted.internet import defer, reactor
 from twisted.web.client import Agent, readBody
 from twisted.web.http_headers import Headers
+from twisted.internet import defer, reactor
+
+from pprint import pformat
+
+import json
+import urllib
 
 
-class HttpClient:
-    """Interface for talking json over http"""
+class HttpClient(object):
+    """ Interface for talking json over http
+    """
 
     def put_json(self, url, data):
-        """Sends the specifed json data using PUT
+        """ Sends the specifed json data using PUT
 
         Args:
             url (str): The URL to PUT data to.
@@ -40,7 +42,7 @@ class HttpClient:
         pass
 
     def get_json(self, url, args=None):
-        """Gets some json from the given host homeserver and path
+        """ Gets some json from the given host homeserver and path
 
         Args:
             url (str): The URL to GET data from.
@@ -57,7 +59,7 @@ class HttpClient:
 
 
 class TwistedHttpClient(HttpClient):
-    """Wrapper around the twisted HTTP client api.
+    """ Wrapper around the twisted HTTP client api.
 
     Attributes:
         agent (twisted.web.client.Agent): The twisted Agent used to send the
@@ -70,7 +72,9 @@ class TwistedHttpClient(HttpClient):
     @defer.inlineCallbacks
     def put_json(self, url, data):
         response = yield self._create_put_request(
-            url, data, headers_dict={"Content-Type": ["application/json"]}
+            url,
+            data,
+            headers_dict={"Content-Type": ["application/json"]}
         )
         body = yield readBody(response)
         defer.returnValue((response.code, body))
@@ -85,46 +89,45 @@ class TwistedHttpClient(HttpClient):
         body = yield readBody(response)
         defer.returnValue(json.loads(body))
 
-    def _create_put_request(self, url, json_data, headers_dict: Optional[dict] = None):
-        """Wrapper of _create_request to issue a PUT request"""
-        headers_dict = headers_dict or {}
+    def _create_put_request(self, url, json_data, headers_dict={}):
+        """ Wrapper of _create_request to issue a PUT request
+        """
 
         if "Content-Type" not in headers_dict:
-            raise defer.error(RuntimeError("Must include Content-Type header for PUTs"))
+            raise defer.error(
+                RuntimeError("Must include Content-Type header for PUTs"))
 
         return self._create_request(
-            "PUT", url, producer=_JsonProducer(json_data), headers_dict=headers_dict
+            "PUT",
+            url,
+            producer=_JsonProducer(json_data),
+            headers_dict=headers_dict
         )
 
-    def _create_get_request(self, url, headers_dict: Optional[dict] = None):
-        """Wrapper of _create_request to issue a GET request"""
-        return self._create_request("GET", url, headers_dict=headers_dict or {})
+    def _create_get_request(self, url, headers_dict={}):
+        """ Wrapper of _create_request to issue a GET request
+        """
+        return self._create_request(
+            "GET",
+            url,
+            headers_dict=headers_dict
+        )
 
     @defer.inlineCallbacks
-    def do_request(
-        self,
-        method,
-        url,
-        data=None,
-        qparams=None,
-        jsonreq=True,
-        headers: Optional[dict] = None,
-    ):
-        headers = headers or {}
-
+    def do_request(self, method, url, data=None, qparams=None, jsonreq=True, headers={}):
         if qparams:
             url = "%s?%s" % (url, urllib.urlencode(qparams, True))
 
         if jsonreq:
             prod = _JsonProducer(data)
-            headers["Content-Type"] = ["application/json"]
+            headers['Content-Type'] = ["application/json"];
         else:
             prod = _RawProducer(data)
 
         if method in ["POST", "PUT"]:
-            response = yield self._create_request(
-                method, url, producer=prod, headers_dict=headers
-            )
+            response = yield self._create_request(method, url,
+                    producer=prod,
+                    headers_dict=headers)
         else:
             response = yield self._create_request(method, url)
 
@@ -132,33 +135,33 @@ class TwistedHttpClient(HttpClient):
         defer.returnValue(json.loads(body))
 
     @defer.inlineCallbacks
-    def _create_request(
-        self, method, url, producer=None, headers_dict: Optional[dict] = None
-    ):
-        """Creates and sends a request to the given url"""
-        headers_dict = headers_dict or {}
-
+    def _create_request(self, method, url, producer=None, headers_dict={}):
+        """ Creates and sends a request to the given url
+        """
         headers_dict["User-Agent"] = ["Synapse Cmd Client"]
 
         retries_left = 5
-        print("%s to %s with headers %s" % (method, url, headers_dict))
+        print "%s to %s with headers %s" % (method, url, headers_dict)
         if self.verbose and producer:
             if "password" in producer.data:
                 temp = producer.data["password"]
                 producer.data["password"] = "[REDACTED]"
-                print(json.dumps(producer.data, indent=4))
+                print json.dumps(producer.data, indent=4)
                 producer.data["password"] = temp
             else:
-                print(json.dumps(producer.data, indent=4))
+                print json.dumps(producer.data, indent=4)
 
         while True:
             try:
                 response = yield self.agent.request(
-                    method, url.encode("UTF8"), Headers(headers_dict), producer
+                    method,
+                    url.encode("UTF8"),
+                    Headers(headers_dict),
+                    producer
                 )
                 break
             except Exception as e:
-                print("uh oh: %s" % e)
+                print "uh oh: %s" % e
                 if retries_left:
                     yield self.sleep(2 ** (5 - retries_left))
                     retries_left -= 1
@@ -166,8 +169,8 @@ class TwistedHttpClient(HttpClient):
                     raise e
 
         if self.verbose:
-            print("Status %s %s" % (response.code, response.phrase))
-            print(pformat(list(response.headers.getAllRawHeaders())))
+            print "Status %s %s" % (response.code, response.phrase)
+            print pformat(list(response.headers.getAllRawHeaders()))
         defer.returnValue(response)
 
     def sleep(self, seconds):
@@ -175,8 +178,7 @@ class TwistedHttpClient(HttpClient):
         reactor.callLater(seconds, d.callback, seconds)
         return d
 
-
-class _RawProducer:
+class _RawProducer(object):
     def __init__(self, data):
         self.data = data
         self.body = data
@@ -192,10 +194,9 @@ class _RawProducer:
     def stopProducing(self):
         pass
 
-
-class _JsonProducer:
-    """Used by the twisted http client to create the HTTP body from json"""
-
+class _JsonProducer(object):
+    """ Used by the twisted http client to create the HTTP body from json
+    """
     def __init__(self, jsn):
         self.data = jsn
         self.body = json.dumps(jsn).encode("utf8")

contrib/docker/README.md

@@ -1,26 +1,35 @@
-
 # Synapse Docker
 
-### Configuration
+### Automated configuration
+
+It is recommended that you use Docker Compose to run your containers, including
+this image and a Postgres server. A sample ``docker-compose.yml`` is provided,
+including example labels for reverse proxying and other artifacts.
+
+Read the section about environment variables and set at least mandatory variables,
+then run the server:
+
+```
+docker-compose up -d
+```
+
+If secrets are not specified in the environment variables, they will be generated
+as part of the startup. Please ensure these secrets are kept between launches of the
+Docker container, as their loss may require users to log in again.
+
+### Manual configuration
 
 A sample ``docker-compose.yml`` is provided, including example labels for
 reverse proxying and other artifacts. The docker-compose file is an example,
 please comment/uncomment sections that are not suitable for your usecase.
 
 Specify a ``SYNAPSE_CONFIG_PATH``, preferably to a persistent path,
-to use manual configuration.
-
-To generate a fresh `homeserver.yaml`, you can use the `generate` command.
-(See the [documentation](../../docker/README.md#generating-a-configuration-file)
-for more information.) You will need to specify appropriate values for at least the
-`SYNAPSE_SERVER_NAME` and `SYNAPSE_REPORT_STATS` environment variables. For example:
+to use manual configuration. To generate a fresh ``homeserver.yaml``, simply run:
 
 ```
-docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host -e SYNAPSE_REPORT_STATS=yes synapse generate
+docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host synapse generate
 ```
 
-(This will also generate necessary signing keys.)
-
 Then, customize your configuration and run the server:
 
 ```