WIP

.buildkite/.env

@@ -1,13 +0,0 @@
-CI
-BUILDKITE
-BUILDKITE_BUILD_NUMBER
-BUILDKITE_BRANCH
-BUILDKITE_BUILD_NUMBER
-BUILDKITE_JOB_ID
-BUILDKITE_BUILD_URL
-BUILDKITE_PROJECT_SLUG
-BUILDKITE_COMMIT
-BUILDKITE_PULL_REQUEST
-BUILDKITE_TAG
-CODECOV_TOKEN
-TRIAL_FLAGS

.buildkite/postgres-config.yaml

@@ -1,21 +0,0 @@
-# Configuration file used for testing the 'synapse_port_db' script.
-# Tells the script to connect to the postgresql database that will be available in the
-# CI's Docker setup at the point where this file is considered.
-server_name: "localhost:8800"
-
-signing_key_path: "/src/.buildkite/test.signing.key"
-
-report_stats: false
-
-database:
-  name: "psycopg2"
-  args:
-    user: postgres
-    host: postgres
-    password: postgres
-    database: synapse
-
-# Suppress the key server warning.
-trusted_key_servers:
-  - server_name: "matrix.org"
-suppress_key_server_warning: true

.buildkite/scripts/postgres_exec.py

@@ -1,31 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import sys
-
-import psycopg2
-
-# a very simple replacment for `psql`, to make up for the lack of the postgres client
-# libraries in the synapse docker image.
-
-# We use "postgres" as a database because it's bound to exist and the "synapse" one
-# doesn't exist yet.
-db_conn = psycopg2.connect(
-    user="postgres", host="postgres", password="postgres", dbname="postgres"
-)
-db_conn.autocommit = True
-cur = db_conn.cursor()
-for c in sys.argv[1:]:
-    cur.execute(c)

.buildkite/scripts/test_old_deps.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-# this script is run by buildkite in a plain `bionic` container; it installs the
-# minimal requirements for tox and hands over to the py3-old tox environment.
-
-set -ex
-
-apt-get update
-apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox
-
-export LANG="C.UTF-8"
-
-# Prevent virtualenv from auto-updating pip to an incompatible version
-export VIRTUALENV_NO_DOWNLOAD=1
-
-exec tox -e py3-old,combine

.buildkite/scripts/test_synapse_port_db.sh

@@ -1,53 +0,0 @@
-#!/usr/bin/env bash
-#
-# Test script for 'synapse_port_db'.
-#   - sets up synapse and deps
-#   - runs the port script on a prepopulated test sqlite db
-#   - also runs it against an new sqlite db
-
-
-set -xe
-cd `dirname $0`/../..
-
-echo "--- Install dependencies"
-
-# Install dependencies for this test.
-pip install psycopg2 coverage coverage-enable-subprocess
-
-# Install Synapse itself. This won't update any libraries.
-pip install -e .
-
-echo "--- Generate the signing key"
-
-# Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .buildkite/sqlite-config.yaml
-
-echo "--- Prepare test database"
-
-# Make sure the SQLite3 database is using the latest schema and has no pending background update.
-scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
-
-# Create the PostgreSQL database.
-./.buildkite/scripts/postgres_exec.py "CREATE DATABASE synapse"
-
-echo "+++ Run synapse_port_db against test database"
-coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
-
-#####
-
-# Now do the same again, on an empty database.
-
-echo "--- Prepare empty SQLite database"
-
-# we do this by deleting the sqlite db, and then doing the same again.
-rm .buildkite/test_db.db
-
-scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
-
-# re-create the PostgreSQL database.
-./.buildkite/scripts/postgres_exec.py \
-  "DROP DATABASE synapse" \
-  "CREATE DATABASE synapse"
-
-echo "+++ Run synapse_port_db against empty database"
-coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml

.buildkite/sqlite-config.yaml

@@ -1,18 +0,0 @@
-# Configuration file used for testing the 'synapse_port_db' script.
-# Tells the 'update_database' script to connect to the test SQLite database to upgrade its
-# schema and run background updates on it.
-server_name: "localhost:8800"
-
-signing_key_path: "/src/.buildkite/test.signing.key"
-
-report_stats: false
-
-database:
-  name: "sqlite3"
-  args:
-    database: ".buildkite/test_db.db"
-
-# Suppress the key server warning.
-trusted_key_servers:
-  - server_name: "matrix.org"
-suppress_key_server_warning: true

.buildkite/worker-blacklist

@@ -1,10 +0,0 @@
-# This file serves as a blacklist for SyTest tests that we expect will fail in
-# Synapse when run under worker mode. For more details, see sytest-blacklist.
-
-Can re-join room if re-invited
-
-# new failures as of https://github.com/matrix-org/sytest/pull/732
-Device list doesn't change if remote server is down
-
-# https://buildkite.com/matrix-dot-org/synapse/builds/6134#6f67bf47-e234-474d-80e8-c6e1868b15c5
-Server correctly handles incoming m.device_list_update

.circleci/config.yml

@@ -1,37 +1,161 @@
-version: 2.1
+version: 2
 jobs:
   dockerhubuploadrelease:
-    docker:
-      - image: docker:git
+    machine: true
     steps:
       - checkout
-      - docker_prepare
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      # for release builds, we want to get the amd64 image out asap, so first
-      # we do an amd64-only build, before following up with a multiarch build.
-      - docker_build:
-          tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
-          platforms: linux/amd64
-      - docker_build:
-          tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
-          platforms: linux/amd64,linux/arm64
-
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
   dockerhubuploadlatest:
-    docker:
-      - image: docker:git
+    machine: true
     steps:
       - checkout
-      - docker_prepare
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest-py3 --build-arg PYTHON_VERSION=3.6 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      # for `latest`, we don't want the arm images to disappear, so don't update the tag
-      # until all of the platforms are built.
-      - docker_build:
-          tag: -t matrixdotorg/synapse:latest
-          platforms: linux/amd64,linux/arm64
+      - run: docker push matrixdotorg/synapse:latest
+      - run: docker push matrixdotorg/synapse:latest-py3
+  sytestpy2:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
+    steps:
+      - checkout
+      - run: /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy2postgres:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
+    steps:
+      - checkout
+      - run: POSTGRES=1 /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy2merged:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy2postgresmerged:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: POSTGRES=1 /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+
+  sytestpy3:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
+    steps:
+      - checkout
+      - run: /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy3postgres:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
+    steps:
+      - checkout
+      - run: POSTGRES=1 /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy3merged:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy3postgresmerged:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: POSTGRES=1 /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
 
 workflows:
+  version: 2
   build:
     jobs:
+      - sytestpy2:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy2postgres:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy3:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy3postgres:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy2merged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
+      - sytestpy2postgresmerged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
+      - sytestpy3merged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
+      - sytestpy3postgresmerged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
       - dockerhubuploadrelease:
           filters:
             tags:
@@ -42,37 +166,3 @@ workflows:
           filters:
             branches:
               only: master
-
-commands:
-  docker_prepare:
-    description: Sets up a remote docker server, downloads the buildx cli plugin, and enables multiarch images
-    parameters:
-      buildx_version:
-        type: string
-        default: "v0.4.1"
-    steps:
-      - setup_remote_docker:
-          # 19.03.13 was the most recent available on circleci at the time of
-          # writing.
-          version: 19.03.13
-      - run: apk add --no-cache curl
-      - run: mkdir -vp ~/.docker/cli-plugins/ ~/dockercache
-      - run: curl --silent -L "https://github.com/docker/buildx/releases/download/<< parameters.buildx_version >>/buildx-<< parameters.buildx_version >>.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
-      - run: chmod a+x ~/.docker/cli-plugins/docker-buildx
-      # install qemu links in /proc/sys/fs/binfmt_misc on the docker instance running the circleci job
-      - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-      # create a context named `builder` for the builds
-      - run: docker context create builder
-      # create a buildx builder using the new context, and set it as the default
-      - run: docker buildx create builder --use
-
-  docker_build:
-    description: Builds and pushed images to dockerhub using buildx
-    parameters:
-      platforms:
-        type: string
-        default: linux/amd64
-      tag:
-        type: string
-    steps:
-      - run: docker buildx build -f docker/Dockerfile --push --platform << parameters.platforms >> --label gitsha1=${CIRCLE_SHA1} << parameters.tag >> --progress=plain .

.circleci/merge_base_branch.sh

@@ -2,24 +2,23 @@
 
 set -e
 
-if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs|release-.*)$ ]]; then
-    echo "Not merging forward, as this is a release branch"
-    exit 0
-fi
+# CircleCI doesn't give CIRCLE_PR_NUMBER in the environment for non-forked PRs. Wonderful.
+# In this case, we just need to do some ~shell magic~ to strip it out of the PULL_REQUEST URL.
+echo 'export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}"' >> $BASH_ENV
+source $BASH_ENV
 
-if [[ -z $BUILDKITE_PULL_REQUEST_BASE_BRANCH ]]; then
-    echo "Not a pull request, or hasn't had a PR opened yet..."
+if [[ -z "${CIRCLE_PR_NUMBER}" ]]
+then
+    echo "Can't figure out what the PR number is! Assuming merge target is develop."
 
     # It probably hasn't had a PR opened yet. Since all PRs land on develop, we
     # can probably assume it's based on it and will be merged into it.
     GITBASE="develop"
 else
     # Get the reference, using the GitHub API
-    GITBASE=$BUILDKITE_PULL_REQUEST_BASE_BRANCH
+    GITBASE=`wget -O- https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
 fi
 
-echo "--- merge_base_branch $GITBASE"
-
 # Show what we are before
 git --no-pager show -s
 
@@ -29,7 +28,7 @@ git config --global user.name "A robot"
 
 # Fetch and merge. If it doesn't work, it will raise due to set -e.
 git fetch -u origin $GITBASE
-git merge --no-edit --no-commit origin/$GITBASE
+git merge --no-edit origin/$GITBASE
 
 # Show what we are after.
 git --no-pager show -s

.codecov.yml

@@ -1,4 +1,5 @@
-comment: off
+comment:
+  layout: "diff"
 
 coverage:
   status:

.coveragerc

@@ -1,8 +1,7 @@
 [run]
 branch = True
 parallel = True
-include=$TOP/synapse/*
-data_file = $TOP/.coverage
+include = synapse/*
 
 [report]
 precision = 2

.dockerignore

@@ -1,13 +1,9 @@
-# ignore everything by default
-*
-
-# things to include
-!docker
-!scripts
-!synapse
-!MANIFEST.in
-!README.rst
-!setup.py
-!synctl
-
-**/__pycache__
+Dockerfile
+.travis.yml
+.gitignore
+demo/etc
+tox.ini
+.git/*
+.tox/*
+debian/matrix-synapse/
+debian/matrix-synapse-*/

.git-blame-ignore-revs

@@ -1,8 +0,0 @@
-# Black reformatting (#5482).
-32e7c9e7f20b57dd081023ac42d6931a8da9b3a3
-
-# Target Python 3.5 with black (#8664).
-aff1eb7c671b0a3813407321d2702ec46c71fa56
-
-# Update black to 20.8b1 (#9381).
-0a00b7ff14890987f09112a2ae696c61001e6cf1

.github/FUNDING.yml

@@ -1,4 +0,0 @@
-# One username per supported platform and one custom link
-patreon: matrixdotorg
-liberapay: matrixdotorg
-custom: https://paypal.me/matrixdotorg

.github/ISSUE_TEMPLATE.md

@@ -1,5 +0,0 @@
-**If you are looking for support** please ask in **#synapse:matrix.org**
-(using a matrix.org account if necessary). We do not use GitHub issues for
-support.
-
-**If you want to report a security issue** please see https://matrix.org/security-disclosure-policy/

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -4,13 +4,11 @@ about: Create a report to help us improve
 
 ---
 
-<!--
+<!-- 
 
-**THIS IS NOT A SUPPORT CHANNEL!**
-**IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**,
-please ask in **#synapse:matrix.org** (using a matrix.org account if necessary)
+**IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**: 
+You will likely get better support more quickly if you ask in ** #matrix:matrix.org ** ;)
 
-If you want to report a security issue, please see https://matrix.org/security-disclosure-policy/
 
 This is a bug report template. By following the instructions below and
 filling out the sections with your information, you will help the us to get all
@@ -19,7 +17,7 @@ the necessary data to fix your issue.
 You can also preview your report before submitting it. You may remove sections
 that aren't relevant to your particular case.
 
-Text between <!-- and --​> marks will be invisible in the report.
+Text between <!-- and --> marks will be invisible in the report.
 
 -->
 
@@ -33,7 +31,7 @@ Text between <!-- and --​> marks will be invisible in the report.
 - that reproduce the bug
 - using hyphens as bullet points
 
-<!--
+<!-- 
 Describe how what happens differs from what you expected.
 
 If you can identify any relevant log snippets from _homeserver.log_, please include
@@ -46,26 +44,22 @@ those (please be careful to remove any personal or private data). Please surroun
 <!-- IMPORTANT: please answer the following questions, to help us narrow down the problem -->
 
 <!-- Was this issue identified on matrix.org or another homeserver? -->
-- **Homeserver**:
+- **Homeserver**: 
 
 If not matrix.org:
 
-<!--
- What version of Synapse is running?
-
-You can find the Synapse version with this command:
-
-$ curl http://localhost:8008/_synapse/admin/v1/server_version
-
-(You may need to replace `localhost:8008` if Synapse is not configured to
-listen on that port.)
+<!-- 
+What version of Synapse is running? 
+You can find the Synapse version by inspecting the server headers (replace matrix.org with
+your own homeserver domain):
+$ curl -v https://matrix.org/_matrix/client/versions 2>&1 | grep "Server:"
 -->
-- **Version**:
+- **Version**: 
 
-- **Install method**:
+- **Install method**: 
 <!-- examples: package manager/git clone/pip  -->
 
-- **Platform**:
+- **Platform**: 
 <!--
 Tell us about the environment in which your homeserver is operating
 distro, hardware, if it's running in a vm/container, etc.

.github/ISSUE_TEMPLATE/SUPPORT_REQUEST.md

@@ -4,7 +4,6 @@ about: I need support for Synapse
 
 ---
 
-Please don't file github issues asking for support.
+# Please ask for support in [**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org)
 
-Instead, please join [`#synapse:matrix.org`](https://matrix.to/#/#synapse:matrix.org)
-(from a matrix.org account if necessary), and ask there.
+## Don't file an issue as a support request.

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,12 +1,7 @@
 ### Pull Request Checklist
 
-<!-- Please read CONTRIBUTING.md before submitting your pull request -->
+<!-- Please read CONTRIBUTING.rst before submitting your pull request -->
 
 * [ ] Pull request is based on the develop branch
-* [ ] Pull request includes a [changelog file](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.md#changelog). The entry should:
-  - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.".
-  - Use markdown where necessary, mostly for `code blocks`.
-  - End with either a period (.) or an exclamation mark (!).
-  - Start with a capital letter.
-* [ ] Pull request includes a [sign off](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.md#sign-off)
-* [ ] Code style is correct (run the [linters](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.md#code-style))
+* [ ] Pull request includes a [changelog file](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst#changelog)
+* [ ] Pull request includes a [sign off](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst#sign-off)

.github/SUPPORT.md

@@ -1,3 +1,3 @@
-[**#synapse:matrix.org**](https://matrix.to/#/#synapse:matrix.org) is the official support room for
-Synapse, and can be accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html.
-Please ask for support there, rather than filing github issues.
+[**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org) is the official support room for Matrix, and can be accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html 
+
+It can also be access via IRC bridge at irc://irc.freenode.net/matrix or on the web here: https://webchat.freenode.net/?channels=matrix

.github/workflows/tests.yml

@@ -1,322 +0,0 @@
-name: Tests
-
-on:
-  push:
-    branches: ["develop", "release-*"]
-  pull_request:
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        toxenv:
-          - "check-sampleconfig"
-          - "check_codestyle"
-          - "check_isort"
-          - "mypy"
-          - "packaging"
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install tox
-      - run: tox -e ${{ matrix.toxenv }}
-
-  lint-crlf:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Check line endings
-        run: scripts-dev/check_line_terminators.sh
-
-  lint-newsfile:
-    if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install tox
-      - name: Patch Buildkite-specific test script
-        run: |
-          sed -i -e 's/\$BUILDKITE_PULL_REQUEST/${{ github.event.number }}/' \
-            scripts-dev/check-newsfragment
-      - run: scripts-dev/check-newsfragment
-
-  lint-sdist:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: "3.x"
-      - run: pip install wheel
-      - run: python setup.py sdist bdist_wheel
-      - uses: actions/upload-artifact@v2
-        with:
-          name: Python Distributions
-          path: dist/*
-
-  # Dummy step to gate other tests on without repeating the whole list
-  linting-done:
-    if: ${{ always() }} # Run this even if prior jobs were skipped
-    needs: [lint, lint-crlf, lint-newsfile, lint-sdist]
-    runs-on: ubuntu-latest
-    steps:
-      - run: "true"
-
-  trial:
-    if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9"]
-        database: ["sqlite"]
-        include:
-          # Newest Python without optional deps
-          - python-version: "3.9"
-            toxenv: "py-noextras,combine"
-
-          # Oldest Python with PostgreSQL
-          - python-version: "3.6"
-            database: "postgres"
-            postgres-version: "9.6"
-
-          # Newest Python with PostgreSQL
-          - python-version: "3.9"
-            database: "postgres"
-            postgres-version: "13"
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
-        if: ${{ matrix.postgres-version }}
-        run: |
-          docker run -d -p 5432:5432 \
-            -e POSTGRES_PASSWORD=postgres \
-            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
-            postgres:${{ matrix.postgres-version }}
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - run: pip install tox
-      - name: Await PostgreSQL
-        if: ${{ matrix.postgres-version }}
-        timeout-minutes: 2
-        run: until pg_isready -h localhost; do sleep 1; done
-      - run: tox -e py,combine
-        env:
-          TRIAL_FLAGS: "--jobs=2"
-          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
-          SYNAPSE_POSTGRES_HOST: localhost
-          SYNAPSE_POSTGRES_USER: postgres
-          SYNAPSE_POSTGRES_PASSWORD: postgres
-      - name: Dump logs
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  trial-olddeps:
-    if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Test with old deps
-        uses: docker://ubuntu:bionic # For old python and sqlite
-        with:
-          workdir: /github/workspace
-          entrypoint: .buildkite/scripts/test_old_deps.sh
-        env:
-          TRIAL_FLAGS: "--jobs=2"
-      - name: Dump logs
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  trial-pypy:
-    # Very slow; only run if the branch name includes 'pypy'
-    if: ${{ contains(github.ref, 'pypy') && !failure() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["pypy-3.6"]
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - run: pip install tox
-      - run: tox -e py,combine
-        env:
-          TRIAL_FLAGS: "--jobs=2"
-      - name: Dump logs
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  sytest:
-    if: ${{ !failure() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    container:
-      image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
-      volumes:
-        - ${{ github.workspace }}:/src
-      env:
-        BUILDKITE_BRANCH: ${{ github.head_ref }}
-        POSTGRES: ${{ matrix.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
-        WORKERS: ${{ matrix.workers && 1 }}
-        REDIS: ${{ matrix.redis && 1 }}
-        BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - sytest-tag: bionic
-
-          - sytest-tag: bionic
-            postgres: postgres
-
-          - sytest-tag: testing
-            postgres: postgres
-
-          - sytest-tag: bionic
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: postgres
-            workers: workers
-            redis: redis
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Prepare test blacklist
-        run: cat sytest-blacklist .buildkite/worker-blacklist > synapse-blacklist-with-workers
-      - name: Run SyTest
-        run: /bootstrap.sh synapse
-        working-directory: /src
-      - name: Dump results.tap
-        if: ${{ always() }}
-        run: cat /logs/results.tap
-      - name: Upload SyTest logs
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
-        with:
-          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
-          path: |
-            /logs/results.tap
-            /logs/**/*.log*
-
-  portdb:
-    if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        include:
-          - python-version: "3.6"
-            postgres-version: "9.6"
-
-          - python-version: "3.9"
-            postgres-version: "13"
-
-    services:
-      postgres:
-        image: postgres:${{ matrix.postgres-version }}
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_PASSWORD: "postgres"
-          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Patch Buildkite-specific test scripts
-        run: |
-          sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/postgres_exec.py
-          sed -i -e 's/host: postgres/host: localhost/' .buildkite/postgres-config.yaml
-          sed -i -e 's|/src/||' .buildkite/{sqlite,postgres}-config.yaml
-          sed -i -e 's/\$TOP/\$GITHUB_WORKSPACE/' .coveragerc
-      - run: .buildkite/scripts/test_synapse_port_db.sh
-
-  complement:
-    if: ${{ !failure() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    container:
-      # https://github.com/matrix-org/complement/blob/master/dockerfiles/ComplementCIBuildkite.Dockerfile
-      image: matrixdotorg/complement:latest
-      env:
-        CI: true
-      ports:
-        - 8448:8448
-      volumes:
-        - /var/run/docker.sock:/var/run/docker.sock
-
-    steps:
-      - name: Run actions/checkout@v2 for synapse
-        uses: actions/checkout@v2
-        with:
-          path: synapse
-
-      - name: Run actions/checkout@v2 for complement
-        uses: actions/checkout@v2
-        with:
-          repository: "matrix-org/complement"
-          path: complement
-
-      # Build initial Synapse image
-      - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
-        working-directory: synapse
-
-      # Build a ready-to-run Synapse image based on the initial image above.
-      # This new image includes a config file, keys for signing and TLS, and
-      # other settings to make it suitable for testing under Complement.
-      - run: docker build -t complement-synapse -f Synapse.Dockerfile .
-        working-directory: complement/dockerfiles
-
-      # Run Complement
-      - run: go test -v -tags synapse_blacklist ./tests
-        env:
-          COMPLEMENT_BASE_IMAGE: complement-synapse:latest
-        working-directory: complement

.gitignore

@@ -1,48 +1,64 @@
-# filename patterns
-*~
+*.pyc
 .*.swp
-.#*
-*.deb
-*.egg
-*.egg-info
+*~
 *.lock
-*.py[cod]
-*.snap
-*.tac
+
+.DS_Store
 _trial_temp/
 _trial_temp*/
-/out
-.DS_Store
-__pycache__/
+logs/
+dbs/
+*.egg
+dist/
+docs/build/
+*.egg-info
+pip-wheel-metadata/
 
-# stuff that is likely to exist when you run a server locally
-/*.db
-/*.log
-/*.log.*
-/*.log.config
-/*.pid
-/.python-version
-/*.signing.key
-/env/
-/.venv*/
+cmdclient_config.json
+homeserver*.db
+homeserver*.log
+homeserver*.log.*
+homeserver*.pid
 /homeserver*.yaml
-/logs
-/media_store/
-/uploads
 
-# IDEs
-/.idea/
-/.ropeproject/
-/.vscode/
+*.signing.key
+*.tls.crt
+*.tls.dh
+*.tls.key
 
-# build products
-!/.coveragerc
-/.coverage*
-/.mypy_cache/
-/.tox
-/build/
-/coverage.*
-/dist/
-/docs/build/
-/htmlcov
-/pip-wheel-metadata/
+.coverage*
+coverage.*
+!.coveragerc
+htmlcov
+
+demo/*/*.db
+demo/*/*.log
+demo/*/*.log.*
+demo/*/*.pid
+demo/media_store.*
+demo/etc
+
+uploads
+cache
+
+.idea/
+media_store/
+
+*.tac
+
+build/
+venv/
+venv*/
+*venv/
+
+localhost-800*/
+static/client/register/register_config.js
+.tox
+
+env/
+*.config
+
+.vscode/
+.ropeproject/
+*.deb
+/debs

.travis.yml

@@ -0,0 +1,83 @@
+sudo: false
+language: python
+
+cache:
+  directories:
+    # we only bother to cache the wheels; parts of the http cache get
+    # invalidated every build (because they get served with a max-age of 600
+    # seconds), which means that we end up re-uploading the whole cache for
+    # every build, which is time-consuming In any case, it's not obvious that
+    # downloading the cache from S3 would be much faster than downloading the
+    # originals from pypi.
+    #
+    - $HOME/.cache/pip/wheels
+
+addons:
+  postgresql: "9.4"
+
+# don't clone the whole repo history, one commit will do
+git:
+  depth: 1
+
+# only build branches we care about (PRs are built seperately)
+branches:
+  only:
+    - master
+    - develop
+    - /^release-v/
+
+# When running the tox environments that call Twisted Trial, we can pass the -j
+# flag to run the tests concurrently. We set this to 2 for CPU bound tests
+# (SQLite) and 4 for I/O bound tests (PostgreSQL).
+matrix:
+  fast_finish: true
+  include:
+  - python: 2.7
+    env: TOX_ENV=packaging
+
+  - python: 3.6
+    env: TOX_ENV="pep8,check_isort"
+
+  - python: 2.7
+    env: TOX_ENV=py27,codecov TRIAL_FLAGS="-j 2"
+
+  - python: 2.7
+    env: TOX_ENV=py27-old TRIAL_FLAGS="-j 2"
+
+  - python: 2.7
+    env: TOX_ENV=py27-postgres,codecov TRIAL_FLAGS="-j 4"
+    services:
+      - postgresql
+
+  - python: 3.5
+    env: TOX_ENV=py35,codecov TRIAL_FLAGS="-j 2"
+
+  - python: 3.6
+    env: TOX_ENV=py36,codecov TRIAL_FLAGS="-j 2"
+
+  - python: 3.6
+    env: TOX_ENV=py36-postgres,codecov TRIAL_FLAGS="-j 4"
+    services:
+      - postgresql
+
+  - # we only need to check for the newsfragment if it's a PR build
+    if: type = pull_request
+    python: 3.6
+    env: TOX_ENV=check-newsfragment
+    script:
+      - git remote set-branches --add origin develop
+      - git fetch origin develop
+      - tox -e $TOX_ENV
+
+install:
+  - pip install tox
+  
+  # if we don't have python3.6 in this environment, travis unhelpfully gives us
+  # a `python3.6` on our path which does nothing but spit out a warning. Tox
+  # tries to run it (even if we're not running a py36 env), so the build logs
+  # then have warnings which look like errors. To reduce the noise, remove the
+  # non-functional python3.6.
+  - ( ! command -v python3.6 || python3.6 --version ) &>/dev/null || rm -f $(command -v python3.6)
+
+script:
+  - tox -e $TOX_ENV

AUTHORS.rst

@@ -1,8 +1,34 @@
-The following is an incomplete list of people outside the core team who have
-contributed to Synapse. It is no longer maintained: more recent contributions
-are listed in the `changelog <CHANGES.md>`_.
+Erik Johnston <erik at matrix.org>
+ * HS core
+ * Federation API impl
 
-----
+Mark Haines <mark at matrix.org>
+ * HS core
+ * Crypto
+ * Content repository
+ * CS v2 API impl
+
+Kegan Dougal <kegan at matrix.org>
+ * HS core
+ * CS v1 API impl
+ * AS API impl
+
+Paul "LeoNerd" Evans <paul at matrix.org>
+ * HS core
+ * Presence
+ * Typing Notifications
+ * Performance metrics and caching layer
+
+Dave Baker <dave at matrix.org>
+ * Push notifications
+ * Auth CS v2 impl
+
+Matthew Hodgson <matthew at matrix.org>
+ * General doc & housekeeping
+ * Vertobot/vertobridge matrix<->verto PoC
+
+Emmanuel Rohee <manu at matrix.org>
+ * Supporting iOS clients (testability and fallback registration)
 
 Turned to Dust <dwinslow86 at gmail.com>
  * ArchLinux installation instructions
@@ -36,16 +62,10 @@ Christoph Witzany <christoph at web.crofting.com>
  * Add LDAP support for authentication
 
 Pierre Jaury <pierre at jaury.eu>
- * Docker packaging
+* Docker packaging
 
 Serban Constantin <serban.constantin at gmail dot com>
  * Small bug fix
 
-Joseph Weston <joseph at weston.cloud>
- * Add admin API for querying HS version
-
-Benjamin Saunders <ben.e.saunders at gmail dot com>
- * Documentation improvements
-
-Werner Sembach <werner.sembach at fau dot de>
- * Automatically remove a group/community when it is empty
+Jason Robinson <jasonr at matrix.org>
+ * Minor fixes

CONTRIBUTING.md

@@ -1,397 +0,0 @@
-Welcome to Synapse
-
-This document aims to get you started with contributing to this repo! 
-
-- [1. Who can contribute to Synapse?](#1-who-can-contribute-to-synapse)
-- [2. What do I need?](#2-what-do-i-need)
-- [3. Get the source.](#3-get-the-source)
-- [4. Install the dependencies](#4-install-the-dependencies)
-  * [Under Unix (macOS, Linux, BSD, ...)](#under-unix-macos-linux-bsd-)
-  * [Under Windows](#under-windows)
-- [5. Get in touch.](#5-get-in-touch)
-- [6. Pick an issue.](#6-pick-an-issue)
-- [7. Turn coffee and documentation into code and documentation!](#7-turn-coffee-and-documentation-into-code-and-documentation)
-- [8. Test, test, test!](#8-test-test-test)
-  * [Run the linters.](#run-the-linters)
-  * [Run the unit tests.](#run-the-unit-tests)
-  * [Run the integration tests.](#run-the-integration-tests)
-- [9. Submit your patch.](#9-submit-your-patch)
-  * [Changelog](#changelog)
-    + [How do I know what to call the changelog file before I create the PR?](#how-do-i-know-what-to-call-the-changelog-file-before-i-create-the-pr)
-    + [Debian changelog](#debian-changelog)
-  * [Sign off](#sign-off)
-- [10. Turn feedback into better code.](#10-turn-feedback-into-better-code)
-- [11. Find a new issue.](#11-find-a-new-issue)
-- [Notes for maintainers on merging PRs etc](#notes-for-maintainers-on-merging-prs-etc)
-- [Conclusion](#conclusion)
-
-# 1. Who can contribute to Synapse?
-
-Everyone is welcome to contribute code to [matrix.org
-projects](https://github.com/matrix-org), provided that they are willing to
-license their contributions under the same license as the project itself. We
-follow a simple 'inbound=outbound' model for contributions: the act of
-submitting an 'inbound' contribution means that the contributor agrees to
-license the code under the same terms as the project's overall 'outbound'
-license - in our case, this is almost always Apache Software License v2 (see
-[LICENSE](LICENSE)).
-
-# 2. What do I need?
-
-The code of Synapse is written in Python 3. To do pretty much anything, you'll need [a recent version of Python 3](https://wiki.python.org/moin/BeginnersGuide/Download).
-
-The source code of Synapse is hosted on GitHub. You will also need [a recent version of git](https://github.com/git-guides/install-git).
-
-For some tests, you will need [a recent version of Docker](https://docs.docker.com/get-docker/).
-
-
-# 3. Get the source.
-
-The preferred and easiest way to contribute changes is to fork the relevant
-project on GitHub, and then [create a pull request](
-https://help.github.com/articles/using-pull-requests/) to ask us to pull your
-changes into our repo.
-
-Please base your changes on the `develop` branch.
-
-```sh
-git clone git@github.com:YOUR_GITHUB_USER_NAME/synapse.git
-git checkout develop
-```
-
-If you need help getting started with git, this is beyond the scope of the document, but you
-can find many good git tutorials on the web.
-
-# 4. Install the dependencies
-
-## Under Unix (macOS, Linux, BSD, ...)
-
-Once you have installed Python 3 and added the source, please open a terminal and
-setup a *virtualenv*, as follows:
-
-```sh
-cd path/where/you/have/cloned/the/repository
-python3 -m venv ./env
-source ./env/bin/activate
-pip install -e ".[all,lint,mypy,test]"
-pip install tox
-```
-
-This will install the developer dependencies for the project.
-
-## Under Windows
-
-TBD
-
-
-# 5. Get in touch.
-
-Join our developer community on Matrix: #synapse-dev:matrix.org !
-
-
-# 6. Pick an issue.
-
-Fix your favorite problem or perhaps find a [Good First Issue](https://github.com/matrix-org/synapse/issues?q=is%3Aopen+is%3Aissue+label%3A%22Good+First+Issue%22)
-to work on.
-
-
-# 7. Turn coffee and documentation into code and documentation!
-
-Synapse's code style is documented [here](docs/code_style.md). Please follow
-it, including the conventions for the [sample configuration
-file](docs/code_style.md#configuration-file-format).
-
-There is a growing amount of documentation located in the [docs](docs)
-directory. This documentation is intended primarily for sysadmins running their
-own Synapse instance, as well as developers interacting externally with
-Synapse. [docs/dev](docs/dev) exists primarily to house documentation for
-Synapse developers. [docs/admin_api](docs/admin_api) houses documentation
-regarding Synapse's Admin API, which is used mostly by sysadmins and external
-service developers.
-
-If you add new files added to either of these folders, please use [GitHub-Flavoured
-Markdown](https://guides.github.com/features/mastering-markdown/).
-
-Some documentation also exists in [Synapse's GitHub
-Wiki](https://github.com/matrix-org/synapse/wiki), although this is primarily
-contributed to by community authors.
-
-
-# 8. Test, test, test!
-<a name="test-test-test"></a>
-
-While you're developing and before submitting a patch, you'll
-want to test your code.
-
-## Run the linters.
-
-The linters look at your code and do two things:
-
-- ensure that your code follows the coding style adopted by the project;
-- catch a number of errors in your code.
-
-They're pretty fast, don't hesitate!
-
-```sh
-source ./env/bin/activate
-./scripts-dev/lint.sh
-```
-
-Note that this script *will modify your files* to fix styling errors.
-Make sure that you have saved all your files.
-
-If you wish to restrict the linters to only the files changed since the last commit
-(much faster!), you can instead run:
-
-```sh
-source ./env/bin/activate
-./scripts-dev/lint.sh -d
-```
-
-Or if you know exactly which files you wish to lint, you can instead run:
-
-```sh
-source ./env/bin/activate
-./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder
-```
-
-## Run the unit tests.
-
-The unit tests run parts of Synapse, including your changes, to see if anything
-was broken. They are slower than the linters but will typically catch more errors.
-
-```sh
-source ./env/bin/activate
-trial tests
-```
-
-If you wish to only run *some* unit tests, you may specify
-another module instead of `tests` - or a test class or a method:
-
-```sh
-source ./env/bin/activate
-trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite
-```
-
-If your tests fail, you may wish to look at the logs:
-
-```sh
-less _trial_temp/test.log
-```
-
-## Run the integration tests.
-
-The integration tests are a more comprehensive suite of tests. They
-run a full version of Synapse, including your changes, to check if
-anything was broken. They are slower than the unit tests but will
-typically catch more errors.
-
-The following command will let you run the integration test with the most common
-configuration:
-
-```sh
-$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:py37
-```
-
-This configuration should generally cover  your needs. For more details about other configurations, see [documentation in the SyTest repo](https://github.com/matrix-org/sytest/blob/develop/docker/README.md).
-
-
-# 9. Submit your patch.
-
-Once you're happy with your patch, it's time to prepare a Pull Request.
-
-To prepare a Pull Request, please:
-
-1. verify that [all the tests pass](#test-test-test), including the coding style;
-2. [sign off](#sign-off) your contribution;
-3. `git push` your commit to your fork of Synapse;
-4. on GitHub, [create the Pull Request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request);
-5. add a [changelog entry](#changelog) and push it to your Pull Request;
-6. for most contributors, that's all - however, if you are a member of the organization `matrix-org`, on GitHub, please request a review from `matrix.org / Synapse Core`.
-
-
-## Changelog
-
-All changes, even minor ones, need a corresponding changelog / newsfragment
-entry. These are managed by [Towncrier](https://github.com/hawkowl/towncrier).
-
-To create a changelog entry, make a new file in the `changelog.d` directory named
-in the format of `PRnumber.type`. The type can be one of the following:
-
-* `feature`
-* `bugfix`
-* `docker` (for updates to the Docker image)
-* `doc` (for updates to the documentation)
-* `removal` (also used for deprecations)
-* `misc` (for internal-only changes)
-
-This file will become part of our [changelog](
-https://github.com/matrix-org/synapse/blob/master/CHANGES.md) at the next
-release, so the content of the file should be a short description of your
-change in the same style as the rest of the changelog. The file can contain Markdown
-formatting, and should end with a full stop (.) or an exclamation mark (!) for
-consistency.
-
-Adding credits to the changelog is encouraged, we value your
-contributions and would like to have you shouted out in the release notes!
-
-For example, a fix in PR #1234 would have its changelog entry in
-`changelog.d/1234.bugfix`, and contain content like:
-
-> The security levels of Florbs are now validated when received
-> via the `/federation/florb` endpoint. Contributed by Jane Matrix.
-
-If there are multiple pull requests involved in a single bugfix/feature/etc,
-then the content for each `changelog.d` file should be the same. Towncrier will
-merge the matching files together into a single changelog entry when we come to
-release.
-
-### How do I know what to call the changelog file before I create the PR?
-
-Obviously, you don't know if you should call your newsfile
-`1234.bugfix` or `5678.bugfix` until you create the PR, which leads to a
-chicken-and-egg problem.
-
-There are two options for solving this:
-
- 1. Open the PR without a changelog file, see what number you got, and *then*
-    add the changelog file to your branch (see [Updating your pull
-    request](#updating-your-pull-request)), or:
-
- 1. Look at the [list of all
-    issues/PRs](https://github.com/matrix-org/synapse/issues?q=), add one to the
-    highest number you see, and quickly open the PR before somebody else claims
-    your number.
-
-    [This
-    script](https://github.com/richvdh/scripts/blob/master/next_github_number.sh)
-    might be helpful if you find yourself doing this a lot.
-
-Sorry, we know it's a bit fiddly, but it's *really* helpful for us when we come
-to put together a release!
-
-### Debian changelog
-
-Changes which affect the debian packaging files (in `debian`) are an
-exception to the rule that all changes require a `changelog.d` file.
-
-In this case, you will need to add an entry to the debian changelog for the
-next release. For this, run the following command:
-
-```
-dch
-```
-
-This will make up a new version number (if there isn't already an unreleased
-version in flight), and open an editor where you can add a new changelog entry.
-(Our release process will ensure that the version number and maintainer name is
-corrected for the release.)
-
-If your change affects both the debian packaging *and* files outside the debian
-directory, you will need both a regular newsfragment *and* an entry in the
-debian changelog. (Though typically such changes should be submitted as two
-separate pull requests.)
-
-## Sign off
-
-In order to have a concrete record that your contribution is intentional
-and you agree to license it under the same terms as the project's license, we've adopted the
-same lightweight approach that the Linux Kernel
-[submitting patches process](
-https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin>),
-[Docker](https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other
-projects use: the DCO (Developer Certificate of Origin:
-http://developercertificate.org/). This is a simple declaration that you wrote
-the contribution or otherwise have the right to contribute it to Matrix:
-
-```
-Developer Certificate of Origin
-Version 1.1
-
-Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
-660 York Street, Suite 102,
-San Francisco, CA 94110 USA
-
-Everyone is permitted to copy and distribute verbatim copies of this
-license document, but changing it is not allowed.
-
-Developer's Certificate of Origin 1.1
-
-By making a contribution to this project, I certify that:
-
-(a) The contribution was created in whole or in part by me and I
-    have the right to submit it under the open source license
-    indicated in the file; or
-
-(b) The contribution is based upon previous work that, to the best
-    of my knowledge, is covered under an appropriate open source
-    license and I have the right under that license to submit that
-    work with modifications, whether created in whole or in part
-    by me, under the same open source license (unless I am
-    permitted to submit under a different license), as indicated
-    in the file; or
-
-(c) The contribution was provided directly to me by some other
-    person who certified (a), (b) or (c) and I have not modified
-    it.
-
-(d) I understand and agree that this project and the contribution
-    are public and that a record of the contribution (including all
-    personal information I submit with it, including my sign-off) is
-    maintained indefinitely and may be redistributed consistent with
-    this project or the open source license(s) involved.
-```
-
-If you agree to this for your contribution, then all that's needed is to
-include the line in your commit or pull request comment:
-
-```
-Signed-off-by: Your Name <your@email.example.org>
-```
-
-We accept contributions under a legally identifiable name, such as
-your name on government documentation or common-law names (names
-claimed by legitimate usage or repute). Unfortunately, we cannot
-accept anonymous contributions at this time.
-
-Git allows you to add this signoff automatically when using the `-s`
-flag to `git commit`, which uses the name and email set in your
-`user.name` and `user.email` git configs.
-
-
-# 10. Turn feedback into better code.
-
-Once the Pull Request is opened, you will see a few things:
-
-1. our automated CI (Continuous Integration) pipeline will run (again) the linters, the unit tests, the integration tests and more;
-2. one or more of the developers will take a look at your Pull Request and offer feedback.
-
-From this point, you should:
-
-1. Look at the results of the CI pipeline.
-   - If there is any error, fix the error.
-2. If a developer has requested changes, make these changes and let us know if it is ready for a developer to review again.
-3. Create a new commit with the changes.
-   - Please do NOT overwrite the history. New commits make the reviewer's life easier.
-   - Push this commits to your Pull Request.
-4. Back to 1.
-
-Once both the CI and the developers are happy, the patch will be merged into Synapse and released shortly!
-
-# 11. Find a new issue.
-
-By now, you know the drill!
-
-# Notes for maintainers on merging PRs etc
-
-There are some notes for those with commit access to the project on how we
-manage git [here](docs/dev/git.md).
-
-# Conclusion
-
-That's it! Matrix is a very open and collaborative project as you might expect
-given our obsession with open communication. If we're going to successfully
-matrix together all the fragmented communication technologies out there we are
-reliant on contributions and collaboration from the community to do so. So
-please get involved - and we hope you have as much fun hacking on Matrix as we
-do!

CONTRIBUTING.rst

@@ -0,0 +1,169 @@
+Contributing code to Matrix
+===========================
+
+Everyone is welcome to contribute code to Matrix
+(https://github.com/matrix-org), provided that they are willing to license
+their contributions under the same license as the project itself. We follow a
+simple 'inbound=outbound' model for contributions: the act of submitting an
+'inbound' contribution means that the contributor agrees to license the code
+under the same terms as the project's overall 'outbound' license - in our
+case, this is almost always Apache Software License v2 (see LICENSE).
+
+How to contribute
+~~~~~~~~~~~~~~~~~
+
+The preferred and easiest way to contribute changes to Matrix is to fork the
+relevant project on github, and then create a pull request to ask us to pull
+your changes into our repo
+(https://help.github.com/articles/using-pull-requests/)
+
+**The single biggest thing you need to know is: please base your changes on
+the develop branch - /not/ master.**
+
+We use the master branch to track the most recent release, so that folks who
+blindly clone the repo and automatically check out master get something that
+works. Develop is the unstable branch where all the development actually
+happens: the workflow is that contributors should fork the develop branch to
+make a 'feature' branch for a particular contribution, and then make a pull
+request to merge this back into the matrix.org 'official' develop branch. We
+use github's pull request workflow to review the contribution, and either ask
+you to make any refinements needed or merge it and make them ourselves. The
+changes will then land on master when we next do a release.
+
+We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Travis CI 
+<https://travis-ci.org/matrix-org/synapse>`_ for continuous integration. All
+pull requests to synapse get automatically tested by Travis and CircleCI.
+If your change breaks the build, this will be shown in GitHub, so please
+keep an eye on the pull request for feedback.
+
+To run unit tests in a local development environment, you can use:
+
+- ``tox -e py27`` (requires tox to be installed by ``pip install tox``) for
+  SQLite-backed Synapse on Python 2.7.
+- ``tox -e py35`` for SQLite-backed Synapse on Python 3.5.
+- ``tox -e py36`` for SQLite-backed Synapse on Python 3.6.
+- ``tox -e py27-postgres`` for PostgreSQL-backed Synapse on Python 2.7
+  (requires a running local PostgreSQL with access to create databases).
+- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 2.7
+  (requires Docker). Entirely self-contained, recommended if you don't want to
+  set up PostgreSQL yourself.
+
+Docker images are available for running the integration tests (SyTest) locally,
+see the `documentation in the SyTest repo
+<https://github.com/matrix-org/sytest/blob/develop/docker/README.md>`_ for more
+information.
+
+Code style
+~~~~~~~~~~
+
+All Matrix projects have a well-defined code-style - and sometimes we've even
+got as far as documenting it... For instance, synapse's code style doc lives
+at https://github.com/matrix-org/synapse/tree/master/docs/code_style.rst.
+
+Please ensure your changes match the cosmetic style of the existing project,
+and **never** mix cosmetic and functional changes in the same commit, as it
+makes it horribly hard to review otherwise.
+
+Changelog
+~~~~~~~~~
+
+All changes, even minor ones, need a corresponding changelog / newsfragment
+entry. These are managed by Towncrier
+(https://github.com/hawkowl/towncrier).
+
+To create a changelog entry, make a new file in the ``changelog.d``
+file named in the format of ``PRnumber.type``. The type can be
+one of ``feature``, ``bugfix``, ``removal`` (also used for
+deprecations), or ``misc`` (for internal-only changes). The content of
+the file is your changelog entry, which can contain Markdown
+formatting. Adding credits to the changelog is encouraged, we value
+your contributions and would like to have you shouted out in the
+release notes!
+
+For example, a fix in PR #1234 would have its changelog entry in
+``changelog.d/1234.bugfix``, and contain content like "The security levels of
+Florbs are now validated when recieved over federation. Contributed by Jane
+Matrix".
+
+Attribution
+~~~~~~~~~~~
+
+Everyone who contributes anything to Matrix is welcome to be listed in the
+AUTHORS.rst file for the project in question. Please feel free to include a
+change to AUTHORS.rst in your pull request to list yourself and a short
+description of the area(s) you've worked on. Also, we sometimes have swag to
+give away to contributors - if you feel that Matrix-branded apparel is missing
+from your life, please mail us your shipping address to matrix at matrix.org and
+we'll try to fix it :)
+
+Sign off
+~~~~~~~~
+
+In order to have a concrete record that your contribution is intentional
+and you agree to license it under the same terms as the project's license, we've adopted the
+same lightweight approach that the Linux Kernel
+`submitting patches process <https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin>`_, Docker
+(https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other
+projects use: the DCO (Developer Certificate of Origin:
+http://developercertificate.org/). This is a simple declaration that you wrote
+the contribution or otherwise have the right to contribute it to Matrix::
+
+    Developer Certificate of Origin
+    Version 1.1
+
+    Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
+    660 York Street, Suite 102,
+    San Francisco, CA 94110 USA
+
+    Everyone is permitted to copy and distribute verbatim copies of this
+    license document, but changing it is not allowed.
+
+    Developer's Certificate of Origin 1.1
+
+    By making a contribution to this project, I certify that:
+
+    (a) The contribution was created in whole or in part by me and I
+        have the right to submit it under the open source license
+        indicated in the file; or
+
+    (b) The contribution is based upon previous work that, to the best
+        of my knowledge, is covered under an appropriate open source
+        license and I have the right under that license to submit that
+        work with modifications, whether created in whole or in part
+        by me, under the same open source license (unless I am
+        permitted to submit under a different license), as indicated
+        in the file; or
+
+    (c) The contribution was provided directly to me by some other
+        person who certified (a), (b) or (c) and I have not modified
+        it.
+
+    (d) I understand and agree that this project and the contribution
+        are public and that a record of the contribution (including all
+        personal information I submit with it, including my sign-off) is
+        maintained indefinitely and may be redistributed consistent with
+        this project or the open source license(s) involved.
+
+If you agree to this for your contribution, then all that's needed is to
+include the line in your commit or pull request comment::
+
+    Signed-off-by: Your Name <your@email.example.org>
+
+We accept contributions under a legally identifiable name, such as
+your name on government documentation or common-law names (names
+claimed by legitimate usage or repute). Unfortunately, we cannot
+accept anonymous contributions at this time.
+
+Git allows you to add this signoff automatically when using the ``-s``
+flag to ``git commit``, which uses the name and email set in your
+``user.name`` and ``user.email`` git configs.
+
+Conclusion
+~~~~~~~~~~
+
+That's it!  Matrix is a very open and collaborative project as you might expect
+given our obsession with open communication.  If we're going to successfully
+matrix together all the fragmented communication technologies out there we are
+reliant on contributions and collaboration from the community to do so.  So
+please get involved - and we hope you have as much fun hacking on Matrix as we
+do!

INSTALL.md

@@ -1,594 +0,0 @@
-# Installation Instructions
-
-There are 3 steps to follow under **Installation Instructions**.
-
-- [Installation Instructions](#installation-instructions)
-  - [Choosing your server name](#choosing-your-server-name)
-  - [Installing Synapse](#installing-synapse)
-    - [Installing from source](#installing-from-source)
-      - [Platform-specific prerequisites](#platform-specific-prerequisites)
-        - [Debian/Ubuntu/Raspbian](#debianubunturaspbian)
-        - [ArchLinux](#archlinux)
-        - [CentOS/Fedora](#centosfedora)
-        - [macOS](#macos)
-        - [OpenSUSE](#opensuse)
-        - [OpenBSD](#openbsd)
-        - [Windows](#windows)
-    - [Prebuilt packages](#prebuilt-packages)
-      - [Docker images and Ansible playbooks](#docker-images-and-ansible-playbooks)
-      - [Debian/Ubuntu](#debianubuntu)
-        - [Matrix.org packages](#matrixorg-packages)
-        - [Downstream Debian packages](#downstream-debian-packages)
-        - [Downstream Ubuntu packages](#downstream-ubuntu-packages)
-      - [Fedora](#fedora)
-      - [OpenSUSE](#opensuse-1)
-      - [SUSE Linux Enterprise Server](#suse-linux-enterprise-server)
-      - [ArchLinux](#archlinux-1)
-      - [Void Linux](#void-linux)
-      - [FreeBSD](#freebsd)
-      - [OpenBSD](#openbsd-1)
-      - [NixOS](#nixos)
-  - [Setting up Synapse](#setting-up-synapse)
-    - [Using PostgreSQL](#using-postgresql)
-    - [TLS certificates](#tls-certificates)
-    - [Client Well-Known URI](#client-well-known-uri)
-    - [Email](#email)
-    - [Registering a user](#registering-a-user)
-    - [Setting up a TURN server](#setting-up-a-turn-server)
-    - [URL previews](#url-previews)
-    - [Troubleshooting Installation](#troubleshooting-installation)
-
-
-## Choosing your server name
-
-It is important to choose the name for your server before you install Synapse,
-because it cannot be changed later.
-
-The server name determines the "domain" part of user-ids for users on your
-server: these will all be of the format `@user:my.domain.name`. It also
-determines how other matrix servers will reach yours for federation.
-
-For a test configuration, set this to the hostname of your server. For a more
-production-ready setup, you will probably want to specify your domain
-(`example.com`) rather than a matrix-specific hostname here (in the same way
-that your email address is probably `user@example.com` rather than
-`user@email.example.com`) - but doing so may require more advanced setup: see
-[Setting up Federation](docs/federate.md).
-
-## Installing Synapse
-
-### Installing from source
-
-(Prebuilt packages are available for some platforms - see [Prebuilt packages](#prebuilt-packages).)
-
-When installing from source please make sure that the [Platform-specific prerequisites](#platform-specific-prerequisites) are already installed.
-
-System requirements:
-
-- POSIX-compliant system (tested on Linux & OS X)
-- Python 3.5.2 or later, up to Python 3.9.
-- At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
-
-
-To install the Synapse homeserver run:
-
-```sh
-mkdir -p ~/synapse
-virtualenv -p python3 ~/synapse/env
-source ~/synapse/env/bin/activate
-pip install --upgrade pip
-pip install --upgrade setuptools
-pip install matrix-synapse
-```
-
-This will download Synapse from [PyPI](https://pypi.org/project/matrix-synapse)
-and install it, along with the python libraries it uses, into a virtual environment
-under `~/synapse/env`.  Feel free to pick a different directory if you
-prefer.
-
-This Synapse installation can then be later upgraded by using pip again with the
-update flag:
-
-```sh
-source ~/synapse/env/bin/activate
-pip install -U matrix-synapse
-```
-
-Before you can start Synapse, you will need to generate a configuration
-file. To do this, run (in your virtualenv, as before):
-
-```sh
-cd ~/synapse
-python -m synapse.app.homeserver \
-    --server-name my.domain.name \
-    --config-path homeserver.yaml \
-    --generate-config \
-    --report-stats=[yes|no]
-```
-
-... substituting an appropriate value for `--server-name`.
-
-This command will generate you a config file that you can then customise, but it will
-also generate a set of keys for you. These keys will allow your homeserver to
-identify itself to other homeserver, so don't lose or delete them. It would be
-wise to back them up somewhere safe. (If, for whatever reason, you do need to
-change your homeserver's keys, you may find that other homeserver have the
-old key cached. If you update the signing key, you should change the name of the
-key in the `<server name>.signing.key` file (the second word) to something
-different. See the [spec](https://matrix.org/docs/spec/server_server/latest.html#retrieving-server-keys) for more information on key management).
-
-To actually run your new homeserver, pick a working directory for Synapse to
-run (e.g. `~/synapse`), and:
-
-```sh
-cd ~/synapse
-source env/bin/activate
-synctl start
-```
-
-#### Platform-specific prerequisites
-
-Synapse is written in Python but some of the libraries it uses are written in
-C. So before we can install Synapse itself we need a working C compiler and the
-header files for Python C extensions.
-
-##### Debian/Ubuntu/Raspbian
-
-Installing prerequisites on Ubuntu or Debian:
-
-```sh
-sudo apt install build-essential python3-dev libffi-dev \
-                     python3-pip python3-setuptools sqlite3 \
-                     libssl-dev virtualenv libjpeg-dev libxslt1-dev
-```
-
-##### ArchLinux
-
-Installing prerequisites on ArchLinux:
-
-```sh
-sudo pacman -S base-devel python python-pip \
-               python-setuptools python-virtualenv sqlite3
-```
-
-##### CentOS/Fedora
-
-Installing prerequisites on CentOS or Fedora Linux:
-
-```sh
-sudo dnf install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
-                 libwebp-devel libxml2-devel libxslt-devel libpq-devel \
-                 python3-virtualenv libffi-devel openssl-devel python3-devel
-sudo dnf groupinstall "Development Tools"
-```
-
-##### macOS
-
-Installing prerequisites on macOS:
-
-```sh
-xcode-select --install
-sudo easy_install pip
-sudo pip install virtualenv
-brew install pkg-config libffi
-```
-
-On macOS Catalina (10.15) you may need to explicitly install OpenSSL
-via brew and inform `pip` about it so that `psycopg2` builds:
-
-```sh
-brew install openssl@1.1
-export LDFLAGS="-L/usr/local/opt/openssl/lib"
-export CPPFLAGS="-I/usr/local/opt/openssl/include"
-```
-
-##### OpenSUSE
-
-Installing prerequisites on openSUSE:
-
-```sh
-sudo zypper in -t pattern devel_basis
-sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \
-               python-devel libffi-devel libopenssl-devel libjpeg62-devel
-```
-
-##### OpenBSD
-
-A port of Synapse is available under `net/synapse`. The filesystem
-underlying the homeserver directory (defaults to `/var/synapse`) has to be
-mounted with `wxallowed` (cf. `mount(8)`), so creating a separate filesystem
-and mounting it to `/var/synapse` should be taken into consideration.
-
-To be able to build Synapse's dependency on python the `WRKOBJDIR`
-(cf. `bsd.port.mk(5)`) for building python, too, needs to be on a filesystem
-mounted with `wxallowed` (cf. `mount(8)`).
-
-Creating a `WRKOBJDIR` for building python under `/usr/local` (which on a
-default OpenBSD installation is mounted with `wxallowed`):
-
-```sh
-doas mkdir /usr/local/pobj_wxallowed
-```
-
-Assuming `PORTS_PRIVSEP=Yes` (cf. `bsd.port.mk(5)`) and `SUDO=doas` are
-configured in `/etc/mk.conf`:
-
-```sh
-doas chown _pbuild:_pbuild /usr/local/pobj_wxallowed
-```
-
-Setting the `WRKOBJDIR` for building python:
-
-```sh
-echo WRKOBJDIR_lang/python/3.7=/usr/local/pobj_wxallowed  \\nWRKOBJDIR_lang/python/2.7=/usr/local/pobj_wxallowed >> /etc/mk.conf
-```
-
-Building Synapse:
-
-```sh
-cd /usr/ports/net/synapse
-make install
-```
-
-##### Windows
-
-If you wish to run or develop Synapse on Windows, the Windows Subsystem For
-Linux provides a Linux environment on Windows 10 which is capable of using the
-Debian, Fedora, or source installation methods. More information about WSL can
-be found at <https://docs.microsoft.com/en-us/windows/wsl/install-win10> for
-Windows 10 and <https://docs.microsoft.com/en-us/windows/wsl/install-on-server>
-for Windows Server.
-
-### Prebuilt packages
-
-As an alternative to installing from source, prebuilt packages are available
-for a number of platforms.
-
-#### Docker images and Ansible playbooks
-
-There is an official synapse image available at
-<https://hub.docker.com/r/matrixdotorg/synapse> which can be used with
-the docker-compose file available at [contrib/docker](contrib/docker). Further
-information on this including configuration options is available in the README
-on hub.docker.com.
-
-Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
-Dockerfile to automate a synapse server in a single Docker image, at
-<https://hub.docker.com/r/avhost/docker-matrix/tags/>
-
-Slavi Pantaleev has created an Ansible playbook,
-which installs the offical Docker image of Matrix Synapse
-along with many other Matrix-related services (Postgres database, Element, coturn,
-ma1sd, SSL support, etc.).
-For more details, see
-<https://github.com/spantaleev/matrix-docker-ansible-deploy>
-
-#### Debian/Ubuntu
-
-##### Matrix.org packages
-
-Matrix.org provides Debian/Ubuntu packages of the latest stable version of
-Synapse via <https://packages.matrix.org/debian/>. They are available for Debian
-9 (Stretch), Ubuntu 16.04 (Xenial), and later. To use them:
-
-```sh
-sudo apt install -y lsb-release wget apt-transport-https
-sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
-echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" |
-    sudo tee /etc/apt/sources.list.d/matrix-org.list
-sudo apt update
-sudo apt install matrix-synapse-py3
-```
-
-**Note**: if you followed a previous version of these instructions which
-recommended using `apt-key add` to add an old key from
-`https://matrix.org/packages/debian/`, you should note that this key has been
-revoked. You should remove the old key with `sudo apt-key remove
-C35EB17E1EAE708E6603A9B3AD0592FE47F0DF61`, and follow the above instructions to
-update your configuration.
-
-The fingerprint of the repository signing key (as shown by `gpg
-/usr/share/keyrings/matrix-org-archive-keyring.gpg`) is
-`AAF9AE843A7584B5A3E4CD2BCF45A512DE2DA058`.
-
-##### Downstream Debian packages
-
-We do not recommend using the packages from the default Debian `buster`
-repository at this time, as they are old and suffer from known security
-vulnerabilities. You can install the latest version of Synapse from
-[our repository](#matrixorg-packages) or from `buster-backports`. Please
-see the [Debian documentation](https://backports.debian.org/Instructions/)
-for information on how to use backports.
-
-If you are using Debian `sid` or testing, Synapse is available in the default
-repositories and it should be possible to install it simply with:
-
-```sh
-sudo apt install matrix-synapse
-```
-
-##### Downstream Ubuntu packages
-
-We do not recommend using the packages in the default Ubuntu repository
-at this time, as they are old and suffer from known security vulnerabilities.
-The latest version of Synapse can be installed from [our repository](#matrixorg-packages).
-
-#### Fedora
-
-Synapse is in the Fedora repositories as `matrix-synapse`:
-
-```sh
-sudo dnf install matrix-synapse
-```
-
-Oleg Girko provides Fedora RPMs at
-<https://obs.infoserver.lv/project/monitor/matrix-synapse>
-
-#### OpenSUSE
-
-Synapse is in the OpenSUSE repositories as `matrix-synapse`:
-
-```sh
-sudo zypper install matrix-synapse
-```
-
-#### SUSE Linux Enterprise Server
-
-Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 repository at
-<https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/>
-
-#### ArchLinux
-
-The quickest way to get up and running with ArchLinux is probably with the community package
-<https://www.archlinux.org/packages/community/any/matrix-synapse/>, which should pull in most of
-the necessary dependencies.
-
-pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):
-
-```sh
-sudo pip install --upgrade pip
-```
-
-If you encounter an error with lib bcrypt causing an Wrong ELF Class:
-ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly
-compile it under the right architecture. (This should not be needed if
-installing under virtualenv):
-
-```sh
-sudo pip uninstall py-bcrypt
-sudo pip install py-bcrypt
-```
-
-#### Void Linux
-
-Synapse can be found in the void repositories as 'synapse':
-
-```sh
-xbps-install -Su
-xbps-install -S synapse
-```
-
-#### FreeBSD
-
-Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from:
-
-- Ports: `cd /usr/ports/net-im/py-matrix-synapse && make install clean`
-- Packages: `pkg install py37-matrix-synapse`
-
-#### OpenBSD
-
-As of OpenBSD 6.7 Synapse is available as a pre-compiled binary. The filesystem
-underlying the homeserver directory (defaults to `/var/synapse`) has to be
-mounted with `wxallowed` (cf. `mount(8)`), so creating a separate filesystem
-and mounting it to `/var/synapse` should be taken into consideration.
-
-Installing Synapse:
-
-```sh
-doas pkg_add synapse
-```
-
-#### NixOS
-
-Robin Lambertz has packaged Synapse for NixOS at:
-<https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-synapse.nix>
-
-## Setting up Synapse
-
-Once you have installed synapse as above, you will need to configure it.
-
-### Using PostgreSQL
-
-By default Synapse uses [SQLite](https://sqlite.org/) and in doing so trades performance for convenience.
-SQLite is only recommended in Synapse for testing purposes or for servers with
-very light workloads.
-
-Almost all installations should opt to use [PostgreSQL](https://www.postgresql.org). Advantages include:
-
-- significant performance improvements due to the superior threading and
-  caching model, smarter query optimiser
-- allowing the DB to be run on separate hardware
-
-For information on how to install and use PostgreSQL in Synapse, please see
-[docs/postgres.md](docs/postgres.md)
-
-### TLS certificates
-
-The default configuration exposes a single HTTP port on the local
-interface: `http://localhost:8008`. It is suitable for local testing,
-but for any practical use, you will need Synapse's APIs to be served
-over HTTPS.
-
-The recommended way to do so is to set up a reverse proxy on port
-`8448`. You can find documentation on doing so in
-[docs/reverse_proxy.md](docs/reverse_proxy.md).
-
-Alternatively, you can configure Synapse to expose an HTTPS port. To do
-so, you will need to edit `homeserver.yaml`, as follows:
-
-- First, under the `listeners` section, uncomment the configuration for the
-  TLS-enabled listener. (Remove the hash sign (`#`) at the start of
-  each line). The relevant lines are like this:
-
-```yaml
-  - port: 8448
-    type: http
-    tls: true
-    resources:
-      - names: [client, federation]
-  ```
-
-- You will also need to uncomment the `tls_certificate_path` and
-  `tls_private_key_path` lines under the `TLS` section. You will need to manage
-  provisioning of these certificates yourself — Synapse had built-in ACME
-  support, but the ACMEv1 protocol Synapse implements is deprecated, not
-  allowed by LetsEncrypt for new sites, and will break for existing sites in
-  late 2020. See [ACME.md](docs/ACME.md).
-
-  If you are using your own certificate, be sure to use a `.pem` file that
-  includes the full certificate chain including any intermediate certificates
-  (for instance, if using certbot, use `fullchain.pem` as your certificate, not
-  `cert.pem`).
-
-For a more detailed guide to configuring your server for federation, see
-[federate.md](docs/federate.md).
-
-### Client Well-Known URI
-
-Setting up the client Well-Known URI is optional but if you set it up, it will
-allow users to enter their full username (e.g. `@user:<server_name>`) into clients
-which support well-known lookup to automatically configure the homeserver and
-identity server URLs. This is useful so that users don't have to memorize or think
-about the actual homeserver URL you are using.
-
-The URL `https://<server_name>/.well-known/matrix/client` should return JSON in
-the following format.
-
-```json
-{
-  "m.homeserver": {
-    "base_url": "https://<matrix.example.com>"
-  }
-}
-```
-
-It can optionally contain identity server information as well.
-
-```json
-{
-  "m.homeserver": {
-    "base_url": "https://<matrix.example.com>"
-  },
-  "m.identity_server": {
-    "base_url": "https://<identity.example.com>"
-  }
-}
-```
-
-To work in browser based clients, the file must be served with the appropriate
-Cross-Origin Resource Sharing (CORS) headers. A recommended value would be
-`Access-Control-Allow-Origin: *` which would allow all browser based clients to
-view it.
-
-In nginx this would be something like:
-
-```nginx
-location /.well-known/matrix/client {
-    return 200 '{"m.homeserver": {"base_url": "https://<matrix.example.com>"}}';
-    default_type application/json;
-    add_header Access-Control-Allow-Origin *;
-}
-```
-
-You should also ensure the `public_baseurl` option in `homeserver.yaml` is set
-correctly. `public_baseurl` should be set to the URL that clients will use to
-connect to your server. This is the same URL you put for the `m.homeserver`
-`base_url` above.
-
-```yaml
-public_baseurl: "https://<matrix.example.com>"
-```
-
-### Email
-
-It is desirable for Synapse to have the capability to send email. This allows
-Synapse to send password reset emails, send verifications when an email address
-is added to a user's account, and send email notifications to users when they
-receive new messages.
-
-To configure an SMTP server for Synapse, modify the configuration section
-headed `email`, and be sure to have at least the `smtp_host`, `smtp_port`
-and `notif_from` fields filled out.  You may also need to set `smtp_user`,
-`smtp_pass`, and `require_transport_security`.
-
-If email is not configured, password reset, registration and notifications via
-email will be disabled.
-
-### Registering a user
-
-The easiest way to create a new user is to do so from a client like [Element](https://element.io/).
-
-Alternatively, you can do so from the command line. This can be done as follows:
-
- 1. If synapse was installed via pip, activate the virtualenv as follows (if Synapse was
-    installed via a prebuilt package, `register_new_matrix_user` should already be
-    on the search path):
-    ```sh
-    cd ~/synapse
-    source env/bin/activate
-    synctl start # if not already running
-    ```
- 2. Run the following command:
-    ```sh
-    register_new_matrix_user -c homeserver.yaml http://localhost:8008
-    ```
-
-This will prompt you to add details for the new user, and will then connect to
-the running Synapse to create the new user. For example:
-```
-New user localpart: erikj
-Password:
-Confirm password:
-Make admin [no]:
-Success!
-```
-
-This process uses a setting `registration_shared_secret` in
-`homeserver.yaml`, which is shared between Synapse itself and the
-`register_new_matrix_user` script. It doesn't matter what it is (a random
-value is generated by `--generate-config`), but it should be kept secret, as
-anyone with knowledge of it can register users, including admin accounts,
-on your server even if `enable_registration` is `false`.
-
-### Setting up a TURN server
-
-For reliable VoIP calls to be routed via this homeserver, you MUST configure
-a TURN server. See [docs/turn-howto.md](docs/turn-howto.md) for details.
-
-### URL previews
-
-Synapse includes support for previewing URLs, which is disabled by default.  To
-turn it on you must enable the `url_preview_enabled: True` config parameter
-and explicitly specify the IP ranges that Synapse is not allowed to spider for
-previewing in the `url_preview_ip_range_blacklist` configuration parameter.
-This is critical from a security perspective to stop arbitrary Matrix users
-spidering 'internal' URLs on your network. At the very least we recommend that
-your loopback and RFC1918 IP addresses are blacklisted.
-
-This also requires the optional `lxml` python dependency to be  installed. This
-in turn requires the `libxml2` library to be available - on  Debian/Ubuntu this
-means `apt-get install libxml2-dev`, or equivalent for your OS.
-
-### Troubleshooting Installation
-
-`pip` seems to leak *lots* of memory during installation. For instance, a Linux
-host with 512MB of RAM may run out of memory whilst installing Twisted. If this
-happens, you will have to individually install the dependencies which are
-failing, e.g.:
-
-```sh
-pip install twisted
-```
-
-If you have any other problems, feel free to ask in
-[#synapse:matrix.org](https://matrix.to/#/#synapse:matrix.org).

MANIFEST.in

@@ -8,22 +8,15 @@ include demo/demo.tls.dh
 include demo/*.py
 include demo/*.sh
 
-recursive-include synapse/storage *.sql
-recursive-include synapse/storage *.sql.postgres
-recursive-include synapse/storage *.sql.sqlite
-recursive-include synapse/storage *.py
-recursive-include synapse/storage *.txt
-recursive-include synapse/storage *.md
+recursive-include synapse/storage/schema *.sql
+recursive-include synapse/storage/schema *.py
 
 recursive-include docs *
 recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
-recursive-include tests *.py
 recursive-include tests *.pem
-recursive-include tests *.p8
-recursive-include tests *.crt
-recursive-include tests *.key
+recursive-include tests *.py
 
 recursive-include synapse/res *
 recursive-include synapse/static *.css
@@ -31,24 +24,21 @@ recursive-include synapse/static *.gif
 recursive-include synapse/static *.html
 recursive-include synapse/static *.js
 
-exclude .codecov.yml
-exclude .coveragerc
-exclude .dockerignore
-exclude .editorconfig
 exclude Dockerfile
-exclude mypy.ini
-exclude sytest-blacklist
+exclude .dockerignore
 exclude test_postgresql.sh
+exclude .editorconfig
 
 include pyproject.toml
 recursive-include changelog.d *
 
-prune .buildkite
-prune .circleci
 prune .github
-prune contrib
-prune debian
 prune demo/etc
 prune docker
-prune snap
-prune stubs
+prune .circleci
+prune .coveragerc
+prune debian
+prune .codecov.yml
+
+exclude jenkins*
+recursive-exclude jenkins *.sh

changelog.d/3902.feature

@@ -0,0 +1 @@
+Include m.room.encryption on invites by default

changelog.d/xxxx.misc

@@ -1 +0,0 @@
-Ensure python uses `malloc` when running Synapse in Docker.

contrib/cmdclient/console.py

@@ -15,6 +15,10 @@
 # limitations under the License.
 
 """ Starts a synapse client console. """
+
+from twisted.internet import reactor, defer, threads
+from http import TwistedHttpClient
+
 import argparse
 import cmd
 import getpass
@@ -23,22 +27,18 @@ import shlex
 import sys
 import time
 import urllib
-from http import TwistedHttpClient
-from typing import Optional
-
-import nacl.encoding
-import nacl.signing
 import urlparse
-from signedjson.sign import SignatureVerifyException, verify_signed_json
 
-from twisted.internet import defer, reactor, threads
+import nacl.signing
+import nacl.encoding
+
+from signedjson.sign import verify_signed_json, SignatureVerifyException
 
 CONFIG_JSON = "cmdclient_config.json"
 
-# TODO: The concept of trusted identity servers has been deprecated. This option and checks
-#  should be removed
-TRUSTED_ID_SERVERS = ["localhost:8001"]
-
+TRUSTED_ID_SERVERS = [
+    'localhost:8001'
+]
 
 class SynapseCmd(cmd.Cmd):
 
@@ -58,7 +58,7 @@ class SynapseCmd(cmd.Cmd):
             "token": token,
             "verbose": "on",
             "complete_usernames": "on",
-            "send_delivery_receipts": "on",
+            "send_delivery_receipts": "on"
         }
         self.path_prefix = "/_matrix/client/api/v1"
         self.event_stream_token = "END"
@@ -93,7 +93,7 @@ class SynapseCmd(cmd.Cmd):
         return self.config["user"].split(":")[1]
 
     def do_config(self, line):
-        """Show the config for this client: "config"
+        """ Show the config for this client: "config"
         Edit a key value mapping: "config key value" e.g. "config token 1234"
         Config variables:
             user: The username to auth with.
@@ -109,7 +109,7 @@ class SynapseCmd(cmd.Cmd):
         by using $. E.g. 'config roomid room1' then 'raw get /rooms/$roomid'.
         """
         if len(line) == 0:
-            print(json.dumps(self.config, indent=4))
+            print json.dumps(self.config, indent=4)
             return
 
         try:
@@ -119,11 +119,12 @@ class SynapseCmd(cmd.Cmd):
             config_rules = [  # key, valid_values
                 ("verbose", ["on", "off"]),
                 ("complete_usernames", ["on", "off"]),
-                ("send_delivery_receipts", ["on", "off"]),
+                ("send_delivery_receipts", ["on", "off"])
             ]
             for key, valid_vals in config_rules:
                 if key == args["key"] and args["val"] not in valid_vals:
-                    print("%s value must be one of %s" % (args["key"], valid_vals))
+                    print "%s value must be one of %s" % (args["key"],
+                                                          valid_vals)
                     return
 
             # toggle the http client verbosity
@@ -132,11 +133,11 @@ class SynapseCmd(cmd.Cmd):
 
             # assign the new config
             self.config[args["key"]] = args["val"]
-            print(json.dumps(self.config, indent=4))
+            print json.dumps(self.config, indent=4)
 
             save_config(self.config)
         except Exception as e:
-            print(e)
+            print e
 
     def do_register(self, line):
         """Registers for a new account: "register <userid> <noupdate>"
@@ -152,32 +153,33 @@ class SynapseCmd(cmd.Cmd):
             pwd = getpass.getpass("Type a password for this user: ")
             pwd2 = getpass.getpass("Retype the password: ")
             if pwd != pwd2 or len(pwd) == 0:
-                print("Password mismatch.")
+                print "Password mismatch."
                 pwd = None
             else:
                 password = pwd
 
-        body = {"type": "m.login.password"}
+        body = {
+            "type": "m.login.password"
+        }
         if "userid" in args:
             body["user"] = args["userid"]
         if password:
             body["password"] = password
 
-        reactor.callFromThread(self._do_register, body, "noupdate" not in args)
+        reactor.callFromThread(self._do_register, body,
+                               "noupdate" not in args)
 
     @defer.inlineCallbacks
     def _do_register(self, data, update_config):
         # check the registration flows
         url = self._url() + "/register"
         json_res = yield self.http_client.do_request("GET", url)
-        print(json.dumps(json_res, indent=4))
+        print json.dumps(json_res, indent=4)
 
         passwordFlow = None
         for flow in json_res["flows"]:
-            if flow["type"] == "m.login.recaptcha" or (
-                "stages" in flow and "m.login.recaptcha" in flow["stages"]
-            ):
-                print("Unable to register: Home server requires captcha.")
+            if flow["type"] == "m.login.recaptcha" or ("stages" in flow and "m.login.recaptcha" in flow["stages"]):
+                print "Unable to register: Home server requires captcha."
                 return
             if flow["type"] == "m.login.password" and "stages" not in flow:
                 passwordFlow = flow
@@ -187,7 +189,7 @@ class SynapseCmd(cmd.Cmd):
             return
 
         json_res = yield self.http_client.do_request("POST", url, data=data)
-        print(json.dumps(json_res, indent=4))
+        print json.dumps(json_res, indent=4)
         if update_config and "user_id" in json_res:
             self.config["user"] = json_res["user_id"]
             self.config["token"] = json_res["access_token"]
@@ -199,7 +201,9 @@ class SynapseCmd(cmd.Cmd):
         """
         try:
             args = self._parse(line, ["user_id"], force_keys=True)
-            can_login = threads.blockingCallFromThread(reactor, self._check_can_login)
+            can_login = threads.blockingCallFromThread(
+                reactor,
+                self._check_can_login)
             if can_login:
                 p = getpass.getpass("Enter your password: ")
                 user = args["user_id"]
@@ -207,25 +211,29 @@ class SynapseCmd(cmd.Cmd):
                     domain = self._domain()
                     if domain:
                         user = "@" + user + ":" + domain
-
+                
                 reactor.callFromThread(self._do_login, user, p)
-                # print " got %s " % p
+                #print " got %s " % p
         except Exception as e:
-            print(e)
+            print e
 
     @defer.inlineCallbacks
     def _do_login(self, user, password):
         path = "/login"
-        data = {"user": user, "password": password, "type": "m.login.password"}
+        data = {
+            "user": user,
+            "password": password,
+            "type": "m.login.password"
+        }
         url = self._url() + path
         json_res = yield self.http_client.do_request("POST", url, data=data)
-        print(json_res)
+        print json_res
 
         if "access_token" in json_res:
             self.config["user"] = user
             self.config["token"] = json_res["access_token"]
             save_config(self.config)
-            print("Login successful.")
+            print "Login successful."
 
     @defer.inlineCallbacks
     def _check_can_login(self):
@@ -234,19 +242,18 @@ class SynapseCmd(cmd.Cmd):
         # submitting!
         url = self._url() + path
         json_res = yield self.http_client.do_request("GET", url)
-        print(json_res)
+        print json_res
 
         if "flows" not in json_res:
-            print("Failed to find any login flows.")
+            print "Failed to find any login flows."
             defer.returnValue(False)
 
-        flow = json_res["flows"][0]  # assume first is the one we want.
-        if "type" not in flow or "m.login.password" != flow["type"] or "stages" in flow:
+        flow = json_res["flows"][0] # assume first is the one we want.
+        if ("type" not in flow or "m.login.password" != flow["type"] or
+                "stages" in flow):
             fallback_url = self._url() + "/login/fallback"
-            print(
-                "Unable to login via the command line client. Please visit "
-                "%s to login." % fallback_url
-            )
+            print ("Unable to login via the command line client. Please visit "
+                "%s to login." % fallback_url)
             defer.returnValue(False)
         defer.returnValue(True)
 
@@ -256,34 +263,21 @@ class SynapseCmd(cmd.Cmd):
         <clientSecret> A string of characters generated when requesting an email that you'll supply in subsequent calls to identify yourself
         <sendAttempt> The number of times the user has requested an email. Leave this the same between requests to retry the request at the transport level. Increment it to request that the email be sent again.
         """
-        args = self._parse(line, ["address", "clientSecret", "sendAttempt"])
+        args = self._parse(line, ['address', 'clientSecret', 'sendAttempt'])
 
-        postArgs = {
-            "email": args["address"],
-            "clientSecret": args["clientSecret"],
-            "sendAttempt": args["sendAttempt"],
-        }
+        postArgs = {'email': args['address'], 'clientSecret': args['clientSecret'], 'sendAttempt': args['sendAttempt']}
 
         reactor.callFromThread(self._do_emailrequest, postArgs)
 
     @defer.inlineCallbacks
     def _do_emailrequest(self, args):
-        # TODO: Update to use v2 Identity Service API endpoint
-        url = (
-            self._identityServerUrl()
-            + "/_matrix/identity/api/v1/validate/email/requestToken"
-        )
+        url = self._identityServerUrl()+"/_matrix/identity/api/v1/validate/email/requestToken"
 
-        json_res = yield self.http_client.do_request(
-            "POST",
-            url,
-            data=urllib.urlencode(args),
-            jsonreq=False,
-            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
-        )
-        print(json_res)
-        if "sid" in json_res:
-            print("Token sent. Your session ID is %s" % (json_res["sid"]))
+        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
+                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
+        print json_res
+        if 'sid' in json_res:
+            print "Token sent. Your session ID is %s" % (json_res['sid'])
 
     def do_emailvalidate(self, line):
         """Validate and associate a third party ID
@@ -291,58 +285,39 @@ class SynapseCmd(cmd.Cmd):
         <token> The token sent to your third party identifier address
         <clientSecret> The same clientSecret you supplied in requestToken
         """
-        args = self._parse(line, ["sid", "token", "clientSecret"])
+        args = self._parse(line, ['sid', 'token', 'clientSecret'])
 
-        postArgs = {
-            "sid": args["sid"],
-            "token": args["token"],
-            "clientSecret": args["clientSecret"],
-        }
+        postArgs = { 'sid' : args['sid'], 'token' : args['token'], 'clientSecret': args['clientSecret'] }
 
         reactor.callFromThread(self._do_emailvalidate, postArgs)
 
     @defer.inlineCallbacks
     def _do_emailvalidate(self, args):
-        # TODO: Update to use v2 Identity Service API endpoint
-        url = (
-            self._identityServerUrl()
-            + "/_matrix/identity/api/v1/validate/email/submitToken"
-        )
+        url = self._identityServerUrl()+"/_matrix/identity/api/v1/validate/email/submitToken"
 
-        json_res = yield self.http_client.do_request(
-            "POST",
-            url,
-            data=urllib.urlencode(args),
-            jsonreq=False,
-            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
-        )
-        print(json_res)
+        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
+                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
+        print json_res
 
     def do_3pidbind(self, line):
         """Validate and associate a third party ID
         <sid> The session ID (sid) given to you in the response to requestToken
         <clientSecret> The same clientSecret you supplied in requestToken
         """
-        args = self._parse(line, ["sid", "clientSecret"])
+        args = self._parse(line, ['sid', 'clientSecret'])
 
-        postArgs = {"sid": args["sid"], "clientSecret": args["clientSecret"]}
-        postArgs["mxid"] = self.config["user"]
+        postArgs = { 'sid' : args['sid'], 'clientSecret': args['clientSecret'] }
+        postArgs['mxid'] = self.config["user"]
 
         reactor.callFromThread(self._do_3pidbind, postArgs)
 
     @defer.inlineCallbacks
     def _do_3pidbind(self, args):
-        # TODO: Update to use v2 Identity Service API endpoint
-        url = self._identityServerUrl() + "/_matrix/identity/api/v1/3pid/bind"
+        url = self._identityServerUrl()+"/_matrix/identity/api/v1/3pid/bind"
 
-        json_res = yield self.http_client.do_request(
-            "POST",
-            url,
-            data=urllib.urlencode(args),
-            jsonreq=False,
-            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
-        )
-        print(json_res)
+        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
+                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
+        print json_res
 
     def do_join(self, line):
         """Joins a room: "join <roomid>" """
@@ -350,7 +325,7 @@ class SynapseCmd(cmd.Cmd):
             args = self._parse(line, ["roomid"], force_keys=True)
             self._do_membership_change(args["roomid"], "join", self._usr())
         except Exception as e:
-            print(e)
+            print e
 
     def do_joinalias(self, line):
         try:
@@ -358,34 +333,36 @@ class SynapseCmd(cmd.Cmd):
             path = "/join/%s" % urllib.quote(args["roomname"])
             reactor.callFromThread(self._run_and_pprint, "POST", path, {})
         except Exception as e:
-            print(e)
+            print e
 
     def do_topic(self, line):
-        """ "topic [set|get] <roomid> [<newtopic>]"
+        """"topic [set|get] <roomid> [<newtopic>]"
         Set the topic for a room: topic set <roomid> <newtopic>
         Get the topic for a room: topic get <roomid>
         """
         try:
             args = self._parse(line, ["action", "roomid", "topic"])
             if "action" not in args or "roomid" not in args:
-                print("Must specify set|get and a room ID.")
+                print "Must specify set|get and a room ID."
                 return
             if args["action"].lower() not in ["set", "get"]:
-                print("Must specify set|get, not %s" % args["action"])
+                print "Must specify set|get, not %s" % args["action"]
                 return
 
             path = "/rooms/%s/topic" % urllib.quote(args["roomid"])
 
             if args["action"].lower() == "set":
                 if "topic" not in args:
-                    print("Must specify a new topic.")
+                    print "Must specify a new topic."
                     return
-                body = {"topic": args["topic"]}
+                body = {
+                    "topic": args["topic"]
+                }
                 reactor.callFromThread(self._run_and_pprint, "PUT", path, body)
             elif args["action"].lower() == "get":
                 reactor.callFromThread(self._run_and_pprint, "GET", path)
         except Exception as e:
-            print(e)
+            print e
 
     def do_invite(self, line):
         """Invite a user to a room: "invite <userid> <roomid>" """
@@ -396,66 +373,49 @@ class SynapseCmd(cmd.Cmd):
 
             reactor.callFromThread(self._do_invite, args["roomid"], user_id)
         except Exception as e:
-            print(e)
+            print e
 
     @defer.inlineCallbacks
     def _do_invite(self, roomid, userstring):
-        if not userstring.startswith("@") and self._is_on("complete_usernames"):
-            # TODO: Update to use v2 Identity Service API endpoint
-            url = self._identityServerUrl() + "/_matrix/identity/api/v1/lookup"
+        if (not userstring.startswith('@') and
+                    self._is_on("complete_usernames")):
+            url = self._identityServerUrl()+"/_matrix/identity/api/v1/lookup"
 
-            json_res = yield self.http_client.do_request(
-                "GET", url, qparams={"medium": "email", "address": userstring}
-            )
+            json_res = yield self.http_client.do_request("GET", url, qparams={'medium':'email','address':userstring})
 
             mxid = None
 
-            if "mxid" in json_res and "signatures" in json_res:
-                # TODO: Update to use v2 Identity Service API endpoint
-                url = (
-                    self._identityServerUrl()
-                    + "/_matrix/identity/api/v1/pubkey/ed25519"
-                )
+            if 'mxid' in json_res and 'signatures' in json_res:
+                url = self._identityServerUrl()+"/_matrix/identity/api/v1/pubkey/ed25519"
 
                 pubKey = None
                 pubKeyObj = yield self.http_client.do_request("GET", url)
-                if "public_key" in pubKeyObj:
-                    pubKey = nacl.signing.VerifyKey(
-                        pubKeyObj["public_key"], encoder=nacl.encoding.HexEncoder
-                    )
+                if 'public_key' in pubKeyObj:
+                    pubKey = nacl.signing.VerifyKey(pubKeyObj['public_key'], encoder=nacl.encoding.HexEncoder)
                 else:
-                    print("No public key found in pubkey response!")
+                    print "No public key found in pubkey response!"
 
                 sigValid = False
 
                 if pubKey:
-                    for signame in json_res["signatures"]:
+                    for signame in json_res['signatures']:
                         if signame not in TRUSTED_ID_SERVERS:
-                            print(
-                                "Ignoring signature from untrusted server %s"
-                                % (signame)
-                            )
+                            print "Ignoring signature from untrusted server %s" % (signame)
                         else:
                             try:
                                 verify_signed_json(json_res, signame, pubKey)
                                 sigValid = True
-                                print(
-                                    "Mapping %s -> %s correctly signed by %s"
-                                    % (userstring, json_res["mxid"], signame)
-                                )
+                                print "Mapping %s -> %s correctly signed by %s" % (userstring, json_res['mxid'], signame)
                                 break
                             except SignatureVerifyException as e:
-                                print("Invalid signature from %s" % (signame))
-                                print(e)
+                                print "Invalid signature from %s" % (signame)
+                                print e
 
                 if sigValid:
-                    print("Resolved 3pid %s to %s" % (userstring, json_res["mxid"]))
-                    mxid = json_res["mxid"]
+                    print "Resolved 3pid %s to %s" % (userstring, json_res['mxid'])
+                    mxid = json_res['mxid']
                 else:
-                    print(
-                        "Got association for %s but couldn't verify signature"
-                        % (userstring)
-                    )
+                    print "Got association for %s but couldn't verify signature" % (userstring)
 
             if not mxid:
                 mxid = "@" + userstring + ":" + self._domain()
@@ -468,17 +428,18 @@ class SynapseCmd(cmd.Cmd):
             args = self._parse(line, ["roomid"], force_keys=True)
             self._do_membership_change(args["roomid"], "leave", self._usr())
         except Exception as e:
-            print(e)
+            print e
 
     def do_send(self, line):
         """Sends a message. "send <roomid> <body>" """
         args = self._parse(line, ["roomid", "body"])
         txn_id = "txn%s" % int(time.time())
-        path = "/rooms/%s/send/m.room.message/%s" % (
-            urllib.quote(args["roomid"]),
-            txn_id,
-        )
-        body_json = {"msgtype": "m.text", "body": args["body"]}
+        path = "/rooms/%s/send/m.room.message/%s" % (urllib.quote(args["roomid"]),
+                                             txn_id)
+        body_json = {
+            "msgtype": "m.text",
+            "body": args["body"]
+        }
         reactor.callFromThread(self._run_and_pprint, "PUT", path, body_json)
 
     def do_list(self, line):
@@ -491,11 +452,11 @@ class SynapseCmd(cmd.Cmd):
         "list messages <roomid> from=END&to=START&limit=3"
         """
         args = self._parse(line, ["type", "roomid", "qp"])
-        if "type" not in args or "roomid" not in args:
-            print("Must specify type and room ID.")
+        if not "type" in args or not "roomid" in args:
+            print "Must specify type and room ID."
             return
         if args["type"] not in ["members", "messages"]:
-            print("Unrecognised type: %s" % args["type"])
+            print "Unrecognised type: %s" % args["type"]
             return
         room_id = args["roomid"]
         path = "/rooms/%s/%s" % (urllib.quote(room_id), args["type"])
@@ -506,11 +467,12 @@ class SynapseCmd(cmd.Cmd):
                 try:
                     key_value = key_value_str.split("=")
                     qp[key_value[0]] = key_value[1]
-                except Exception:
-                    print("Bad query param: %s" % key_value)
+                except:
+                    print "Bad query param: %s" % key_value
                     return
 
-        reactor.callFromThread(self._run_and_pprint, "GET", path, query_params=qp)
+        reactor.callFromThread(self._run_and_pprint, "GET", path,
+                               query_params=qp)
 
     def do_create(self, line):
         """Creates a room.
@@ -546,22 +508,14 @@ class SynapseCmd(cmd.Cmd):
         args = self._parse(line, ["method", "path", "data"])
         # sanity check
         if "method" not in args or "path" not in args:
-            print("Must specify path and method.")
+            print "Must specify path and method."
             return
 
         args["method"] = args["method"].upper()
-        valid_methods = [
-            "PUT",
-            "GET",
-            "POST",
-            "DELETE",
-            "XPUT",
-            "XGET",
-            "XPOST",
-            "XDELETE",
-        ]
+        valid_methods = ["PUT", "GET", "POST", "DELETE",
+                         "XPUT", "XGET", "XPOST", "XDELETE"]
         if args["method"] not in valid_methods:
-            print("Unsupported method: %s" % args["method"])
+            print "Unsupported method: %s" % args["method"]
             return
 
         if "data" not in args:
@@ -570,7 +524,7 @@ class SynapseCmd(cmd.Cmd):
             try:
                 args["data"] = json.loads(args["data"])
             except Exception as e:
-                print("Data is not valid JSON. %s" % e)
+                print "Data is not valid JSON. %s" % e
                 return
 
         qp = {"access_token": self._tok()}
@@ -583,16 +537,13 @@ class SynapseCmd(cmd.Cmd):
                 parsed_url = urlparse.urlparse(args["path"])
                 qp.update(urlparse.parse_qs(parsed_url.query))
                 args["path"] = parsed_url.path
-            except Exception:
+            except:
                 pass
 
-        reactor.callFromThread(
-            self._run_and_pprint,
-            args["method"],
-            args["path"],
-            args["data"],
-            query_params=qp,
-        )
+        reactor.callFromThread(self._run_and_pprint, args["method"],
+                                                     args["path"],
+                                                     args["data"],
+                                                     query_params=qp)
 
     def do_stream(self, line):
         """Stream data from the server: "stream <longpoll timeout ms>" """
@@ -602,31 +553,26 @@ class SynapseCmd(cmd.Cmd):
             try:
                 timeout = int(args["timeout"])
             except ValueError:
-                print("Timeout must be in milliseconds.")
+                print "Timeout must be in milliseconds."
                 return
         reactor.callFromThread(self._do_event_stream, timeout)
 
     @defer.inlineCallbacks
     def _do_event_stream(self, timeout):
-        res = yield defer.ensureDeferred(
-            self.http_client.get_json(
+        res = yield self.http_client.get_json(
                 self._url() + "/events",
                 {
                     "access_token": self._tok(),
                     "timeout": str(timeout),
-                    "from": self.event_stream_token,
-                },
-            )
-        )
-        print(json.dumps(res, indent=4))
+                    "from": self.event_stream_token
+                })
+        print json.dumps(res, indent=4)
 
         if "chunk" in res:
             for event in res["chunk"]:
-                if (
-                    event["type"] == "m.room.message"
-                    and self._is_on("send_delivery_receipts")
-                    and event["user_id"] != self._usr()
-                ):  # not sent by us
+                if (event["type"] == "m.room.message" and
+                        self._is_on("send_delivery_receipts") and
+                        event["user_id"] != self._usr()):  # not sent by us
                     self._send_receipt(event, "d")
 
         # update the position in the stram
@@ -634,28 +580,18 @@ class SynapseCmd(cmd.Cmd):
             self.event_stream_token = res["end"]
 
     def _send_receipt(self, event, feedback_type):
-        path = "/rooms/%s/messages/%s/%s/feedback/%s/%s" % (
-            urllib.quote(event["room_id"]),
-            event["user_id"],
-            event["msg_id"],
-            self._usr(),
-            feedback_type,
-        )
+        path = ("/rooms/%s/messages/%s/%s/feedback/%s/%s" %
+               (urllib.quote(event["room_id"]), event["user_id"], event["msg_id"],
+                self._usr(), feedback_type))
         data = {}
-        reactor.callFromThread(
-            self._run_and_pprint,
-            "PUT",
-            path,
-            data=data,
-            alt_text="Sent receipt for %s" % event["msg_id"],
-        )
+        reactor.callFromThread(self._run_and_pprint, "PUT", path, data=data,
+                               alt_text="Sent receipt for %s" % event["msg_id"])
 
     def _do_membership_change(self, roomid, membership, userid):
-        path = "/rooms/%s/state/m.room.member/%s" % (
-            urllib.quote(roomid),
-            urllib.quote(userid),
-        )
-        data = {"membership": membership}
+        path = "/rooms/%s/state/m.room.member/%s" % (urllib.quote(roomid), urllib.quote(userid))
+        data = {
+            "membership": membership
+        }
         reactor.callFromThread(self._run_and_pprint, "PUT", path, data=data)
 
     def do_displayname(self, line):
@@ -691,7 +627,7 @@ class SynapseCmd(cmd.Cmd):
         self._do_presence_state(2, line)
 
     def _parse(self, line, keys, force_keys=False):
-        """Parses the given line.
+        """ Parses the given line.
 
         Args:
             line : The line to parse
@@ -708,21 +644,16 @@ class SynapseCmd(cmd.Cmd):
         for i, arg in enumerate(line_args):
             for config_key in self.config:
                 if ("$" + config_key) in arg:
-                    arg = arg.replace("$" + config_key, self.config[config_key])
+                    arg = arg.replace("$" + config_key,
+                                      self.config[config_key])
             line_args[i] = arg
 
         return dict(zip(keys, line_args))
 
     @defer.inlineCallbacks
-    def _run_and_pprint(
-        self,
-        method,
-        path,
-        data=None,
-        query_params: Optional[dict] = None,
-        alt_text=None,
-    ):
-        """Runs an HTTP request and pretty prints the output.
+    def _run_and_pprint(self, method, path, data=None,
+                        query_params={"access_token": None}, alt_text=None):
+        """ Runs an HTTP request and pretty prints the output.
 
         Args:
             method: HTTP method
@@ -730,37 +661,35 @@ class SynapseCmd(cmd.Cmd):
             data: Raw JSON data if any
             query_params: dict of query parameters to add to the url
         """
-        query_params = query_params or {"access_token": None}
-
         url = self._url() + path
         if "access_token" in query_params:
             query_params["access_token"] = self._tok()
 
-        json_res = yield self.http_client.do_request(
-            method, url, data=data, qparams=query_params
-        )
+        json_res = yield self.http_client.do_request(method, url,
+                                                    data=data,
+                                                    qparams=query_params)
         if alt_text:
-            print(alt_text)
+            print alt_text
         else:
-            print(json.dumps(json_res, indent=4))
+            print json.dumps(json_res, indent=4)
 
 
 def save_config(config):
-    with open(CONFIG_JSON, "w") as out:
+    with open(CONFIG_JSON, 'w') as out:
         json.dump(config, out)
 
 
 def main(server_url, identity_server_url, username, token, config_path):
-    print("Synapse command line client")
-    print("===========================")
-    print("Server: %s" % server_url)
-    print("Type 'help' to get started.")
-    print("Close this console with CTRL+C then CTRL+D.")
+    print "Synapse command line client"
+    print "==========================="
+    print "Server: %s" % server_url
+    print "Type 'help' to get started."
+    print "Close this console with CTRL+C then CTRL+D."
     if not username or not token:
-        print("-  'register <username>' - Register an account")
-        print("-  'stream' - Connect to the event stream")
-        print("-  'create <roomid>' - Create a room")
-        print("-  'send <roomid> <message>' - Send a message")
+        print "-  'register <username>' - Register an account"
+        print "-  'stream' - Connect to the event stream"
+        print "-  'create <roomid>' - Create a room"
+        print "-  'send <roomid> <message>' - Send a message"
     http_client = TwistedHttpClient()
 
     # the command line client
@@ -770,14 +699,14 @@ def main(server_url, identity_server_url, username, token, config_path):
     global CONFIG_JSON
     CONFIG_JSON = config_path  # bit cheeky, but just overwrite the global
     try:
-        with open(config_path, "r") as config:
+        with open(config_path, 'r') as config:
             syn_cmd.config = json.load(config)
             try:
                 http_client.verbose = "on" == syn_cmd.config["verbose"]
-            except Exception:
+            except:
                 pass
-            print("Loaded config from %s" % config_path)
-    except Exception:
+            print "Loaded config from %s" % config_path
+    except:
         pass
 
     # Twisted-specific: Runs the command processor in Twisted's event loop
@@ -787,37 +716,27 @@ def main(server_url, identity_server_url, username, token, config_path):
     reactor.run()
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     parser = argparse.ArgumentParser("Starts a synapse client.")
     parser.add_argument(
-        "-s",
-        "--server",
-        dest="server",
-        default="http://localhost:8008",
-        help="The URL of the home server to talk to.",
-    )
+        "-s", "--server", dest="server", default="http://localhost:8008",
+        help="The URL of the home server to talk to.")
     parser.add_argument(
-        "-i",
-        "--identity-server",
-        dest="identityserver",
-        default="http://localhost:8090",
-        help="The URL of the identity server to talk to.",
-    )
+        "-i", "--identity-server", dest="identityserver", default="http://localhost:8090",
+        help="The URL of the identity server to talk to.")
     parser.add_argument(
-        "-u", "--username", dest="username", help="Your username on the server."
-    )
-    parser.add_argument("-t", "--token", dest="token", help="Your access token.")
+        "-u", "--username", dest="username",
+        help="Your username on the server.")
     parser.add_argument(
-        "-c",
-        "--config",
-        dest="config",
-        default=CONFIG_JSON,
-        help="The location of the config.json file to read from.",
-    )
+        "-t", "--token", dest="token",
+        help="Your access token.")
+    parser.add_argument(
+        "-c", "--config", dest="config", default=CONFIG_JSON,
+        help="The location of the config.json file to read from.")
     args = parser.parse_args()
 
     if not args.server:
-        print("You must supply a server URL to communicate with.")
+        print "You must supply a server URL to communicate with."
         parser.print_help()
         sys.exit(1)
 

contrib/cmdclient/http.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,21 +13,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
-import urllib
-from pprint import pformat
-from typing import Optional
-
-from twisted.internet import defer, reactor
 from twisted.web.client import Agent, readBody
 from twisted.web.http_headers import Headers
+from twisted.internet import defer, reactor
+
+from pprint import pformat
+
+import json
+import urllib
 
 
-class HttpClient:
-    """Interface for talking json over http"""
+class HttpClient(object):
+    """ Interface for talking json over http
+    """
 
     def put_json(self, url, data):
-        """Sends the specifed json data using PUT
+        """ Sends the specifed json data using PUT
 
         Args:
             url (str): The URL to PUT data to.
@@ -40,7 +42,7 @@ class HttpClient:
         pass
 
     def get_json(self, url, args=None):
-        """Gets some json from the given host homeserver and path
+        """ Gets some json from the given host homeserver and path
 
         Args:
             url (str): The URL to GET data from.
@@ -57,7 +59,7 @@ class HttpClient:
 
 
 class TwistedHttpClient(HttpClient):
-    """Wrapper around the twisted HTTP client api.
+    """ Wrapper around the twisted HTTP client api.
 
     Attributes:
         agent (twisted.web.client.Agent): The twisted Agent used to send the
@@ -70,7 +72,9 @@ class TwistedHttpClient(HttpClient):
     @defer.inlineCallbacks
     def put_json(self, url, data):
         response = yield self._create_put_request(
-            url, data, headers_dict={"Content-Type": ["application/json"]}
+            url,
+            data,
+            headers_dict={"Content-Type": ["application/json"]}
         )
         body = yield readBody(response)
         defer.returnValue((response.code, body))
@@ -85,46 +89,45 @@ class TwistedHttpClient(HttpClient):
         body = yield readBody(response)
         defer.returnValue(json.loads(body))
 
-    def _create_put_request(self, url, json_data, headers_dict: Optional[dict] = None):
-        """Wrapper of _create_request to issue a PUT request"""
-        headers_dict = headers_dict or {}
+    def _create_put_request(self, url, json_data, headers_dict={}):
+        """ Wrapper of _create_request to issue a PUT request
+        """
 
         if "Content-Type" not in headers_dict:
-            raise defer.error(RuntimeError("Must include Content-Type header for PUTs"))
+            raise defer.error(
+                RuntimeError("Must include Content-Type header for PUTs"))
 
         return self._create_request(
-            "PUT", url, producer=_JsonProducer(json_data), headers_dict=headers_dict
+            "PUT",
+            url,
+            producer=_JsonProducer(json_data),
+            headers_dict=headers_dict
         )
 
-    def _create_get_request(self, url, headers_dict: Optional[dict] = None):
-        """Wrapper of _create_request to issue a GET request"""
-        return self._create_request("GET", url, headers_dict=headers_dict or {})
+    def _create_get_request(self, url, headers_dict={}):
+        """ Wrapper of _create_request to issue a GET request
+        """
+        return self._create_request(
+            "GET",
+            url,
+            headers_dict=headers_dict
+        )
 
     @defer.inlineCallbacks
-    def do_request(
-        self,
-        method,
-        url,
-        data=None,
-        qparams=None,
-        jsonreq=True,
-        headers: Optional[dict] = None,
-    ):
-        headers = headers or {}
-
+    def do_request(self, method, url, data=None, qparams=None, jsonreq=True, headers={}):
         if qparams:
             url = "%s?%s" % (url, urllib.urlencode(qparams, True))
 
         if jsonreq:
             prod = _JsonProducer(data)
-            headers["Content-Type"] = ["application/json"]
+            headers['Content-Type'] = ["application/json"];
         else:
             prod = _RawProducer(data)
 
         if method in ["POST", "PUT"]:
-            response = yield self._create_request(
-                method, url, producer=prod, headers_dict=headers
-            )
+            response = yield self._create_request(method, url,
+                    producer=prod,
+                    headers_dict=headers)
         else:
             response = yield self._create_request(method, url)
 
@@ -132,33 +135,33 @@ class TwistedHttpClient(HttpClient):
         defer.returnValue(json.loads(body))
 
     @defer.inlineCallbacks
-    def _create_request(
-        self, method, url, producer=None, headers_dict: Optional[dict] = None
-    ):
-        """Creates and sends a request to the given url"""
-        headers_dict = headers_dict or {}
-
+    def _create_request(self, method, url, producer=None, headers_dict={}):
+        """ Creates and sends a request to the given url
+        """
         headers_dict["User-Agent"] = ["Synapse Cmd Client"]
 
         retries_left = 5
-        print("%s to %s with headers %s" % (method, url, headers_dict))
+        print "%s to %s with headers %s" % (method, url, headers_dict)
         if self.verbose and producer:
             if "password" in producer.data:
                 temp = producer.data["password"]
                 producer.data["password"] = "[REDACTED]"
-                print(json.dumps(producer.data, indent=4))
+                print json.dumps(producer.data, indent=4)
                 producer.data["password"] = temp
             else:
-                print(json.dumps(producer.data, indent=4))
+                print json.dumps(producer.data, indent=4)
 
         while True:
             try:
                 response = yield self.agent.request(
-                    method, url.encode("UTF8"), Headers(headers_dict), producer
+                    method,
+                    url.encode("UTF8"),
+                    Headers(headers_dict),
+                    producer
                 )
                 break
             except Exception as e:
-                print("uh oh: %s" % e)
+                print "uh oh: %s" % e
                 if retries_left:
                     yield self.sleep(2 ** (5 - retries_left))
                     retries_left -= 1
@@ -166,8 +169,8 @@ class TwistedHttpClient(HttpClient):
                     raise e
 
         if self.verbose:
-            print("Status %s %s" % (response.code, response.phrase))
-            print(pformat(list(response.headers.getAllRawHeaders())))
+            print "Status %s %s" % (response.code, response.phrase)
+            print pformat(list(response.headers.getAllRawHeaders()))
         defer.returnValue(response)
 
     def sleep(self, seconds):
@@ -175,8 +178,7 @@ class TwistedHttpClient(HttpClient):
         reactor.callLater(seconds, d.callback, seconds)
         return d
 
-
-class _RawProducer:
+class _RawProducer(object):
     def __init__(self, data):
         self.data = data
         self.body = data
@@ -192,10 +194,9 @@ class _RawProducer:
     def stopProducing(self):
         pass
 
-
-class _JsonProducer:
-    """Used by the twisted http client to create the HTTP body from json"""
-
+class _JsonProducer(object):
+    """ Used by the twisted http client to create the HTTP body from json
+    """
     def __init__(self, jsn):
         self.data = jsn
         self.body = json.dumps(jsn).encode("utf8")

contrib/docker/README.md

@@ -1,26 +1,35 @@
-
 # Synapse Docker
 
-### Configuration
+### Automated configuration
+
+It is recommended that you use Docker Compose to run your containers, including
+this image and a Postgres server. A sample ``docker-compose.yml`` is provided,
+including example labels for reverse proxying and other artifacts.
+
+Read the section about environment variables and set at least mandatory variables,
+then run the server:
+
+```
+docker-compose up -d
+```
+
+If secrets are not specified in the environment variables, they will be generated
+as part of the startup. Please ensure these secrets are kept between launches of the
+Docker container, as their loss may require users to log in again.
+
+### Manual configuration
 
 A sample ``docker-compose.yml`` is provided, including example labels for
 reverse proxying and other artifacts. The docker-compose file is an example,
 please comment/uncomment sections that are not suitable for your usecase.
 
 Specify a ``SYNAPSE_CONFIG_PATH``, preferably to a persistent path,
-to use manual configuration.
-
-To generate a fresh `homeserver.yaml`, you can use the `generate` command.
-(See the [documentation](../../docker/README.md#generating-a-configuration-file)
-for more information.) You will need to specify appropriate values for at least the
-`SYNAPSE_SERVER_NAME` and `SYNAPSE_REPORT_STATS` environment variables. For example:
+to use manual configuration. To generate a fresh ``homeserver.yaml``, simply run:
 
 ```
-docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host -e SYNAPSE_REPORT_STATS=yes synapse generate
+docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host synapse generate
 ```
 
-(This will also generate necessary signing keys.)
-
 Then, customize your configuration and run the server:
 
 ```

contrib/docker/docker-compose.yml

@@ -15,7 +15,11 @@ services:
     restart: unless-stopped
     # See the readme for a full documentation of the environment settings
     environment:
-      - SYNAPSE_CONFIG_PATH=/data/homeserver.yaml
+      - SYNAPSE_SERVER_NAME=my.matrix.host
+      - SYNAPSE_REPORT_STATS=no
+      - SYNAPSE_ENABLE_REGISTRATION=yes
+      - SYNAPSE_LOG_LEVEL=INFO
+      - POSTGRES_PASSWORD=changeme
     volumes:
       # You may either store all the files in a local folder
       - ./files:/data
@@ -31,33 +35,16 @@ services:
       - 8448:8448/tcp
     # ... or use a reverse proxy, here is an example for traefik:
     labels:
-      # The following lines are valid for Traefik version 1.x:
       - traefik.enable=true
       - traefik.frontend.rule=Host:my.matrix.Host
       - traefik.port=8008
-      # Alternatively, for Traefik version 2.0:
-      - traefik.enable=true
-      - traefik.http.routers.http-synapse.entryPoints=http
-      - traefik.http.routers.http-synapse.rule=Host(`my.matrix.host`)
-      - traefik.http.middlewares.https_redirect.redirectscheme.scheme=https
-      - traefik.http.middlewares.https_redirect.redirectscheme.permanent=true
-      - traefik.http.routers.http-synapse.middlewares=https_redirect
-      - traefik.http.routers.https-synapse.entryPoints=https
-      - traefik.http.routers.https-synapse.rule=Host(`my.matrix.host`)
-      - traefik.http.routers.https-synapse.service=synapse
-      - traefik.http.routers.https-synapse.tls=true
-      - traefik.http.services.synapse.loadbalancer.server.port=8008
-      - traefik.http.routers.https-synapse.tls.certResolver=le-ssl
 
   db:
-    image: docker.io/postgres:12-alpine
+    image: docker.io/postgres:10-alpine
     # Change that password, of course!
     environment:
       - POSTGRES_USER=synapse
       - POSTGRES_PASSWORD=changeme
-      # ensure the database gets created correctly
-      # https://github.com/matrix-org/synapse/blob/master/docs/postgres.md#set-up-database
-      - POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=C --lc-ctype=C
     volumes:
       # You may store the database tables in a local folder..
       - ./schemas:/var/lib/postgresql/data

contrib/example_log_config.yaml

@@ -1,7 +1,7 @@
-# Example log_config file for synapse. To enable, point `log_config` to it in
+# Example log_config file for synapse. To enable, point `log_config` to it in 
 # `homeserver.yaml`, and restart synapse.
 #
-# This configuration will produce similar results to the defaults within
+# This configuration will produce similar results to the defaults within 
 # synapse, but can be edited to give more flexibility.
 
 version: 1
@@ -12,14 +12,13 @@ formatters:
 
 filters:
   context:
-    (): synapse.logging.context.LoggingContextFilter
+    (): synapse.util.logcontext.LoggingContextFilter
     request: ""
 
 handlers:
   # example output to console
   console:
     class: logging.StreamHandler
-    formatter: fmt
     filters: [context]
 
   # example output to file - to enable, edit 'root' config below.
@@ -30,12 +29,12 @@ handlers:
     maxBytes: 100000000
     backupCount: 3
     filters: [context]
-    encoding: utf8
+
 
 root:
     level: INFO
     handlers: [console] # to use file handler instead, switch to [file]
-
+    
 loggers:
     synapse:
         level: INFO

contrib/experiments/cursesio.py

@@ -19,13 +19,13 @@ from curses.ascii import isprint
 from twisted.internet import reactor
 
 
-class CursesStdIO:
+class CursesStdIO():
     def __init__(self, stdscr, callback=None):
         self.statusText = "Synapse test app -"
-        self.searchText = ""
+        self.searchText = ''
         self.stdscr = stdscr
 
-        self.logLine = ""
+        self.logLine = ''
 
         self.callback = callback
 
@@ -63,14 +63,16 @@ class CursesStdIO:
         self.redraw()
 
     def redraw(self):
-        """method for redisplaying lines based on internal list of lines"""
+        """ method for redisplaying lines
+            based on internal list of lines """
 
         self.stdscr.clear()
         self.paintStatus(self.statusText)
         i = 0
         index = len(self.lines) - 1
         while i < (self.rows - 3) and index >= 0:
-            self.stdscr.addstr(self.rows - 3 - i, 0, self.lines[index], curses.A_NORMAL)
+            self.stdscr.addstr(self.rows - 3 - i, 0, self.lines[index],
+                               curses.A_NORMAL)
             i = i + 1
             index = index - 1
 
@@ -83,13 +85,15 @@ class CursesStdIO:
             raise RuntimeError("TextTooLongError")
 
         self.stdscr.addstr(
-            self.rows - 2, 0, text + " " * (self.cols - len(text)), curses.A_STANDOUT
-        )
+            self.rows - 2, 0,
+            text + ' ' * (self.cols - len(text)),
+            curses.A_STANDOUT)
 
     def printLogLine(self, text):
         self.stdscr.addstr(
-            0, 0, text + " " * (self.cols - len(text)), curses.A_STANDOUT
-        )
+            0, 0,
+            text + ' ' * (self.cols - len(text)),
+            curses.A_STANDOUT)
 
     def doRead(self):
         """ Input is ready! """
@@ -101,7 +105,7 @@ class CursesStdIO:
 
         elif c == curses.KEY_ENTER or c == 10:
             text = self.searchText
-            self.searchText = ""
+            self.searchText = ''
 
             self.print_line(">> %s" % text)
 
@@ -118,13 +122,11 @@ class CursesStdIO:
                 return
             self.searchText = self.searchText + chr(c)
 
-        self.stdscr.addstr(
-            self.rows - 1,
-            0,
-            self.searchText + (" " * (self.cols - len(self.searchText) - 2)),
-        )
+        self.stdscr.addstr(self.rows - 1, 0,
+                           self.searchText + (' ' * (
+                           self.cols - len(self.searchText) - 2)))
 
-        self.paintStatus(self.statusText + " %d" % len(self.searchText))
+        self.paintStatus(self.statusText + ' %d' % len(self.searchText))
         self.stdscr.move(self.rows - 1, len(self.searchText))
         self.stdscr.refresh()
 
@@ -140,7 +142,8 @@ class CursesStdIO:
         curses.endwin()
 
 
-class Callback:
+class Callback(object):
+
     def __init__(self, stdio):
         self.stdio = stdio
 
@@ -149,7 +152,7 @@ class Callback:
 
 
 def main(stdscr):
-    screen = CursesStdIO(stdscr)  # create Screen object
+    screen = CursesStdIO(stdscr)   # create Screen object
 
     callback = Callback(screen)
 
@@ -161,5 +164,5 @@ def main(stdscr):
     screen.close()
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     curses.wrapper(main)

contrib/experiments/test_messaging.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -27,24 +28,29 @@ Currently assumes the local address is localhost:<port>
 """
 
 
+from synapse.federation import (
+    ReplicationHandler
+)
+
+from synapse.federation.units import Pdu
+
+from synapse.util import origin_from_ucid
+
+from synapse.app.homeserver import SynapseHomeServer
+
+#from synapse.util.logutils import log_function
+
+from twisted.internet import reactor, defer
+from twisted.python import log
+
 import argparse
-import curses.wrapper
 import json
 import logging
 import os
 import re
 
 import cursesio
-
-from twisted.internet import defer, reactor
-from twisted.python import log
-
-from synapse.app.homeserver import SynapseHomeServer
-from synapse.federation import ReplicationHandler
-from synapse.federation.units import Pdu
-from synapse.util import origin_from_ucid
-
-# from synapse.logging.utils import log_function
+import curses.wrapper
 
 
 logger = logging.getLogger("example")
@@ -54,8 +60,8 @@ def excpetion_errback(failure):
     logging.exception(failure)
 
 
-class InputOutput:
-    """This is responsible for basic I/O so that a user can interact with
+class InputOutput(object):
+    """ This is responsible for basic I/O so that a user can interact with
     the example app.
     """
 
@@ -67,40 +73,41 @@ class InputOutput:
         self.server = server
 
     def on_line(self, line):
-        """This is where we process commands."""
+        """ This is where we process commands.
+        """
 
         try:
-            m = re.match(r"^join (\S+)$", line)
+            m = re.match("^join (\S+)$", line)
             if m:
                 # The `sender` wants to join a room.
-                (room_name,) = m.groups()
+                room_name, = m.groups()
                 self.print_line("%s joining %s" % (self.user, room_name))
                 self.server.join_room(room_name, self.user, self.user)
-                # self.print_line("OK.")
+                #self.print_line("OK.")
                 return
 
-            m = re.match(r"^invite (\S+) (\S+)$", line)
+            m = re.match("^invite (\S+) (\S+)$", line)
             if m:
                 # `sender` wants to invite someone to a room
                 room_name, invitee = m.groups()
                 self.print_line("%s invited to %s" % (invitee, room_name))
                 self.server.invite_to_room(room_name, self.user, invitee)
-                # self.print_line("OK.")
+                #self.print_line("OK.")
                 return
 
-            m = re.match(r"^send (\S+) (.*)$", line)
+            m = re.match("^send (\S+) (.*)$", line)
             if m:
                 # `sender` wants to message a room
                 room_name, body = m.groups()
                 self.print_line("%s send to %s" % (self.user, room_name))
                 self.server.send_message(room_name, self.user, body)
-                # self.print_line("OK.")
+                #self.print_line("OK.")
                 return
 
-            m = re.match(r"^backfill (\S+)$", line)
+            m = re.match("^backfill (\S+)$", line)
             if m:
                 # we want to backfill a room
-                (room_name,) = m.groups()
+                room_name, = m.groups()
                 self.print_line("backfill %s" % room_name)
                 self.server.backfill(room_name)
                 return
@@ -118,6 +125,7 @@ class InputOutput:
 
 
 class IOLoggerHandler(logging.Handler):
+
     def __init__(self, io):
         logging.Handler.__init__(self)
         self.io = io
@@ -130,11 +138,10 @@ class IOLoggerHandler(logging.Handler):
         self.io.print_log(msg)
 
 
-class Room:
-    """Used to store (in memory) the current membership state of a room, and
+class Room(object):
+    """ Used to store (in memory) the current membership state of a room, and
     which home servers we should send PDUs associated with the room to.
     """
-
     def __init__(self, room_name):
         self.room_name = room_name
         self.invited = set()
@@ -146,7 +153,8 @@ class Room:
         self.have_got_metadata = False
 
     def add_participant(self, participant):
-        """Someone has joined the room"""
+        """ Someone has joined the room
+        """
         self.participants.add(participant)
         self.invited.discard(participant)
 
@@ -157,16 +165,16 @@ class Room:
             self.oldest_server = server
 
     def add_invited(self, invitee):
-        """Someone has been invited to the room"""
+        """ Someone has been invited to the room
+        """
         self.invited.add(invitee)
         self.servers.add(origin_from_ucid(invitee))
 
 
 class HomeServer(ReplicationHandler):
-    """A very basic home server implentation that allows people to join a
+    """ A very basic home server implentation that allows people to join a
     room and then invite other people.
     """
-
     def __init__(self, server_name, replication_layer, output):
         self.server_name = server_name
         self.replication_layer = replication_layer
@@ -177,7 +185,8 @@ class HomeServer(ReplicationHandler):
         self.output = output
 
     def on_receive_pdu(self, pdu):
-        """We just received a PDU"""
+        """ We just received a PDU
+        """
         pdu_type = pdu.pdu_type
 
         if pdu_type == "sy.room.message":
@@ -188,30 +197,46 @@ class HomeServer(ReplicationHandler):
             elif pdu.content["membership"] == "invite":
                 self._on_invite(pdu.origin, pdu.context, pdu.state_key)
         else:
-            self.output.print_line(
-                "#%s (unrec) %s = %s"
-                % (pdu.context, pdu.pdu_type, json.dumps(pdu.content))
+            self.output.print_line("#%s (unrec) %s = %s" %
+                (pdu.context, pdu.pdu_type, json.dumps(pdu.content))
             )
 
+    #def on_state_change(self, pdu):
+        ##self.output.print_line("#%s (state) %s *** %s" %
+                ##(pdu.context, pdu.state_key, pdu.pdu_type)
+            ##)
+
+        #if "joinee" in pdu.content:
+            #self._on_join(pdu.context, pdu.content["joinee"])
+        #elif "invitee" in pdu.content:
+            #self._on_invite(pdu.origin, pdu.context, pdu.content["invitee"])
+
     def _on_message(self, pdu):
-        """We received a message"""
-        self.output.print_line(
-            "#%s %s %s" % (pdu.context, pdu.content["sender"], pdu.content["body"])
-        )
+        """ We received a message
+        """
+        self.output.print_line("#%s %s %s" %
+                (pdu.context, pdu.content["sender"], pdu.content["body"])
+            )
 
     def _on_join(self, context, joinee):
-        """Someone has joined a room, either a remote user or a local user"""
+        """ Someone has joined a room, either a remote user or a local user
+        """
         room = self._get_or_create_room(context)
         room.add_participant(joinee)
 
-        self.output.print_line("#%s %s %s" % (context, joinee, "*** JOINED"))
+        self.output.print_line("#%s %s %s" %
+                (context, joinee, "*** JOINED")
+            )
 
     def _on_invite(self, origin, context, invitee):
-        """Someone has been invited"""
+        """ Someone has been invited
+        """
         room = self._get_or_create_room(context)
         room.add_invited(invitee)
 
-        self.output.print_line("#%s %s %s" % (context, invitee, "*** INVITED"))
+        self.output.print_line("#%s %s %s" %
+                (context, invitee, "*** INVITED")
+            )
 
         if not room.have_got_metadata and origin is not self.server_name:
             logger.debug("Get room state")
@@ -220,7 +245,8 @@ class HomeServer(ReplicationHandler):
 
     @defer.inlineCallbacks
     def send_message(self, room_name, sender, body):
-        """Send a message to a room!"""
+        """ Send a message to a room!
+        """
         destinations = yield self.get_servers_for_context(room_name)
 
         try:
@@ -238,28 +264,30 @@ class HomeServer(ReplicationHandler):
 
     @defer.inlineCallbacks
     def join_room(self, room_name, sender, joinee):
-        """Join a room!"""
+        """ Join a room!
+        """
         self._on_join(room_name, joinee)
 
         destinations = yield self.get_servers_for_context(room_name)
 
         try:
             pdu = Pdu.create_new(
-                context=room_name,
-                pdu_type="sy.room.member",
-                is_state=True,
-                state_key=joinee,
-                content={"membership": "join"},
-                origin=self.server_name,
-                destinations=destinations,
-            )
+                    context=room_name,
+                    pdu_type="sy.room.member",
+                    is_state=True,
+                    state_key=joinee,
+                    content={"membership": "join"},
+                    origin=self.server_name,
+                    destinations=destinations,
+                )
             yield self.replication_layer.send_pdu(pdu)
         except Exception as e:
             logger.exception(e)
 
     @defer.inlineCallbacks
     def invite_to_room(self, room_name, sender, invitee):
-        """Invite someone to a room!"""
+        """ Invite someone to a room!
+        """
         self._on_invite(self.server_name, room_name, invitee)
 
         destinations = yield self.get_servers_for_context(room_name)
@@ -290,33 +318,32 @@ class HomeServer(ReplicationHandler):
         return self.replication_layer.backfill(dest, room_name, limit)
 
     def _get_room_remote_servers(self, room_name):
-        return list(self.joined_rooms.setdefault(room_name).servers)
+        return [i for i in self.joined_rooms.setdefault(room_name,).servers]
 
     def _get_or_create_room(self, room_name):
         return self.joined_rooms.setdefault(room_name, Room(room_name))
 
     def get_servers_for_context(self, context):
         return defer.succeed(
-            self.joined_rooms.setdefault(context, Room(context)).servers
-        )
+                self.joined_rooms.setdefault(context, Room(context)).servers
+            )
 
 
 def main(stdscr):
     parser = argparse.ArgumentParser()
-    parser.add_argument("user", type=str)
-    parser.add_argument("-v", "--verbose", action="count")
+    parser.add_argument('user', type=str)
+    parser.add_argument('-v', '--verbose', action='count')
     args = parser.parse_args()
 
     user = args.user
     server_name = origin_from_ucid(user)
 
-    # Set up logging
+    ## Set up logging ##
 
     root_logger = logging.getLogger()
 
-    formatter = logging.Formatter(
-        "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s"
-    )
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(lineno)d - '
+            '%(levelname)s - %(message)s')
     if not os.path.exists("logs"):
         os.makedirs("logs")
     fh = logging.FileHandler("logs/%s" % user)
@@ -330,7 +357,7 @@ def main(stdscr):
     observer = log.PythonLoggingObserver()
     observer.start()
 
-    # Set up synapse server
+    ## Set up synapse server
 
     curses_stdio = cursesio.CursesStdIO(stdscr)
     input_output = InputOutput(curses_stdio, user)
@@ -344,16 +371,16 @@ def main(stdscr):
 
     input_output.set_home_server(hs)
 
-    # Add input_output logger
+    ## Add input_output logger
     io_logger = IOLoggerHandler(input_output)
     io_logger.setFormatter(formatter)
     root_logger.addHandler(io_logger)
 
-    # Start!
+    ## Start! ##
 
     try:
         port = int(server_name.split(":")[1])
-    except Exception:
+    except:
         port = 12345
 
     app_hs.get_http_server().start_listening(port)

contrib/grafana/README.md

@@ -1,6 +1,6 @@
 # Using the Synapse Grafana dashboard
 
 0. Set up Prometheus and Grafana. Out of scope for this readme. Useful documentation about using Grafana with Prometheus: http://docs.grafana.org/features/datasources/prometheus/
-1. Have your Prometheus scrape your Synapse. https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md
+1. Have your Prometheus scrape your Synapse. https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.rst
 2. Import dashboard into Grafana. Download `synapse.json`. Import it to Grafana and select the correct Prometheus datasource. http://docs.grafana.org/reference/export_import/
-3. Set up required recording rules. https://github.com/matrix-org/synapse/tree/master/contrib/prometheus
+3. Set up additional recording rules

contrib/graph/graph.py

@@ -1,11 +1,3 @@
-import argparse
-import cgi
-import datetime
-import json
-
-import pydot
-import urllib2
-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,6 +13,15 @@ import urllib2
 # limitations under the License.
 
 
+import sqlite3
+import pydot
+import cgi
+import json
+import datetime
+import argparse
+import urllib2
+
+
 def make_name(pdu_id, origin):
     return "%s@%s" % (pdu_id, origin)
 
@@ -30,7 +31,7 @@ def make_graph(pdus, room, filename_prefix):
     node_map = {}
 
     origins = set()
-    colors = {"red", "green", "blue", "yellow", "purple"}
+    colors = set(("red", "green", "blue", "yellow", "purple"))
 
     for pdu in pdus:
         origins.add(pdu.get("origin"))
@@ -46,8 +47,8 @@ def make_graph(pdus, room, filename_prefix):
         try:
             c = colors.pop()
             color_map[o] = c
-        except Exception:
-            print("Run out of colours!")
+        except:
+            print "Run out of colours!"
             color_map[o] = "black"
 
     graph = pydot.Dot(graph_name="Test")
@@ -56,9 +57,9 @@ def make_graph(pdus, room, filename_prefix):
         name = make_name(pdu.get("pdu_id"), pdu.get("origin"))
         pdu_map[name] = pdu
 
-        t = datetime.datetime.fromtimestamp(float(pdu["ts"]) / 1000).strftime(
-            "%Y-%m-%d %H:%M:%S,%f"
-        )
+        t = datetime.datetime.fromtimestamp(
+            float(pdu["ts"]) / 1000
+        ).strftime('%Y-%m-%d %H:%M:%S,%f')
 
         label = (
             "<"
@@ -78,7 +79,11 @@ def make_graph(pdus, room, filename_prefix):
             "depth": pdu.get("depth"),
         }
 
-        node = pydot.Node(name=name, label=label, color=color_map[pdu.get("origin")])
+        node = pydot.Node(
+            name=name,
+            label=label,
+            color=color_map[pdu.get("origin")]
+        )
         node_map[name] = node
         graph.add_node(node)
 
@@ -88,7 +93,7 @@ def make_graph(pdus, room, filename_prefix):
             end_name = make_name(i, o)
 
             if end_name not in node_map:
-                print("%s not in nodes" % end_name)
+                print "%s not in nodes" % end_name
                 continue
 
             edge = pydot.Edge(node_map[start_name], node_map[end_name])
@@ -102,13 +107,14 @@ def make_graph(pdus, room, filename_prefix):
 
             if prev_state_name in node_map:
                 state_edge = pydot.Edge(
-                    node_map[start_name], node_map[prev_state_name], style="dotted"
+                    node_map[start_name], node_map[prev_state_name],
+                    style='dotted'
                 )
                 graph.add_edge(state_edge)
 
-    graph.write("%s.dot" % filename_prefix, format="raw", prog="dot")
-    #    graph.write_png("%s.png" % filename_prefix, prog='dot')
-    graph.write_svg("%s.svg" % filename_prefix, prog="dot")
+    graph.write('%s.dot' % filename_prefix, format='raw', prog='dot')
+#    graph.write_png("%s.png" % filename_prefix, prog='dot')
+    graph.write_svg("%s.svg" % filename_prefix, prog='dot')
 
 
 def get_pdus(host, room):
@@ -124,14 +130,15 @@ def get_pdus(host, room):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Generate a PDU graph for a given room by talking "
-        "to the given homeserver to get the list of PDUs. \n"
-        "Requires pydot."
+                    "to the given homeserver to get the list of PDUs. \n"
+                    "Requires pydot."
     )
     parser.add_argument(
-        "-p", "--prefix", dest="prefix", help="String to prefix output files with"
+        "-p", "--prefix", dest="prefix",
+        help="String to prefix output files with"
     )
-    parser.add_argument("host")
-    parser.add_argument("room")
+    parser.add_argument('host')
+    parser.add_argument('room')
 
     args = parser.parse_args()
 

contrib/graph/graph2.py

@@ -13,13 +13,12 @@
 # limitations under the License.
 
 
-import argparse
-import cgi
-import datetime
-import json
 import sqlite3
-
 import pydot
+import cgi
+import json
+import datetime
+import argparse
 
 from synapse.events import FrozenEvent
 from synapse.util.frozenutils import unfreeze
@@ -37,7 +36,10 @@ def make_graph(db_name, room_id, file_prefix, limit):
     args = [room_id]
 
     if limit:
-        sql += " ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ?"
+        sql += (
+            " ORDER BY topological_ordering DESC, stream_ordering DESC "
+            "LIMIT ?"
+        )
 
         args.append(limit)
 
@@ -54,8 +56,9 @@ def make_graph(db_name, room_id, file_prefix, limit):
 
     for event in events:
         c = conn.execute(
-            "SELECT state_group FROM event_to_state_groups WHERE event_id = ?",
-            (event.event_id,),
+            "SELECT state_group FROM event_to_state_groups "
+            "WHERE event_id = ?",
+            (event.event_id,)
         )
 
         res = c.fetchone()
@@ -66,7 +69,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
 
         t = datetime.datetime.fromtimestamp(
             float(event.origin_server_ts) / 1000
-        ).strftime("%Y-%m-%d %H:%M:%S,%f")
+        ).strftime('%Y-%m-%d %H:%M:%S,%f')
 
         content = json.dumps(unfreeze(event.get_dict()["content"]))
 
@@ -90,7 +93,10 @@ def make_graph(db_name, room_id, file_prefix, limit):
             "state_group": state_group,
         }
 
-        node = pydot.Node(name=event.event_id, label=label)
+        node = pydot.Node(
+            name=event.event_id,
+            label=label,
+        )
 
         node_map[event.event_id] = node
         graph.add_node(node)
@@ -99,8 +105,11 @@ def make_graph(db_name, room_id, file_prefix, limit):
         for prev_id, _ in event.prev_events:
             try:
                 end_node = node_map[prev_id]
-            except Exception:
-                end_node = pydot.Node(name=prev_id, label="<<b>%s</b>>" % (prev_id,))
+            except:
+                end_node = pydot.Node(
+                    name=prev_id,
+                    label="<<b>%s</b>>" % (prev_id,),
+                )
 
                 node_map[prev_id] = end_node
                 graph.add_node(end_node)
@@ -112,33 +121,36 @@ def make_graph(db_name, room_id, file_prefix, limit):
         if len(event_ids) <= 1:
             continue
 
-        cluster = pydot.Cluster(str(group), label="<State Group: %s>" % (str(group),))
+        cluster = pydot.Cluster(
+            str(group),
+            label="<State Group: %s>" % (str(group),)
+        )
 
         for event_id in event_ids:
             cluster.add_node(node_map[event_id])
 
         graph.add_subgraph(cluster)
 
-    graph.write("%s.dot" % file_prefix, format="raw", prog="dot")
-    graph.write_svg("%s.svg" % file_prefix, prog="dot")
-
+    graph.write('%s.dot' % file_prefix, format='raw', prog='dot')
+    graph.write_svg("%s.svg" % file_prefix, prog='dot')
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Generate a PDU graph for a given room by talking "
-        "to the given homeserver to get the list of PDUs. \n"
-        "Requires pydot."
+                    "to the given homeserver to get the list of PDUs. \n"
+                    "Requires pydot."
     )
     parser.add_argument(
-        "-p",
-        "--prefix",
-        dest="prefix",
+        "-p", "--prefix", dest="prefix",
         help="String to prefix output files with",
-        default="graph_output",
+        default="graph_output"
     )
-    parser.add_argument("-l", "--limit", help="Only retrieve the last N events.")
-    parser.add_argument("db")
-    parser.add_argument("room")
+    parser.add_argument(
+        "-l", "--limit",
+        help="Only retrieve the last N events.",
+    )
+    parser.add_argument('db')
+    parser.add_argument('room')
 
     args = parser.parse_args()
 

contrib/graph/graph3.py

@@ -1,13 +1,3 @@
-import argparse
-import cgi
-import datetime
-
-import pydot
-import simplejson as json
-
-from synapse.events import FrozenEvent
-from synapse.util.frozenutils import unfreeze
-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,23 +13,35 @@ from synapse.util.frozenutils import unfreeze
 # limitations under the License.
 
 
+import pydot
+import cgi
+import simplejson as json
+import datetime
+import argparse
+
+from synapse.events import FrozenEvent
+from synapse.util.frozenutils import unfreeze
+
+from six import string_types
+
+
 def make_graph(file_name, room_id, file_prefix, limit):
-    print("Reading lines")
+    print "Reading lines"
     with open(file_name) as f:
         lines = f.readlines()
 
-    print("Read lines")
+    print "Read lines"
 
     events = [FrozenEvent(json.loads(line)) for line in lines]
 
-    print("Loaded events.")
+    print "Loaded events."
 
     events.sort(key=lambda e: e.depth)
 
-    print("Sorted events")
+    print "Sorted events"
 
     if limit:
-        events = events[-int(limit) :]
+        events = events[-int(limit):]
 
     node_map = {}
 
@@ -48,32 +50,31 @@ def make_graph(file_name, room_id, file_prefix, limit):
     for event in events:
         t = datetime.datetime.fromtimestamp(
             float(event.origin_server_ts) / 1000
-        ).strftime("%Y-%m-%d %H:%M:%S,%f")
+        ).strftime('%Y-%m-%d %H:%M:%S,%f')
 
         content = json.dumps(unfreeze(event.get_dict()["content"]), indent=4)
         content = content.replace("\n", "<br/>\n")
 
-        print(content)
+        print content
         content = []
         for key, value in unfreeze(event.get_dict()["content"]).items():
             if value is None:
                 value = "<null>"
-            elif isinstance(value, str):
+            elif isinstance(value, string_types):
                 pass
             else:
                 value = json.dumps(value)
 
             content.append(
-                "<b>%s</b>: %s,"
-                % (
-                    cgi.escape(key, quote=True).encode("ascii", "xmlcharrefreplace"),
-                    cgi.escape(value, quote=True).encode("ascii", "xmlcharrefreplace"),
+                "<b>%s</b>: %s," % (
+                    cgi.escape(key, quote=True).encode("ascii", 'xmlcharrefreplace'),
+                    cgi.escape(value, quote=True).encode("ascii", 'xmlcharrefreplace'),
                 )
             )
 
         content = "<br/>\n".join(content)
 
-        print(content)
+        print content
 
         label = (
             "<"
@@ -93,19 +94,25 @@ def make_graph(file_name, room_id, file_prefix, limit):
             "depth": event.depth,
         }
 
-        node = pydot.Node(name=event.event_id, label=label)
+        node = pydot.Node(
+            name=event.event_id,
+            label=label,
+        )
 
         node_map[event.event_id] = node
         graph.add_node(node)
 
-    print("Created Nodes")
+    print "Created Nodes"
 
     for event in events:
         for prev_id, _ in event.prev_events:
             try:
                 end_node = node_map[prev_id]
-            except Exception:
-                end_node = pydot.Node(name=prev_id, label="<<b>%s</b>>" % (prev_id,))
+            except:
+                end_node = pydot.Node(
+                    name=prev_id,
+                    label="<<b>%s</b>>" % (prev_id,),
+                )
 
                 node_map[prev_id] = end_node
                 graph.add_node(end_node)
@@ -113,33 +120,33 @@ def make_graph(file_name, room_id, file_prefix, limit):
             edge = pydot.Edge(node_map[event.event_id], end_node)
             graph.add_edge(edge)
 
-    print("Created edges")
+    print "Created edges"
 
-    graph.write("%s.dot" % file_prefix, format="raw", prog="dot")
+    graph.write('%s.dot' % file_prefix, format='raw', prog='dot')
 
-    print("Created Dot")
+    print "Created Dot"
 
-    graph.write_svg("%s.svg" % file_prefix, prog="dot")
-
-    print("Created svg")
+    graph.write_svg("%s.svg" % file_prefix, prog='dot')
 
+    print "Created svg"
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Generate a PDU graph for a given room by reading "
-        "from a file with line deliminated events. \n"
-        "Requires pydot."
+                    "from a file with line deliminated events. \n"
+                    "Requires pydot."
     )
     parser.add_argument(
-        "-p",
-        "--prefix",
-        dest="prefix",
+        "-p", "--prefix", dest="prefix",
         help="String to prefix output files with",
-        default="graph_output",
+        default="graph_output"
     )
-    parser.add_argument("-l", "--limit", help="Only retrieve the last N events.")
-    parser.add_argument("event_file")
-    parser.add_argument("room")
+    parser.add_argument(
+        "-l", "--limit",
+        help="Only retrieve the last N events.",
+    )
+    parser.add_argument('event_file')
+    parser.add_argument('room')
 
     args = parser.parse_args()
 

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -8,35 +8,35 @@ we set the remote SDP at which point the stream ends. Our video never gets to
 the bridge.
 
 Requires:
-npm install jquery jsdom
+npm install jquery jsdom 
 """
-import json
-import subprocess
-import time
 
 import gevent
 import grequests
 from BeautifulSoup import BeautifulSoup
+import json
+import urllib
+import subprocess
+import time
 
-ACCESS_TOKEN = ""
+#ACCESS_TOKEN="" #
 
-MATRIXBASE = "https://matrix.org/_matrix/client/api/v1/"
-MYUSERNAME = "@davetest:matrix.org"
+MATRIXBASE = 'https://matrix.org/_matrix/client/api/v1/'
+MYUSERNAME = '@davetest:matrix.org'
 
-HTTPBIND = "https://meet.jit.si/http-bind"
-# HTTPBIND = 'https://jitsi.vuc.me/http-bind'
-# ROOMNAME = "matrix"
+HTTPBIND = 'https://meet.jit.si/http-bind'
+#HTTPBIND = 'https://jitsi.vuc.me/http-bind'
+#ROOMNAME = "matrix"
 ROOMNAME = "pibble"
 
-HOST = "guest.jit.si"
-# HOST="jitsi.vuc.me"
+HOST="guest.jit.si"
+#HOST="jitsi.vuc.me"
 
-TURNSERVER = "turn.guest.jit.si"
-# TURNSERVER="turn.jitsi.vuc.me"
-
-ROOMDOMAIN = "meet.jit.si"
-# ROOMDOMAIN="conference.jitsi.vuc.me"
+TURNSERVER="turn.guest.jit.si"
+#TURNSERVER="turn.jitsi.vuc.me"
 
+ROOMDOMAIN="meet.jit.si"
+#ROOMDOMAIN="conference.jitsi.vuc.me"
 
 class TrivialMatrixClient:
     def __init__(self, access_token):
@@ -45,50 +45,38 @@ class TrivialMatrixClient:
 
     def getEvent(self):
         while True:
-            url = (
-                MATRIXBASE
-                + "events?access_token="
-                + self.access_token
-                + "&timeout=60000"
-            )
+            url = MATRIXBASE+'events?access_token='+self.access_token+"&timeout=60000"
             if self.token:
-                url += "&from=" + self.token
+                url += "&from="+self.token
             req = grequests.get(url)
             resps = grequests.map([req])
             obj = json.loads(resps[0].content)
-            print("incoming from matrix", obj)
-            if "end" not in obj:
+            print "incoming from matrix",obj
+            if 'end' not in obj:
                 continue
-            self.token = obj["end"]
-            if len(obj["chunk"]):
-                return obj["chunk"][0]
+            self.token = obj['end']
+            if len(obj['chunk']):
+                return obj['chunk'][0]
 
     def joinRoom(self, roomId):
-        url = MATRIXBASE + "rooms/" + roomId + "/join?access_token=" + self.access_token
-        print(url)
-        headers = {"Content-Type": "application/json"}
-        req = grequests.post(url, headers=headers, data="{}")
+        url = MATRIXBASE+'rooms/'+roomId+'/join?access_token='+self.access_token
+        print url
+        headers={ 'Content-Type': 'application/json' }
+        req = grequests.post(url, headers=headers, data='{}')
         resps = grequests.map([req])
         obj = json.loads(resps[0].content)
-        print("response: ", obj)
+        print "response: ",obj
 
     def sendEvent(self, roomId, evType, event):
-        url = (
-            MATRIXBASE
-            + "rooms/"
-            + roomId
-            + "/send/"
-            + evType
-            + "?access_token="
-            + self.access_token
-        )
-        print(url)
-        print(json.dumps(event))
-        headers = {"Content-Type": "application/json"}
+        url = MATRIXBASE+'rooms/'+roomId+'/send/'+evType+'?access_token='+self.access_token
+        print url
+        print json.dumps(event)
+        headers={ 'Content-Type': 'application/json' }
         req = grequests.post(url, headers=headers, data=json.dumps(event))
         resps = grequests.map([req])
         obj = json.loads(resps[0].content)
-        print("response: ", obj)
+        print "response: ",obj
+
 
 
 xmppClients = {}
@@ -97,40 +85,39 @@ xmppClients = {}
 def matrixLoop():
     while True:
         ev = matrixCli.getEvent()
-        print(ev)
-        if ev["type"] == "m.room.member":
-            print("membership event")
-            if ev["membership"] == "invite" and ev["state_key"] == MYUSERNAME:
-                roomId = ev["room_id"]
-                print("joining room %s" % (roomId))
+        print ev
+        if ev['type'] == 'm.room.member':
+            print 'membership event'
+            if ev['membership'] == 'invite' and ev['state_key'] == MYUSERNAME:
+                roomId = ev['room_id']
+                print "joining room %s" % (roomId)
                 matrixCli.joinRoom(roomId)
-        elif ev["type"] == "m.room.message":
-            if ev["room_id"] in xmppClients:
-                print("already have a bridge for that user, ignoring")
+        elif ev['type'] == 'm.room.message':
+            if ev['room_id'] in xmppClients:
+                print "already have a bridge for that user, ignoring"
                 continue
-            print("got message, connecting")
-            xmppClients[ev["room_id"]] = TrivialXmppClient(ev["room_id"], ev["user_id"])
-            gevent.spawn(xmppClients[ev["room_id"]].xmppLoop)
-        elif ev["type"] == "m.call.invite":
-            print("Incoming call")
-            # sdp = ev['content']['offer']['sdp']
-            # print "sdp: %s" % (sdp)
-            # xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
-            # gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
-        elif ev["type"] == "m.call.answer":
-            print("Call answered")
-            sdp = ev["content"]["answer"]["sdp"]
-            if ev["room_id"] not in xmppClients:
-                print("We didn't have a call for that room")
+            print "got message, connecting"
+            xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
+            gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
+        elif ev['type'] == 'm.call.invite':
+            print "Incoming call"
+            #sdp = ev['content']['offer']['sdp']
+            #print "sdp: %s" % (sdp)
+            #xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
+            #gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
+        elif ev['type'] == 'm.call.answer':
+            print "Call answered"
+            sdp = ev['content']['answer']['sdp']
+            if ev['room_id'] not in xmppClients:
+                print "We didn't have a call for that room"
                 continue
             # should probably check call ID too
-            xmppCli = xmppClients[ev["room_id"]]
+            xmppCli = xmppClients[ev['room_id']]
             xmppCli.sendAnswer(sdp)
-        elif ev["type"] == "m.call.hangup":
-            if ev["room_id"] in xmppClients:
-                xmppClients[ev["room_id"]].stop()
-                del xmppClients[ev["room_id"]]
-
+        elif ev['type'] == 'm.call.hangup':
+            if ev['room_id'] in xmppClients:
+                xmppClients[ev['room_id']].stop()
+                del xmppClients[ev['room_id']]
 
 class TrivialXmppClient:
     def __init__(self, matrixRoom, userId):
@@ -144,152 +131,130 @@ class TrivialXmppClient:
 
     def nextRid(self):
         self.rid += 1
-        return "%d" % (self.rid)
+        return '%d' % (self.rid)
 
     def sendIq(self, xml):
-        fullXml = (
-            "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s'>%s</body>"
-            % (self.nextRid(), self.sid, xml)
-        )
-        # print "\t>>>%s" % (fullXml)
+        fullXml = "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s'>%s</body>" % (self.nextRid(), self.sid, xml)
+        #print "\t>>>%s" % (fullXml)
         return self.xmppPoke(fullXml)
 
     def xmppPoke(self, xml):
-        headers = {"Content-Type": "application/xml"}
+        headers = {'Content-Type': 'application/xml'}
         req = grequests.post(HTTPBIND, verify=False, headers=headers, data=xml)
         resps = grequests.map([req])
         obj = BeautifulSoup(resps[0].content)
         return obj
 
     def sendAnswer(self, answer):
-        print("sdp from matrix client", answer)
-        p = subprocess.Popen(
-            ["node", "unjingle/unjingle.js", "--sdp"],
-            stdin=subprocess.PIPE,
-            stdout=subprocess.PIPE,
-        )
+        print "sdp from matrix client",answer
+        p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--sdp'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
         jingle, out_err = p.communicate(answer)
         jingle = jingle % {
-            "tojid": self.callfrom,
-            "action": "session-accept",
-            "initiator": self.callfrom,
-            "responder": self.jid,
-            "sid": self.callsid,
+            'tojid': self.callfrom,
+            'action': 'session-accept',
+            'initiator': self.callfrom,
+            'responder': self.jid,
+            'sid': self.callsid
         }
-        print("answer jingle from sdp", jingle)
+        print "answer jingle from sdp",jingle
         res = self.sendIq(jingle)
-        print("reply from answer: ", res)
+        print "reply from answer: ",res
 
         self.ssrcs = {}
         jingleSoup = BeautifulSoup(jingle)
-        for cont in jingleSoup.iq.jingle.findAll("content"):
+        for cont in jingleSoup.iq.jingle.findAll('content'):
             if cont.description:
-                self.ssrcs[cont["name"]] = cont.description["ssrc"]
-        print("my ssrcs:", self.ssrcs)
+                self.ssrcs[cont['name']] = cont.description['ssrc']
+        print "my ssrcs:",self.ssrcs
 
-        gevent.joinall([gevent.spawn(self.advertiseSsrcs)])
+        gevent.joinall([
+                gevent.spawn(self.advertiseSsrcs)
+        ])
 
     def advertiseSsrcs(self):
-        time.sleep(7)
-        print("SSRC spammer started")
+                time.sleep(7)
+        print "SSRC spammer started"
         while self.running:
-            ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % {
-                "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
-                "nick": self.userId,
-                "assrc": self.ssrcs["audio"],
-                "vssrc": self.ssrcs["video"],
-            }
+            ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % { 'tojid': "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid), 'nick': self.userId, 'assrc': self.ssrcs['audio'], 'vssrc': self.ssrcs['video'] }
             res = self.sendIq(ssrcMsg)
-            print("reply from ssrc announce: ", res)
+            print "reply from ssrc announce: ",res
             time.sleep(10)
 
+
+
     def xmppLoop(self):
         self.matrixCallId = time.time()
-        res = self.xmppPoke(
-            "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' to='%s' xml:lang='en' wait='60' hold='1' content='text/xml; charset=utf-8' ver='1.6' xmpp:version='1.0' xmlns:xmpp='urn:xmpp:xbosh'/>"
-            % (self.nextRid(), HOST)
-        )
+        res = self.xmppPoke("<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' to='%s' xml:lang='en' wait='60' hold='1' content='text/xml; charset=utf-8' ver='1.6' xmpp:version='1.0' xmlns:xmpp='urn:xmpp:xbosh'/>" % (self.nextRid(), HOST))
 
-        print(res)
-        self.sid = res.body["sid"]
-        print("sid %s" % (self.sid))
+        print res
+        self.sid = res.body['sid']
+        print "sid %s" % (self.sid)
 
-        res = self.sendIq(
-            "<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='ANONYMOUS'/>"
-        )
+        res = self.sendIq("<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='ANONYMOUS'/>")
 
-        res = self.xmppPoke(
-            "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s' to='%s' xml:lang='en' xmpp:restart='true' xmlns:xmpp='urn:xmpp:xbosh'/>"
-            % (self.nextRid(), self.sid, HOST)
-        )
+        res = self.xmppPoke("<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s' to='%s' xml:lang='en' xmpp:restart='true' xmlns:xmpp='urn:xmpp:xbosh'/>" % (self.nextRid(), self.sid, HOST))
 
-        res = self.sendIq(
-            "<iq type='set' id='_bind_auth_2' xmlns='jabber:client'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'/></iq>"
-        )
-        print(res)
+        res = self.sendIq("<iq type='set' id='_bind_auth_2' xmlns='jabber:client'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'/></iq>")
+        print res
 
         self.jid = res.body.iq.bind.jid.string
-        print("jid: %s" % (self.jid))
-        self.shortJid = self.jid.split("-")[0]
+        print "jid: %s" % (self.jid)
+        self.shortJid = self.jid.split('-')[0]
 
-        res = self.sendIq(
-            "<iq type='set' id='_session_auth_2' xmlns='jabber:client'><session xmlns='urn:ietf:params:xml:ns:xmpp-session'/></iq>"
-        )
+        res = self.sendIq("<iq type='set' id='_session_auth_2' xmlns='jabber:client'><session xmlns='urn:ietf:params:xml:ns:xmpp-session'/></iq>")
 
-        # randomthing = res.body.iq['to']
-        # whatsitpart = randomthing.split('-')[0]
+        #randomthing = res.body.iq['to']
+        #whatsitpart = randomthing.split('-')[0]
 
-        # print "other random bind thing: %s" % (randomthing)
+        #print "other random bind thing: %s" % (randomthing)
 
         # advertise preence to the jitsi room, with our nick
-        res = self.sendIq(
-            "<iq type='get' to='%s' xmlns='jabber:client' id='1:sendIQ'><services xmlns='urn:xmpp:extdisco:1'><service host='%s'/></services></iq><presence to='%s@%s/d98f6c40' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%s</nick></presence>"
-            % (HOST, TURNSERVER, ROOMNAME, ROOMDOMAIN, self.userId)
-        )
-        self.muc = {"users": []}
-        for p in res.body.findAll("presence"):
+        res = self.sendIq("<iq type='get' to='%s' xmlns='jabber:client' id='1:sendIQ'><services xmlns='urn:xmpp:extdisco:1'><service host='%s'/></services></iq><presence to='%s@%s/d98f6c40' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%s</nick></presence>" % (HOST, TURNSERVER, ROOMNAME, ROOMDOMAIN, self.userId))
+        self.muc = {'users': []}
+        for p in res.body.findAll('presence'):
             u = {}
-            u["shortJid"] = p["from"].split("/")[1]
+            u['shortJid'] = p['from'].split('/')[1]
             if p.c and p.c.nick:
-                u["nick"] = p.c.nick.string
-            self.muc["users"].append(u)
-        print("muc: ", self.muc)
+                u['nick'] = p.c.nick.string
+            self.muc['users'].append(u)
+        print "muc: ",self.muc
 
         # wait for stuff
         while True:
-            print("waiting...")
+            print "waiting..."
             res = self.sendIq("")
-            print("got from stream: ", res)
+            print "got from stream: ",res
             if res.body.iq:
-                jingles = res.body.iq.findAll("jingle")
+                jingles = res.body.iq.findAll('jingle')
                 if len(jingles):
-                    self.callfrom = res.body.iq["from"]
+                    self.callfrom = res.body.iq['from']
                     self.handleInvite(jingles[0])
-            elif "type" in res.body and res.body["type"] == "terminate":
+            elif 'type' in res.body and res.body['type'] == 'terminate':
                 self.running = False
                 del xmppClients[self.matrixRoom]
-                return
+                        return
 
     def handleInvite(self, jingle):
-        self.initiator = jingle["initiator"]
-        self.callsid = jingle["sid"]
-        p = subprocess.Popen(
-            ["node", "unjingle/unjingle.js", "--jingle"],
-            stdin=subprocess.PIPE,
-            stdout=subprocess.PIPE,
-        )
-        print("raw jingle invite", str(jingle))
+        self.initiator = jingle['initiator']
+        self.callsid = jingle['sid']
+        p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--jingle'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+        print "raw jingle invite",str(jingle)
         sdp, out_err = p.communicate(str(jingle))
-        print("transformed remote offer sdp", sdp)
+        print "transformed remote offer sdp",sdp
         inviteEvent = {
-            "offer": {"type": "offer", "sdp": sdp},
-            "call_id": self.matrixCallId,
-            "version": 0,
-            "lifetime": 30000,
+            'offer': {
+                'type': 'offer',
+                'sdp': sdp
+            },
+            'call_id': self.matrixCallId,
+            'version': 0,
+            'lifetime': 30000
         }
-        matrixCli.sendEvent(self.matrixRoom, "m.call.invite", inviteEvent)
+        matrixCli.sendEvent(self.matrixRoom, 'm.call.invite', inviteEvent)
 
+matrixCli = TrivialMatrixClient(ACCESS_TOKEN)
 
-matrixCli = TrivialMatrixClient(ACCESS_TOKEN)  # Undefined name
+gevent.joinall([
+    gevent.spawn(matrixLoop)
+])
 
-gevent.joinall([gevent.spawn(matrixLoop)])

contrib/prometheus/README

@@ -6,10 +6,8 @@ To use it, first install prometheus by following the instructions at
   http://prometheus.io/
 
 ### for Prometheus v1
-
 Add a new job to the main prometheus.conf file:
 
-```yaml
   job: {
     name: "synapse"
 
@@ -17,31 +15,22 @@ Add a new job to the main prometheus.conf file:
       target: "http://SERVER.LOCATION.HERE:PORT/_synapse/metrics"
     }
   }
-```
 
 ### for Prometheus v2
-
 Add a new job to the main prometheus.yml file:
 
-```yaml
   - job_name: "synapse"
     metrics_path: "/_synapse/metrics"
     # when endpoint uses https:
     scheme: "https"
 
     static_configs:
-    - targets: ["my.server.here:port"]
-```
-
-An example of a Prometheus configuration with workers can be found in
-[metrics-howto.md](https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md).
+    - targets: ['SERVER.LOCATION:PORT']
 
 To use `synapse.rules` add
 
-```yaml
-  rule_files:
-    - "/PATH/TO/synapse-v2.rules"
-```
+    rule_files:
+      - "/PATH/TO/synapse-v2.rules"
 
 Metrics are disabled by default when running synapse; they must be enabled
 with the 'enable-metrics' option, either in the synapse config file or as a

contrib/prometheus/consoles/synapse.html

@@ -9,7 +9,7 @@
 new PromConsole.Graph({
   node: document.querySelector("#process_resource_utime"),
   expr: "rate(process_cpu_seconds_total[2m]) * 100",
-  name: "[[job]]-[[index]]",
+  name: "[[job]]",  
   min: 0,
   max: 100,
   renderer: "line",
@@ -22,12 +22,12 @@ new PromConsole.Graph({
 </script>
 
 <h3>Memory</h3>
-<div id="process_resident_memory_bytes"></div>
+<div id="process_resource_maxrss"></div>
 <script>
 new PromConsole.Graph({
-  node: document.querySelector("#process_resident_memory_bytes"),
-  expr: "process_resident_memory_bytes",
-  name: "[[job]]-[[index]]",
+  node: document.querySelector("#process_resource_maxrss"),
+  expr: "process_psutil_rss:max",
+  name: "Maxrss",
   min: 0,
   renderer: "line",
   height: 150,
@@ -43,8 +43,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#process_fds"),
-  expr: "process_open_fds",
-  name: "[[job]]-[[index]]",
+  expr: "process_open_fds{job='synapse'}",
+  name: "FDs",
   min: 0,
   renderer: "line",
   height: 150,
@@ -62,8 +62,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#reactor_total_time"),
-  expr: "rate(python_twisted_reactor_tick_time_sum[2m])",
-  name: "[[job]]-[[index]]",
+  expr: "rate(python_twisted_reactor_tick_time:total[2m]) / 1000",
+  name: "time",
   max: 1,
   min: 0,
   renderer: "area",
@@ -80,8 +80,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#reactor_average_time"),
-  expr: "rate(python_twisted_reactor_tick_time_sum[2m]) / rate(python_twisted_reactor_tick_time_count[2m])",
-  name: "[[job]]-[[index]]",
+  expr: "rate(python_twisted_reactor_tick_time:total[2m]) / rate(python_twisted_reactor_tick_time:count[2m]) / 1000",
+  name: "time",
   min: 0,
   renderer: "line",
   height: 150,
@@ -97,14 +97,14 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#reactor_pending_calls"),
-  expr: "rate(python_twisted_reactor_pending_calls_sum[30s]) / rate(python_twisted_reactor_pending_calls_count[30s])",
-  name: "[[job]]-[[index]]",
+  expr: "rate(python_twisted_reactor_pending_calls:total[30s])/rate(python_twisted_reactor_pending_calls:count[30s])",
+  name: "calls",
   min: 0,
   renderer: "line",
   height: 150,
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
-  yTitle: "Pending Calls"
+  yTitle: "Pending Cals"
 })
 </script>
 
@@ -115,7 +115,7 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_storage_query_time"),
-  expr: "sum(rate(synapse_storage_query_time_count[2m])) by (verb)",
+  expr: "rate(synapse_storage_query_time:count[2m])",
   name: "[[verb]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
@@ -129,8 +129,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_storage_transaction_time"),
-  expr: "topk(10, rate(synapse_storage_transaction_time_count[2m]))",
-  name: "[[job]]-[[index]] [[desc]]",
+  expr: "rate(synapse_storage_transaction_time:count[2m])",
+  name: "[[desc]]",
   min: 0,
   yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
@@ -140,12 +140,12 @@ new PromConsole.Graph({
 </script>
 
 <h3>Transaction execution time</h3>
-<div id="synapse_storage_transactions_time_sec"></div>
+<div id="synapse_storage_transactions_time_msec"></div>
 <script>
 new PromConsole.Graph({
-  node: document.querySelector("#synapse_storage_transactions_time_sec"),
-  expr: "rate(synapse_storage_transaction_time_sum[2m])",
-  name: "[[job]]-[[index]] [[desc]]",
+  node: document.querySelector("#synapse_storage_transactions_time_msec"),
+  expr: "rate(synapse_storage_transaction_time:total[2m]) / 1000",
+  name: "[[desc]]",
   min: 0,
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
@@ -154,33 +154,34 @@ new PromConsole.Graph({
 })
 </script>
 
-<h3>Average time waiting for database connection</h3>
-<div id="synapse_storage_avg_waiting_time"></div>
+<h3>Database scheduling latency</h3>
+<div id="synapse_storage_schedule_time"></div>
 <script>
 new PromConsole.Graph({
-  node: document.querySelector("#synapse_storage_avg_waiting_time"),
-  expr: "rate(synapse_storage_schedule_time_sum[2m]) / rate(synapse_storage_schedule_time_count[2m])",
-  name: "[[job]]-[[index]]",
+  node: document.querySelector("#synapse_storage_schedule_time"),
+  expr: "rate(synapse_storage_schedule_time:total[2m]) / 1000",
+  name: "Total latency",
   min: 0,
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
-  yUnits: "s",
-  yTitle: "Time"
+  yUnits: "s/s",
+  yTitle: "Usage"
 })
 </script>
 
-<h3>Cache request rate</h3>
-<div id="synapse_cache_request_rate"></div>
+<h3>Cache hit ratio</h3>
+<div id="synapse_cache_ratio"></div>
 <script>
 new PromConsole.Graph({
-  node: document.querySelector("#synapse_cache_request_rate"),
-  expr: "rate(synapse_util_caches_cache:total[2m])",
-  name: "[[job]]-[[index]] [[name]]",
+  node: document.querySelector("#synapse_cache_ratio"),
+  expr: "rate(synapse_util_caches_cache:total[2m]) * 100",
+  name: "[[name]]",
   min: 0,
+  max: 100,
   yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-  yUnits: "rps",
-  yTitle: "Cache request rate"
+  yUnits: "%",
+  yTitle: "Percentage"
 })
 </script>
 
@@ -190,7 +191,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_cache_size"),
   expr: "synapse_util_caches_cache:size",
-  name: "[[job]]-[[index]] [[name]]",
+  name: "[[name]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yUnits: "",
@@ -205,8 +206,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_request_count_servlet"),
-  expr: "rate(synapse_http_server_in_flight_requests_count[2m])",
-  name: "[[job]]-[[index]] [[method]] [[servlet]]",
+  expr: "rate(synapse_http_server_request_count:servlet[2m])",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "req/s",
@@ -218,8 +219,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_request_count_servlet_minus_events"),
-  expr: "rate(synapse_http_server_in_flight_requests_count{servlet!=\"EventStreamRestServlet\", servlet!=\"SyncRestServlet\"}[2m])",
-  name: "[[job]]-[[index]] [[method]] [[servlet]]",
+  expr: "rate(synapse_http_server_request_count:servlet{servlet!=\"EventStreamRestServlet\", servlet!=\"SyncRestServlet\"}[2m])",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "req/s",
@@ -232,8 +233,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_response_time_avg"),
-  expr: "rate(synapse_http_server_response_time_seconds_sum[2m]) / rate(synapse_http_server_response_count[2m])",
-  name: "[[job]]-[[index]] [[servlet]]",
+  expr: "rate(synapse_http_server_response_time_seconds[2m]) / rate(synapse_http_server_response_count[2m]) / 1000",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "s/req",
@@ -276,7 +277,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_response_ru_utime"),
   expr: "rate(synapse_http_server_response_ru_utime_seconds[2m])",
-  name: "[[job]]-[[index]] [[servlet]]",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "s/s",
@@ -291,7 +292,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_response_db_txn_duration"),
   expr: "rate(synapse_http_server_response_db_txn_duration_seconds[2m])",
-  name: "[[job]]-[[index]] [[servlet]]",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "s/s",
@@ -305,8 +306,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_send_time_avg"),
-  expr: "rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet'}[2m]) / rate(synapse_http_server_response_count{servlet='RoomSendEventRestServlet'}[2m])",
-  name: "[[job]]-[[index]] [[servlet]]",
+  expr: "rate(synapse_http_server_response_time_second{servlet='RoomSendEventRestServlet'}[2m]) / rate(synapse_http_server_response_count{servlet='RoomSendEventRestServlet'}[2m]) / 1000",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "s/req",
@@ -322,7 +323,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_federation_client_sent"),
   expr: "rate(synapse_federation_client_sent[2m])",
-  name: "[[job]]-[[index]] [[type]]",
+  name: "[[type]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "req/s",
@@ -336,7 +337,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_federation_server_received"),
   expr: "rate(synapse_federation_server_received[2m])",
-  name: "[[job]]-[[index]] [[type]]",
+  name: "[[type]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "req/s",
@@ -366,7 +367,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_notifier_listeners"),
   expr: "synapse_notifier_listeners",
-  name: "[[job]]-[[index]]",
+  name: "listeners",
   min: 0,
   yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
@@ -381,7 +382,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_notifier_notified_events"),
   expr: "rate(synapse_notifier_notified_events[2m])",
-  name: "[[job]]-[[index]]",
+  name: "events",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "events/s",

contrib/prometheus/synapse-v2.rules

@@ -58,21 +58,3 @@ groups:
     labels:
       type: "PDU"
     expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
-
-  - record: synapse_storage_events_persisted_by_source_type
-    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_type="remote"})
-    labels:
-      type: remote
-  - record: synapse_storage_events_persisted_by_source_type
-    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_entity="*client*",origin_type="local"})
-    labels:
-      type: local
-  - record: synapse_storage_events_persisted_by_source_type
-    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_entity!="*client*",origin_type="local"})
-    labels:
-      type: bridges
-  - record: synapse_storage_events_persisted_by_event_type
-    expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep)
-  - record: synapse_storage_events_persisted_by_origin
-    expr: sum without(type) (synapse_storage_events_persisted_events_sep)
-

contrib/purge_api/README.md

@@ -3,7 +3,7 @@ Purge history API examples
 
 # `purge_history.sh`
 
-A bash file, that uses the [purge history API](/docs/admin_api/purge_history_api.rst) to 
+A bash file, that uses the [purge history API](/docs/admin_api/README.rst) to 
 purge all messages in a list of rooms up to a certain event. You can select a 
 timeframe or a number of messages that you want to keep in the room.
 
@@ -12,5 +12,5 @@ the script.
 
 # `purge_remote_media.sh`
 
-A bash file, that uses the [purge history API](/docs/admin_api/purge_history_api.rst) to 
+A bash file, that uses the [purge history API](/docs/admin_api/README.rst) to 
 purge all old cached remote media.

contrib/purge_api/purge_history.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # this script will use the api:
 #    https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst

contrib/purge_api/purge_remote_media.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 DOMAIN=yourserver.tld
 # add this user as admin in your home server:
@@ -51,4 +51,4 @@ TOKEN=$(sql "SELECT token FROM access_tokens WHERE user_id='$ADMIN' ORDER BY id
 # finally start pruning media:
 ###############################################################################
 set -x # for debugging the generated string
-curl --header "Authorization: Bearer $TOKEN" -X POST "$API_URL/admin/purge_media_cache/?before_ts=$UNIX_TIMESTAMP"
+curl --header "Authorization: Bearer $TOKEN" -v POST "$API_URL/admin/purge_media_cache/?before_ts=$UNIX_TIMESTAMP"

contrib/scripts/kick_users.py

@@ -1,36 +1,34 @@
 #!/usr/bin/env python
-
+from argparse import ArgumentParser
 import json
+import requests
 import sys
 import urllib
-from argparse import ArgumentParser
-
-import requests
-
 
 def _mkurl(template, kws):
     for key in kws:
         template = template.replace(key, kws[key])
     return template
 
-
 def main(hs, room_id, access_token, user_id_prefix, why):
     if not why:
         why = "Automated kick."
-    print(
-        "Kicking members on %s in room %s matching %s" % (hs, room_id, user_id_prefix)
-    )
+    print "Kicking members on %s in room %s matching %s" % (hs, room_id, user_id_prefix)
     room_state_url = _mkurl(
         "$HS/_matrix/client/api/v1/rooms/$ROOM/state?access_token=$TOKEN",
-        {"$HS": hs, "$ROOM": room_id, "$TOKEN": access_token},
+        {
+            "$HS": hs,
+            "$ROOM": room_id,
+            "$TOKEN": access_token
+        }
     )
-    print("Getting room state => %s" % room_state_url)
+    print "Getting room state => %s" % room_state_url
     res = requests.get(room_state_url)
-    print("HTTP %s" % res.status_code)
+    print "HTTP %s" % res.status_code
     state_events = res.json()
     if "error" in state_events:
-        print("FATAL")
-        print(state_events)
+        print "FATAL"
+        print state_events
         return
 
     kick_list = []
@@ -46,40 +44,47 @@ def main(hs, room_id, access_token, user_id_prefix, why):
             kick_list.append(event["state_key"])
 
     if len(kick_list) == 0:
-        print("No user IDs match the prefix '%s'" % user_id_prefix)
+        print "No user IDs match the prefix '%s'" % user_id_prefix
         return
 
-    print("The following user IDs will be kicked from %s" % room_name)
+    print "The following user IDs will be kicked from %s" % room_name
     for uid in kick_list:
-        print(uid)
-    doit = input("Continue? [Y]es\n")
-    if len(doit) > 0 and doit.lower() == "y":
-        print("Kicking members...")
+        print uid
+    doit = raw_input("Continue? [Y]es\n")
+    if len(doit) > 0 and doit.lower() == 'y':
+        print "Kicking members..."
         # encode them all
         kick_list = [urllib.quote(uid) for uid in kick_list]
         for uid in kick_list:
             kick_url = _mkurl(
                 "$HS/_matrix/client/api/v1/rooms/$ROOM/state/m.room.member/$UID?access_token=$TOKEN",
-                {"$HS": hs, "$UID": uid, "$ROOM": room_id, "$TOKEN": access_token},
+                {
+                    "$HS": hs,
+                    "$UID": uid,
+                    "$ROOM": room_id,
+                    "$TOKEN": access_token
+                }
             )
-            kick_body = {"membership": "leave", "reason": why}
-            print("Kicking %s" % uid)
+            kick_body = {
+                "membership": "leave",
+                "reason": why
+            }
+            print "Kicking %s" % uid
             res = requests.put(kick_url, data=json.dumps(kick_body))
             if res.status_code != 200:
-                print("ERROR: HTTP %s" % res.status_code)
+                print "ERROR: HTTP %s" % res.status_code
             if res.json().get("error"):
-                print("ERROR: JSON %s" % res.json())
-
+                print "ERROR: JSON %s" % res.json()
+            
+    
 
 if __name__ == "__main__":
     parser = ArgumentParser("Kick members in a room matching a certain user ID prefix.")
-    parser.add_argument("-u", "--user-id", help="The user ID prefix e.g. '@irc_'")
-    parser.add_argument("-t", "--token", help="Your access_token")
-    parser.add_argument("-r", "--room", help="The room ID to kick members in")
-    parser.add_argument(
-        "-s", "--homeserver", help="The base HS url e.g. http://matrix.org"
-    )
-    parser.add_argument("-w", "--why", help="Reason for the kick. Optional.")
+    parser.add_argument("-u","--user-id",help="The user ID prefix e.g. '@irc_'")
+    parser.add_argument("-t","--token",help="Your access_token")
+    parser.add_argument("-r","--room",help="The room ID to kick members in")
+    parser.add_argument("-s","--homeserver",help="The base HS url e.g. http://matrix.org")
+    parser.add_argument("-w","--why",help="Reason for the kick. Optional.")
     args = parser.parse_args()
     if not args.room or not args.token or not args.user_id or not args.homeserver:
         parser.print_help()

contrib/systemd-with-workers/README.md

@@ -1,2 +0,0 @@
-The documentation for using systemd to manage synapse workers is now part of
-the main synapse distribution. See [docs/systemd-with-workers](../../docs/systemd-with-workers).

contrib/systemd/README.md

@@ -1,17 +0,0 @@
-# Setup Synapse with Systemd
-This is a setup for managing synapse with a user contributed systemd unit 
-file. It provides a `matrix-synapse` systemd unit file that should be tailored 
-to accommodate your installation in accordance with the installation 
-instructions provided in [installation instructions](../../INSTALL.md).
-
-## Setup
-1. Under the service section, ensure the `User` variable matches which user
-you installed synapse under and wish to run it as. 
-2. Under the service section, ensure the `WorkingDirectory` variable matches
-where you have installed synapse.
-3. Under the service section, ensure the `ExecStart` variable matches the
-appropriate locations of your installation.
-4. Copy the `matrix-synapse.service` to `/etc/systemd/system/`
-5. Start Synapse: `sudo systemctl start matrix-synapse`
-6. Verify Synapse is running: `sudo systemctl status matrix-synapse`
-7. *optional* Enable Synapse to start at system boot: `sudo systemctl enable matrix-synapse`

contrib/systemd/log_config.yaml

@@ -8,7 +8,7 @@ formatters:
 
 filters:
     context:
-        (): synapse.logging.context.LoggingContextFilter
+        (): synapse.util.logcontext.LoggingContextFilter
         request: ""
 
 handlers:

contrib/systemd/matrix-synapse.service

@@ -4,36 +4,28 @@
 #    systemctl enable matrix-synapse
 #    systemctl start matrix-synapse
 #
-# This assumes that Synapse has been installed by a user named
-# synapse.
-#
 # This assumes that Synapse has been installed in a virtualenv in
-# the user's home directory: `/home/synapse/synapse/env`.
+# /opt/synapse/env.
 #
 # **NOTE:** This is an example service file that may change in the future. If you
 # wish to use this please copy rather than symlink it.
 
 [Unit]
 Description=Synapse Matrix homeserver
-# If you are using postgresql to persist data, uncomment this line to make sure
-# synapse starts after the postgresql service.
-# After=postgresql.service
 
 [Service]
-Type=notify
-NotifyAccess=main
-ExecReload=/bin/kill -HUP $MAINPID
+Type=simple
 Restart=on-abort
 
 User=synapse
 Group=nogroup
 
-WorkingDirectory=/home/synapse/synapse
-ExecStart=/home/synapse/synapse/env/bin/python -m synapse.app.homeserver --config-path=/home/synapse/synapse/homeserver.yaml
-SyslogIdentifier=matrix-synapse
+WorkingDirectory=/opt/synapse
+ExecStart=/opt/synapse/env/bin/python -m synapse.app.homeserver --config-path=/opt/synapse/homeserver.yaml
 
 # adjust the cache factor if necessary
 # Environment=SYNAPSE_CACHE_FACTOR=2.0
 
 [Install]
 WantedBy=multi-user.target
+

debian/build_virtualenv

@@ -33,44 +33,34 @@ esac
 # Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather
 # than the 2/3 compatible `virtualenv`.
 
-# Pin pip to 20.3.4 to fix breakage in 21.0 on py3.5 (xenial)
-
 dh_virtualenv \
     --install-suffix "matrix-synapse" \
     --builtin-venv \
+    --setuptools \
     --python "$SNAKE" \
-    --upgrade-pip-to="20.3.4" \
+    --upgrade-pip \
     --preinstall="lxml" \
     --preinstall="mock" \
     --extra-pip-arg="--no-cache-dir" \
     --extra-pip-arg="--compile" \
-    --extras="all,systemd,test"
+    --extras="all"
 
 PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
 VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"
 TARGET_PYTHON="${VIRTUALENV_DIR}/bin/python"
 
-case "$DEB_BUILD_OPTIONS" in
-    *nocheck*)
-        # Skip running tests if "nocheck" present in $DEB_BUILD_OPTIONS
-        ;;
+# we copy the tests to a temporary directory so that we can put them on the
+# PYTHONPATH without putting the uninstalled synapse on the pythonpath.
+tmpdir=`mktemp -d`
+trap "rm -r $tmpdir" EXIT
 
-    *)
-        # Copy tests to a temporary directory so that we can put them on the
-        # PYTHONPATH without putting the uninstalled synapse on the pythonpath.
-        tmpdir=`mktemp -d`
-        trap "rm -r $tmpdir" EXIT
+cp -r tests "$tmpdir"
 
-        cp -r tests "$tmpdir"
-
-        PYTHONPATH="$tmpdir" \
-            "${TARGET_PYTHON}" -m twisted.trial --reporter=text -j2 tests
-
-        ;;
-esac
+PYTHONPATH="$tmpdir" \
+    "${TARGET_PYTHON}" -B -m twisted.trial --reporter=text -j2 tests
 
 # build the config file
-"${TARGET_PYTHON}" "${VIRTUALENV_DIR}/bin/generate_config" \
+"${TARGET_PYTHON}" -B "${VIRTUALENV_DIR}/bin/generate_config" \
         --config-dir="/etc/matrix-synapse" \
         --data-dir="/var/lib/matrix-synapse" |
     perl -pe '
@@ -95,9 +85,6 @@ esac
 
 ' > "${PACKAGE_BUILD_DIR}/etc/matrix-synapse/homeserver.yaml"
 
-# build the log config file
-"${TARGET_PYTHON}" "${VIRTUALENV_DIR}/bin/generate_log_config" \
-        --output-file="${PACKAGE_BUILD_DIR}/etc/matrix-synapse/log.yaml"
 
 # add a dependency on the right version of python to substvars.
 PYPKG=`basename $SNAKE`

debian/changelog

@@ -1,528 +1,3 @@
-matrix-synapse-py3 (1.33.2) stable; urgency=medium
-
-  * New synapse release 1.33.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 11 May 2021 11:17:59 +0100
-
-matrix-synapse-py3 (1.33.1) stable; urgency=medium
-
-  * New synapse release 1.33.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 06 May 2021 14:06:33 +0100
-
-matrix-synapse-py3 (1.33.0) stable; urgency=medium
-
-  * New synapse release 1.33.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 05 May 2021 14:15:27 +0100
-
-matrix-synapse-py3 (1.32.2) stable; urgency=medium
-
-  * New synapse release 1.32.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 22 Apr 2021 12:43:52 +0100
-
-matrix-synapse-py3 (1.32.1) stable; urgency=medium
-
-  * New synapse release 1.32.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 21 Apr 2021 14:00:55 +0100
-
-matrix-synapse-py3 (1.32.0) stable; urgency=medium
-
-  [ Dan Callahan ]
-  * Skip tests when DEB_BUILD_OPTIONS contains "nocheck".
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.32.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Apr 2021 14:28:39 +0100
-
-matrix-synapse-py3 (1.31.0) stable; urgency=medium
-
-  * New synapse release 1.31.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 06 Apr 2021 13:08:29 +0100
-
-matrix-synapse-py3 (1.30.1) stable; urgency=medium
-
-  * New synapse release 1.30.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 26 Mar 2021 12:01:28 +0000
-
-matrix-synapse-py3 (1.30.0) stable; urgency=medium
-
-  * New synapse release 1.30.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 22 Mar 2021 13:15:34 +0000
-
-matrix-synapse-py3 (1.29.0) stable; urgency=medium
-
-  [ Jonathan de Jong ]
-  * Remove the python -B flag (don't generate bytecode) in scripts and documentation.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.29.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 08 Mar 2021 13:51:50 +0000
-
-matrix-synapse-py3 (1.28.0) stable; urgency=medium
-
-  * New synapse release 1.28.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 25 Feb 2021 10:21:57 +0000
-
-matrix-synapse-py3 (1.27.0) stable; urgency=medium
-
-  [ Dan Callahan ]
-  * Fix build on Ubuntu 16.04 LTS (Xenial).
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.27.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 16 Feb 2021 13:11:28 +0000
-
-matrix-synapse-py3 (1.26.0) stable; urgency=medium
-
-  [ Richard van der Hoff ]
-  * Remove dependency on `python3-distutils`.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.26.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 27 Jan 2021 12:43:35 -0500
-
-matrix-synapse-py3 (1.25.0) stable; urgency=medium
-
-  [ Dan Callahan ]
-  * Update dependencies to account for the removal of the transitional
-    dh-systemd package from Debian Bullseye.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.25.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 13 Jan 2021 10:14:55 +0000
-
-matrix-synapse-py3 (1.24.0) stable; urgency=medium
-
-  * New synapse release 1.24.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 09 Dec 2020 10:14:30 +0000
-
-matrix-synapse-py3 (1.23.1) stable; urgency=medium
-
-  * New synapse release 1.23.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 09 Dec 2020 10:40:39 +0000
-
-matrix-synapse-py3 (1.23.0) stable; urgency=medium
-
-  * New synapse release 1.23.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 18 Nov 2020 11:41:28 +0000
-
-matrix-synapse-py3 (1.22.1) stable; urgency=medium
-
-  * New synapse release 1.22.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 30 Oct 2020 15:25:37 +0000
-
-matrix-synapse-py3 (1.22.0) stable; urgency=medium
-
-  * New synapse release 1.22.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 27 Oct 2020 12:07:12 +0000
-
-matrix-synapse-py3 (1.21.2) stable; urgency=medium
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.21.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 15 Oct 2020 09:23:27 -0400
-
-matrix-synapse-py3 (1.21.1) stable; urgency=medium
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.21.1.
-
-  [ Andrew Morgan ]
-  * Explicitly install "test" python dependencies.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Oct 2020 10:24:13 +0100
-
-matrix-synapse-py3 (1.21.0) stable; urgency=medium
-
-  * New synapse release 1.21.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 12 Oct 2020 15:47:44 +0100
-
-matrix-synapse-py3 (1.20.1) stable; urgency=medium
-
-  * New synapse release 1.20.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 24 Sep 2020 16:25:22 +0100
-
-matrix-synapse-py3 (1.20.0) stable; urgency=medium
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.20.0.
-
-  [ Dexter Chua ]
-  * Use Type=notify in systemd service
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 22 Sep 2020 15:19:32 +0100
-
-matrix-synapse-py3 (1.19.3) stable; urgency=medium
-
-  * New synapse release 1.19.3.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 18 Sep 2020 14:59:30 +0100
-
-matrix-synapse-py3 (1.19.2) stable; urgency=medium
-
-  * New synapse release 1.19.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 16 Sep 2020 12:50:30 +0100
-
-matrix-synapse-py3 (1.19.1) stable; urgency=medium
-
-  * New synapse release 1.19.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 27 Aug 2020 10:50:19 +0100
-
-matrix-synapse-py3 (1.19.0) stable; urgency=medium
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.19.0.
-
-  [ Aaron Raimist ]
-  * Fix outdated documentation for SYNAPSE_CACHE_FACTOR
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 17 Aug 2020 14:06:42 +0100
-
-matrix-synapse-py3 (1.18.0) stable; urgency=medium
-
-  * New synapse release 1.18.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 30 Jul 2020 10:55:53 +0100
-
-matrix-synapse-py3 (1.17.0) stable; urgency=medium
-
-  * New synapse release 1.17.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 13 Jul 2020 10:20:31 +0100
-
-matrix-synapse-py3 (1.16.1) stable; urgency=medium
-
-  * New synapse release 1.16.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 10 Jul 2020 12:09:24 +0100
-
-matrix-synapse-py3 (1.17.0rc1) stable; urgency=medium
-
-  * New synapse release 1.17.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 09 Jul 2020 16:53:12 +0100
-
-matrix-synapse-py3 (1.16.0) stable; urgency=medium
-
-  * New synapse release 1.16.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 08 Jul 2020 11:03:48 +0100
-
-matrix-synapse-py3 (1.15.2) stable; urgency=medium
-
-  * New synapse release 1.15.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 02 Jul 2020 10:34:00 -0400
-
-matrix-synapse-py3 (1.15.1) stable; urgency=medium
-
-  * New synapse release 1.15.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 16 Jun 2020 10:27:50 +0100
-
-matrix-synapse-py3 (1.15.0) stable; urgency=medium
-
-  * New synapse release 1.15.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 11 Jun 2020 13:27:06 +0100
-
-matrix-synapse-py3 (1.14.0) stable; urgency=medium
-
-  * New synapse release 1.14.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 28 May 2020 10:37:27 +0000
-
-matrix-synapse-py3 (1.13.0) stable; urgency=medium
-
-  [ Patrick Cloke ]
-  * Add information about .well-known files to Debian installation scripts.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.13.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 19 May 2020 09:16:56 -0400
-
-matrix-synapse-py3 (1.12.4) stable; urgency=medium
-
-  * New synapse release 1.12.4.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 23 Apr 2020 10:58:14 -0400
-
-matrix-synapse-py3 (1.12.3) stable; urgency=medium
-
-  [ Richard van der Hoff ]
-  * Update the Debian build scripts to handle the new installation paths
-   for the support libraries introduced by Pillow 7.1.1.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.12.3.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 03 Apr 2020 10:55:03 +0100
-
-matrix-synapse-py3 (1.12.2) stable; urgency=medium
-
-  * New synapse release 1.12.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 02 Apr 2020 19:02:17 +0000
-
-matrix-synapse-py3 (1.12.1) stable; urgency=medium
-
-  * New synapse release 1.12.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 02 Apr 2020 11:30:47 +0000
-
-matrix-synapse-py3 (1.12.0) stable; urgency=medium
-
-  * New synapse release 1.12.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 23 Mar 2020 12:13:03 +0000
-
-matrix-synapse-py3 (1.11.1) stable; urgency=medium
-
-  * New synapse release 1.11.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 03 Mar 2020 15:01:22 +0000
-
-matrix-synapse-py3 (1.11.0) stable; urgency=medium
-
-  * New synapse release 1.11.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 21 Feb 2020 08:54:34 +0000
-
-matrix-synapse-py3 (1.10.1) stable; urgency=medium
-
-  * New synapse release 1.10.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 17 Feb 2020 16:27:28 +0000
-
-matrix-synapse-py3 (1.10.0) stable; urgency=medium
-
-  * New synapse release 1.10.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 12 Feb 2020 12:18:54 +0000
-
-matrix-synapse-py3 (1.9.1) stable; urgency=medium
-
-  * New synapse release 1.9.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 28 Jan 2020 13:09:23 +0000
-
-matrix-synapse-py3 (1.9.0) stable; urgency=medium
-
-  * New synapse release 1.9.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 23 Jan 2020 12:56:31 +0000
-
-matrix-synapse-py3 (1.8.0) stable; urgency=medium
-
-  [ Richard van der Hoff ]
-  * Automate generation of the default log configuration file.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.8.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 09 Jan 2020 11:39:27 +0000
-
-matrix-synapse-py3 (1.7.3) stable; urgency=medium
-
-  * New synapse release 1.7.3.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 31 Dec 2019 10:45:04 +0000
-
-matrix-synapse-py3 (1.7.2) stable; urgency=medium
-
-  * New synapse release 1.7.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 20 Dec 2019 10:56:50 +0000
-
-matrix-synapse-py3 (1.7.1) stable; urgency=medium
-
-  * New synapse release 1.7.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 18 Dec 2019 09:37:59 +0000
-
-matrix-synapse-py3 (1.7.0) stable; urgency=medium
-
-  * New synapse release 1.7.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 13 Dec 2019 10:19:38 +0000
-
-matrix-synapse-py3 (1.6.1) stable; urgency=medium
-
-  * New synapse release 1.6.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 28 Nov 2019 11:10:40 +0000
-
-matrix-synapse-py3 (1.6.0) stable; urgency=medium
-
-  * New synapse release 1.6.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 26 Nov 2019 12:15:40 +0000
-
-matrix-synapse-py3 (1.5.1) stable; urgency=medium
-
-  * New synapse release 1.5.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 06 Nov 2019 10:02:14 +0000
-
-matrix-synapse-py3 (1.5.0) stable; urgency=medium
-
-  * New synapse release 1.5.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 29 Oct 2019 14:28:41 +0000
-
-matrix-synapse-py3 (1.4.1) stable; urgency=medium
-
-  * New synapse release 1.4.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 18 Oct 2019 10:13:27 +0100
-
-matrix-synapse-py3 (1.4.0) stable; urgency=medium
-
-  * New synapse release 1.4.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 03 Oct 2019 13:22:25 +0100
-
-matrix-synapse-py3 (1.3.1) stable; urgency=medium
-
-  * New synapse release 1.3.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Sat, 17 Aug 2019 09:15:49 +0100
-
-matrix-synapse-py3 (1.3.0) stable; urgency=medium
-
-  [ Andrew Morgan ]
-  * Remove libsqlite3-dev from required build dependencies.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.3.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 15 Aug 2019 12:04:23 +0100
-
-matrix-synapse-py3 (1.2.0) stable; urgency=medium
-
-  [ Amber Brown ]
-  * Update logging config defaults to match API changes in Synapse.
-
-  [ Richard van der Hoff ]
-  * Add Recommends and Depends for some libraries which you probably want.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.2.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 25 Jul 2019 14:10:07 +0100
-
-matrix-synapse-py3 (1.1.0) stable; urgency=medium
-
-  [ Silke Hofstra ]
-  * Include systemd-python to allow logging to the systemd journal.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.1.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 04 Jul 2019 11:43:41 +0100
-
-matrix-synapse-py3 (1.0.0) stable; urgency=medium
-
-  * New synapse release 1.0.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 11 Jun 2019 17:09:53 +0100
-
-matrix-synapse-py3 (0.99.5.2) stable; urgency=medium
-
-  * New synapse release 0.99.5.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 30 May 2019 16:28:07 +0100
-
-matrix-synapse-py3 (0.99.5.1) stable; urgency=medium
-
-  * New synapse release 0.99.5.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 22 May 2019 16:22:24 +0000
-
-matrix-synapse-py3 (0.99.4) stable; urgency=medium
-
-  [ Christoph Müller ]
-  * Configure the systemd units to have a log identifier of `matrix-synapse`
-
-  [ Synapse Packaging team ]
-  * New synapse release 0.99.4.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 15 May 2019 13:58:08 +0100
-
-matrix-synapse-py3 (0.99.3.2) stable; urgency=medium
-
-  * New synapse release 0.99.3.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 03 May 2019 18:56:20 +0100
-
-matrix-synapse-py3 (0.99.3.1) stable; urgency=medium
-
-  * New synapse release 0.99.3.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 03 May 2019 16:02:43 +0100
-
-matrix-synapse-py3 (0.99.3) stable; urgency=medium
-
-  [ Richard van der Hoff ]
-  * Fix warning during preconfiguration. (Fixes: #4819)
-
-  [ Synapse Packaging team ]
-  * New synapse release 0.99.3.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 01 Apr 2019 12:48:21 +0000
-
-matrix-synapse-py3 (0.99.2) stable; urgency=medium
-
-  * Fix overwriting of config settings on upgrade.
-  * New synapse release 0.99.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 01 Mar 2019 10:55:08 +0000
-
-matrix-synapse-py3 (0.99.1.1) stable; urgency=medium
-
-  * New synapse release 0.99.1.1
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 14 Feb 2019 17:19:44 +0000
-
-matrix-synapse-py3 (0.99.1) stable; urgency=medium
-
-  [ Damjan Georgievski ]
-  * Added ExecReload= in service unit file to send a HUP signal
-
-  [ Synapse Packaging team ]
-  * New synapse release 0.99.1
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 14 Feb 2019 14:12:26 +0000
-
-matrix-synapse-py3 (0.99.0) stable; urgency=medium
-
-  * New synapse release 0.99.0
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 5 Feb 2019 18:25:00 +0000
-
 matrix-synapse-py3 (0.34.1.1++1) stable; urgency=medium
 
   * Update conflicts specifications to allow smoother transition from matrix-synapse.

debian/config

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+
+. /usr/share/debconf/confmodule
+
+db_input high matrix-synapse/server-name || true
+db_input high matrix-synapse/report-stats || true
+db_go

debian/control

@@ -2,15 +2,10 @@ Source: matrix-synapse-py3
 Section: contrib/python
 Priority: extra
 Maintainer: Synapse Packaging team <packages@matrix.org>
-# keep this list in sync with the build dependencies in docker/Dockerfile-dhvirtualenv.
-# TODO: Remove the dependency on dh-systemd after dropping support for Ubuntu xenial
-#       On all other supported releases, it's merely a transitional package which
-#       does nothing but depends on debhelper (> 9.20160709)
 Build-Depends:
- debhelper (>= 9.20160709) | dh-systemd,
+ debhelper (>= 9),
+ dh-systemd,
  dh-virtualenv (>= 1.1),
- libsystemd-dev,
- libpq-dev,
  lsb-release,
  python3-dev,
  python3,
@@ -31,13 +26,11 @@ Pre-Depends: dpkg (>= 1.16.1)
 Depends:
  adduser,
  debconf,
+ python3-distutils|libpython3-stdlib (<< 3.6),
  ${misc:Depends},
- ${shlibs:Depends},
  ${synapse:pydepends},
 # some of our scripts use perl, but none of them are important,
 # so we put perl:Depends in Suggests rather than Depends.
-Recommends:
- ${shlibs1:Recommends},
 Suggests:
  sqlite3,
  ${perl:Depends},

debian/install

@@ -1 +1 @@
-debian/manage_debconf.pl /opt/venvs/matrix-synapse/lib/
+debian/log.yaml etc/matrix-synapse

debian/log.yaml

@@ -0,0 +1,36 @@
+
+version: 1
+
+formatters:
+  precise:
+   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s'
+
+filters:
+  context:
+    (): synapse.util.logcontext.LoggingContextFilter
+    request: ""
+
+handlers:
+  file:
+    class: logging.handlers.RotatingFileHandler
+    formatter: precise
+    filename: /var/log/matrix-synapse/homeserver.log
+    maxBytes: 104857600
+    backupCount: 10
+    filters: [context]
+    encoding: utf8
+  console:
+    class: logging.StreamHandler
+    formatter: precise
+    level: WARN
+
+loggers:
+    synapse:
+        level: INFO
+
+    synapse.storage.SQL:
+        level: INFO
+
+root:
+    level: INFO
+    handlers: [file, console]

debian/manage_debconf.pl

@@ -1,130 +0,0 @@
-#!/usr/bin/perl
-#
-# Interface between our config files and the debconf database.
-#
-# Usage:
-#
-#   manage_debconf.pl <action>
-#
-# where <action> can be:
-#
-#   read:    read the configuration from the yaml into debconf
-#   update:  update the yaml config according to the debconf database
-use strict;
-use warnings;
-
-use Debconf::Client::ConfModule (qw/get set/);
-
-# map from the name of a setting in our .yaml file to the relevant debconf
-# setting.
-my %MAPPINGS=(
-    server_name => 'matrix-synapse/server-name',
-    report_stats => 'matrix-synapse/report-stats',
-);
-
-# enable debug if dpkg --debug
-my $DEBUG = $ENV{DPKG_MAINTSCRIPT_DEBUG};
-
-sub read_config {
-    my @files = @_;
-
-    foreach my $file (@files)  {
-        print STDERR "reading $file\n" if $DEBUG;
-
-        open my $FH, "<", $file or next;
-
-        # rudimentary parsing which (a) avoids having to depend on a yaml library,
-        # and (b) is tolerant of yaml errors
-        while($_ = <$FH>) {
-            while (my ($setting, $debconf) = each %MAPPINGS) {
-                $setting = quotemeta $setting;
-                if(/^${setting}\s*:(.*)$/) {
-                    my $val = $1;
-
-                    # remove leading/trailing whitespace
-                    $val =~ s/^\s*//;
-                    $val =~ s/\s*$//;
-
-                    # remove surrounding quotes
-                    if ($val =~ /^"(.*)"$/ || $val =~ /^'(.*)'$/) {
-                        $val = $1;
-                    }
-
-                    print STDERR ">> $debconf = $val\n" if $DEBUG;
-                    set($debconf, $val);
-                }
-            }
-        }
-        close $FH;
-    }
-}
-
-sub update_config {
-    my @files = @_;
-
-    my %substs = ();
-    while (my ($setting, $debconf) = each %MAPPINGS) {
-        my @res = get($debconf);
-        $substs{$setting} = $res[1] if $res[0] == 0;
-    }
-
-    foreach my $file (@files) {
-        print STDERR "checking $file\n" if $DEBUG;
-
-        open my $FH, "<", $file or next;
-
-        my $updated = 0;
-
-        # read the whole file into memory
-        my @lines = <$FH>;
-
-        while (my ($setting, $val) = each %substs) {
-            $setting = quotemeta $setting;
-
-            map {
-                if (/^${setting}\s*:\s*(.*)\s*$/) {
-                    my $current = $1;
-                    if ($val ne $current) {
-                        $_ = "${setting}: $val\n";
-                        $updated = 1;
-                    }
-                }
-            } @lines;
-        }
-        close $FH;
-
-        next unless $updated;
-
-        print STDERR "updating $file\n" if $DEBUG;
-        open $FH, ">", $file or die "unable to update $file";
-        print $FH @lines;
-        close $FH;
-    }
-}
-
-
-my $cmd = $ARGV[0];
-
-my $read = 0;
-my $update = 0;
-
-if (not $cmd) {
-    die "must specify a command to perform\n";
-} elsif ($cmd eq 'read') {
-    $read = 1;
-} elsif ($cmd eq 'update') {
-    $update = 1;
-} else {
-    die "unknown command '$cmd'\n";
-}
-
-my @files = (
-    "/etc/matrix-synapse/homeserver.yaml",
-    glob("/etc/matrix-synapse/conf.d/*.yaml"),
-);
-
-if ($read) {
-    read_config(@files);
-} elsif ($update) {
-    update_config(@files);
-}

debian/matrix-synapse-py3.config

@@ -1,16 +0,0 @@
-#!/bin/sh
-
-set -e
-
-. /usr/share/debconf/confmodule
-
-# try to update the debconf db according to whatever is in the config files
-#
-# note that we may get run during preconfiguration, in which case the script
-# will not yet be installed.
-[ -x /opt/venvs/matrix-synapse/lib/manage_debconf.pl ] && \
-    /opt/venvs/matrix-synapse/lib/manage_debconf.pl read
-
-db_input high matrix-synapse/server-name || true
-db_input high matrix-synapse/report-stats || true
-db_go

debian/matrix-synapse-py3.postinst

@@ -8,36 +8,19 @@ USER="matrix-synapse"
 
 case "$1" in
   configure|reconfigure)
-
-    # generate template config files if they don't exist
+    # Set server name in config file
     mkdir -p "/etc/matrix-synapse/conf.d/"
-    if [ ! -e "$CONFIGFILE_SERVERNAME" ]; then
-        cat > "$CONFIGFILE_SERVERNAME" <<EOF
-# This file is autogenerated, and will be recreated on upgrade if it is deleted.
-# Any changes you make will be preserved.
+    db_get matrix-synapse/server-name
 
-# The domain name of the server, with optional explicit port.
-# This is used by remote servers to connect to this server,
-# e.g. matrix.org, localhost:8080, etc.
-# This is also the last part of your UserID.
-#
-server_name: ''
-EOF
+    if [ "$RET" ]; then
+        echo "server_name: $RET" > $CONFIGFILE_SERVERNAME
     fi
 
-    if [ ! -e "$CONFIGFILE_REPORTSTATS" ]; then
-        cat > "$CONFIGFILE_REPORTSTATS" <<EOF
-# This file is autogenerated, and will be recreated on upgrade if it is deleted.
-# Any changes you make will be preserved.
-
-# Whether to report anonymized homeserver usage statistics.
-report_stats: false
-EOF
+    db_get matrix-synapse/report-stats
+    if [ "$RET" ]; then
+        echo "report_stats: $RET" > $CONFIGFILE_REPORTSTATS
     fi
 
-    # update the config files according to whatever is in the debconf database
-    /opt/venvs/matrix-synapse/lib/manage_debconf.pl update
-
     if ! getent passwd $USER >/dev/null; then
       adduser --quiet --system --no-create-home --home /var/lib/matrix-synapse $USER
     fi

debian/matrix-synapse.default

@@ -1,2 +1,2 @@
 # Specify environment variables used when running Synapse
-# SYNAPSE_CACHE_FACTOR=0.5 (default)
+# SYNAPSE_CACHE_FACTOR=1 (default)

debian/matrix-synapse.service

@@ -2,16 +2,14 @@
 Description=Synapse Matrix homeserver
 
 [Service]
-Type=notify
+Type=simple
 User=matrix-synapse
 WorkingDirectory=/var/lib/matrix-synapse
 EnvironmentFile=/etc/default/matrix-synapse
 ExecStartPre=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys
 ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/
-ExecReload=/bin/kill -HUP $MAINPID
 Restart=always
 RestartSec=3
-SyslogIdentifier=matrix-synapse
 
 [Install]
 WantedBy=multi-user.target

debian/po/templates.pot

@@ -1,14 +1,14 @@
 # SOME DESCRIPTIVE TITLE.
 # Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
-# This file is distributed under the same license as the matrix-synapse-py3 package.
+# This file is distributed under the same license as the matrix-synapse package.
 # FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
 #
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: matrix-synapse-py3\n"
-"Report-Msgid-Bugs-To: matrix-synapse-py3@packages.debian.org\n"
-"POT-Creation-Date: 2020-04-06 16:39-0400\n"
+"Project-Id-Version: matrix-synapse\n"
+"Report-Msgid-Bugs-To: matrix-synapse@packages.debian.org\n"
+"POT-Creation-Date: 2017-02-21 07:51+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -28,10 +28,7 @@ msgstr ""
 #: ../templates:1001
 msgid ""
 "The name that this homeserver will appear as, to clients and other servers "
-"via federation. This is normally the public hostname of the server running "
-"synapse, but can be different if you set up delegation. Please refer to the "
-"delegation documentation in this case: https://github.com/matrix-org/synapse/"
-"blob/master/docs/delegate.md."
+"via federation. This name should match the SRV record published in DNS."
 msgstr ""
 
 #. Type: boolean

debian/rules

@@ -3,50 +3,15 @@
 # Build Debian package using https://github.com/spotify/dh-virtualenv
 #
 
-# assume we only have one package
-PACKAGE_NAME:=`dh_listpackages`
-
 override_dh_systemd_enable:
 	dh_systemd_enable --name=matrix-synapse
 
 override_dh_installinit:
 	dh_installinit --name=matrix-synapse
 
-# we don't really want to strip the symbols from our object files.
 override_dh_strip:
 
-# dh_shlibdeps calls dpkg-shlibdeps, which finds all the binary files
-# (executables and shared libs) in the package, and looks for the shared
-# libraries that they depend on. It then adds a dependency on the package that
-# contains that library to the package.
-#
-# We make two modifications to that process...
-#
 override_dh_shlibdeps:
-        # Firstly, postgres is not a hard dependency for us, so we want to make
-        # the things that psycopg2 depends on (such as libpq) be
-        # recommendations rather than hard dependencies. We do so by
-        # running dpkg-shlibdeps manually on psycopg2's libs.
-        #
-	find debian/$(PACKAGE_NAME)/ -path '*/site-packages/psycopg2/*.so' | \
-	    xargs dpkg-shlibdeps -Tdebian/$(PACKAGE_NAME).substvars \
-	        -pshlibs1 -dRecommends
-
-        # secondly, we exclude PIL's libraries from the process. They are known
-        # to be self-contained, but they have interdependencies and
-        # dpkg-shlibdeps doesn't know how to resolve them.
-        #
-        # As of Pillow 7.1.0, these libraries are in
-        # site-packages/Pillow.libs. Previously, they were in
-        # site-packages/PIL/.libs.
-        #
-        # (we also need to exclude psycopg2, of course, since we've already
-        # dealt with that.)
-        #
-	dh_shlibdeps \
-	    -X site-packages/PIL/.libs \
-	    -X site-packages/Pillow.libs \
-	    -X site-packages/psycopg2
 
 override_dh_virtualenv:
 	./debian/build_virtualenv

debian/synctl.1

@@ -44,7 +44,7 @@ Configuration file may be generated as follows:
 .
 .nf
 
-$ python \-m synapse\.app\.homeserver \-c config\.yaml \-\-generate\-config \-\-server\-name=<server name>
+$ python \-B \-m synapse\.app\.homeserver \-c config\.yaml \-\-generate\-config \-\-server\-name=<server name>
 .
 .fi
 .

debian/synctl.ronn

@@ -41,25 +41,24 @@ process.
 
 Configuration file may be generated as follows:
 
-    $ python -m synapse.app.homeserver -c config.yaml --generate-config --server-name=<server name>
+    $ python -B -m synapse.app.homeserver -c config.yaml --generate-config --server-name=<server name>
 
 ## ENVIRONMENT
 
   * `SYNAPSE_CACHE_FACTOR`:
-    Synapse's architecture is quite RAM hungry currently - we deliberately
-    cache a lot of recent room data and metadata in RAM in order to speed up
-    common requests. We'll improve this in the future, but for now the easiest
-    way to either reduce the RAM usage (at the risk of slowing things down)
-    is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment
-    variable. The default is 0.5, which can be decreased to reduce RAM usage
-    in memory constrained enviroments, or increased if performance starts to
-    degrade.
-
-    However, degraded performance due to a low cache factor, common on
-    machines with slow disks, often leads to explosions in memory use due
-    backlogged requests. In this case, reducing the cache factor will make
-    things worse. Instead, try increasing it drastically. 2.0 is a good
-    starting value.
+    Synapse's architecture is quite RAM hungry currently - a lot of
+    recent room data and metadata is deliberately cached in RAM in
+    order to speed up common requests.  This will be improved in
+    future, but for now the easiest way to either reduce the RAM usage
+    (at the risk of slowing things down) is to set the
+    SYNAPSE_CACHE_FACTOR environment variable. Roughly speaking, a
+    SYNAPSE_CACHE_FACTOR of 1.0 will max out at around 3-4GB of
+    resident memory - this is what we currently run the matrix.org
+    on. The default setting is currently 0.1, which is probably around
+    a ~700MB footprint. You can dial it down further to 0.02 if
+    desired, which targets roughly ~512MB. Conversely you can dial it
+    up if you need performance for lots of users and have a box with a
+    lot of RAM.
 
 ## COPYRIGHT
 

debian/templates

@@ -2,10 +2,8 @@ Template: matrix-synapse/server-name
 Type: string
 _Description: Name of the server:
  The name that this homeserver will appear as, to clients and other
- servers via federation. This is normally the public hostname of the
- server running synapse, but can be different if you set up delegation.
- Please refer to the delegation documentation in this case:
- https://github.com/matrix-org/synapse/blob/master/docs/delegate.md.
+ servers via federation. This name should match the SRV record
+ published in DNS.
 
 Template: matrix-synapse/report-stats
 Type: boolean

debian/test/.gitignore

@@ -1,2 +0,0 @@
-.vagrant
-*.log

debian/test/provision.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-#
-# provisioning script for vagrant boxes for testing the matrix-synapse debs.
-#
-# Will install the most recent matrix-synapse-py3 deb for this platform from
-# the /debs directory.
-
-set -e
-
-apt-get update
-apt-get install -y lsb-release
-
-deb=`ls /debs/matrix-synapse-py3_*+$(lsb_release -cs)*.deb | sort | tail -n1`
-
-debconf-set-selections <<EOF
-matrix-synapse matrix-synapse/report-stats boolean false
-matrix-synapse matrix-synapse/server-name string localhost:18448
-EOF
-
-dpkg -i "$deb"
-
-sed -i -e '/port: 8...$/{s/8448/18448/; s/8008/18008/}' -e '$aregistration_shared_secret: secret' /etc/matrix-synapse/homeserver.yaml
-systemctl restart matrix-synapse

debian/test/stretch/Vagrantfile

@@ -1,13 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-ver = `cd ../../..; dpkg-parsechangelog -S Version`.strip()
-
-Vagrant.configure("2") do |config|
-  config.vm.box = "debian/stretch64"
-
-  config.vm.synced_folder ".", "/vagrant", disabled: true
-  config.vm.synced_folder "../../../../debs", "/debs", type: "nfs"
-
-  config.vm.provision "shell", path: "../provision.sh"
-end

debian/test/xenial/Vagrantfile

@@ -1,10 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-Vagrant.configure("2") do |config|
-  config.vm.box = "ubuntu/xenial64"
-
-  config.vm.synced_folder ".", "/vagrant", disabled: true
-  config.vm.synced_folder "../../../../debs", "/debs"
-  config.vm.provision "shell", path: "../provision.sh"
-end

demo/.gitignore

@@ -1,7 +0,0 @@
-*.db
-*.log
-*.log.*
-*.pid
-
-/media_store.*
-/etc

demo/README

@@ -1,13 +1,9 @@
-DO NOT USE THESE DEMO SERVERS IN PRODUCTION
-
 Requires you to have done:
     python setup.py develop
 
 
-The demo start.sh will start three synapse servers on ports 8080, 8081 and 8082, with host names localhost:$port. This can be easily changed to `hostname`:$port in start.sh if required.
-
-To enable the servers to communicate untrusted ssl certs are used. In order to do this the servers do not check the certs
-and are configured in a highly insecure way. Do not use these configuration files in production.
+The demo start.sh will start three synapse servers on ports 8080, 8081 and 8082, with host names localhost:$port. This can be easily changed to `hostname`:$port in start.sh if required. 
+It will also start a web server on port 8000 pointed at the webclient.
 
 stop.sh will stop the synapse servers and the webclient.
 

demo/clean.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 set -e
 

demo/start.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
@@ -21,126 +21,23 @@ for port in 8080 8081 8082; do
     pushd demo/$port
 
     #rm $DIR/etc/$port.config
-    python3 -m synapse.app.homeserver \
+    python -m synapse.app.homeserver \
         --generate-config \
         -H "localhost:$https_port" \
         --config-path "$DIR/etc/$port.config" \
         --report-stats no
 
-    if ! grep -F "Customisation made by demo/start.sh" -q  $DIR/etc/$port.config; then
-        printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config
-
-        echo "public_baseurl: http://localhost:$port/" >> $DIR/etc/$port.config
-
-        echo 'enable_registration: true' >> $DIR/etc/$port.config
-
-        # Warning, this heredoc depends on the interaction of tabs and spaces. Please don't
-        # accidentaly bork me with your fancy settings.
-		listeners=$(cat <<-PORTLISTENERS
-		# Configure server to listen on both $https_port and $port
-		# This overides some of the default settings above
-		listeners:
-		  - port: $https_port
-		    type: http
-		    tls: true
-		    resources:
-		      - names: [client, federation]
-
-		  - port: $port
-		    tls: false
-		    bind_addresses: ['::1', '127.0.0.1']
-		    type: http
-		    x_forwarded: true
-		    resources:
-		      - names: [client, federation]
-		        compress: false
-		PORTLISTENERS
-		)
-        echo "${listeners}" >> $DIR/etc/$port.config
-
-        # Disable tls for the servers
-        printf '\n\n# Disable tls on the servers.' >> $DIR/etc/$port.config
-        echo '# DO NOT USE IN PRODUCTION' >> $DIR/etc/$port.config
-        echo 'use_insecure_ssl_client_just_for_testing_do_not_use: true' >> $DIR/etc/$port.config
-        echo 'federation_verify_certificates: false' >> $DIR/etc/$port.config
-
-        # Set tls paths
-        echo "tls_certificate_path: \"$DIR/etc/localhost:$https_port.tls.crt\"" >> $DIR/etc/$port.config
-        echo "tls_private_key_path: \"$DIR/etc/localhost:$https_port.tls.key\"" >> $DIR/etc/$port.config
-
-        # Generate tls keys
-        openssl req -x509 -newkey rsa:4096 -keyout $DIR/etc/localhost\:$https_port.tls.key -out $DIR/etc/localhost\:$https_port.tls.crt -days 365 -nodes -subj "/O=matrix"
-
-        # Ignore keys from the trusted keys server
-        echo '# Ignore keys from the trusted keys server' >> $DIR/etc/$port.config
-        echo 'trusted_key_servers:' >> $DIR/etc/$port.config
-        echo '  - server_name: "matrix.org"' >> $DIR/etc/$port.config
-        echo '    accept_keys_insecurely: true' >> $DIR/etc/$port.config
-
-        # Reduce the blacklist
-        blacklist=$(cat <<-BLACK
-		# Set the blacklist so that it doesn't include 127.0.0.1, ::1
-		federation_ip_range_blacklist:
-		  - '10.0.0.0/8'
-		  - '172.16.0.0/12'
-		  - '192.168.0.0/16'
-		  - '100.64.0.0/10'
-		  - '169.254.0.0/16'
-		  - 'fe80::/64'
-		  - 'fc00::/7'
-		BLACK
-		)
-        echo "${blacklist}" >> $DIR/etc/$port.config
-    fi
-
     # Check script parameters
     if [ $# -eq 1 ]; then
         if [ $1 = "--no-rate-limit" ]; then
-
-            # Disable any rate limiting
-            ratelimiting=$(cat <<-RC
-			rc_message:
-			  per_second: 1000
-			  burst_count: 1000
-			rc_registration:
-			  per_second: 1000
-			  burst_count: 1000
-			rc_login:
-			  address:
-			    per_second: 1000
-			    burst_count: 1000
-			  account:
-			    per_second: 1000
-			    burst_count: 1000
-			  failed_attempts:
-			    per_second: 1000
-			    burst_count: 1000
-			rc_admin_redaction:
-			  per_second: 1000
-			  burst_count: 1000
-			rc_joins:
-			  local:
-			    per_second: 1000
-			    burst_count: 1000
-			  remote:
-			    per_second: 1000
-			    burst_count: 1000
-			rc_3pid_validation:
-			  per_second: 1000
-			  burst_count: 1000
-			rc_invites:
-			  per_room:
-			    per_second: 1000
-			    burst_count: 1000
-			  per_user:
-			    per_second: 1000
-			    burst_count: 1000
-			RC
-			)
-            echo "${ratelimiting}" >> $DIR/etc/$port.config
+            # Set high limits in config file to disable rate limiting
+            perl -p -i -e 's/rc_messages_per_second.*/rc_messages_per_second: 1000/g' $DIR/etc/$port.config
+            perl -p -i -e 's/rc_message_burst_count.*/rc_message_burst_count: 1000/g' $DIR/etc/$port.config
         fi
     fi
 
+    perl -p -i -e 's/^enable_registration:.*/enable_registration: true/g' $DIR/etc/$port.config
+
     if ! grep -F "full_twisted_stacktraces" -q  $DIR/etc/$port.config; then
         echo "full_twisted_stacktraces: true" >> $DIR/etc/$port.config
     fi
@@ -148,9 +45,10 @@ for port in 8080 8081 8082; do
         echo "report_stats: false" >> $DIR/etc/$port.config
     fi
 
-    python3 -m synapse.app.homeserver \
+    python -m synapse.app.homeserver \
         --config-path "$DIR/etc/$port.config" \
         -D \
+        -vv \
 
     popd
 done

demo/stop.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 

demo/webserver.py

@@ -0,0 +1,62 @@
+import argparse
+import BaseHTTPServer
+import os
+import SimpleHTTPServer
+import cgi, logging
+
+from daemonize import Daemonize
+
+class SimpleHTTPRequestHandlerWithPOST(SimpleHTTPServer.SimpleHTTPRequestHandler):
+    UPLOAD_PATH = "upload"
+
+    """
+    Accept all post request as file upload
+    """
+    def do_POST(self):
+
+        path = os.path.join(self.UPLOAD_PATH, os.path.basename(self.path))
+        length = self.headers['content-length']
+        data = self.rfile.read(int(length))
+
+        with open(path, 'wb') as fh:
+            fh.write(data)
+
+        self.send_response(200)
+        self.send_header('Content-Type', 'application/json')
+        self.end_headers()
+
+        # Return the absolute path of the uploaded file
+        self.wfile.write('{"url":"/%s"}' % path)
+
+
+def setup():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("directory")
+    parser.add_argument("-p", "--port", dest="port", type=int, default=8080)
+    parser.add_argument('-P', "--pid-file", dest="pid", default="web.pid")
+    args = parser.parse_args()
+
+    # Get absolute path to directory to serve, as daemonize changes to '/'
+    os.chdir(args.directory)
+    dr = os.getcwd()
+
+    httpd = BaseHTTPServer.HTTPServer(
+        ('', args.port),
+        SimpleHTTPRequestHandlerWithPOST
+    )
+
+    def run():
+        os.chdir(dr)
+        httpd.serve_forever()
+
+    daemon = Daemonize(
+            app="synapse-webclient",
+            pid=args.pid,
+            action=run,
+            auto_close_fds=False,
+        )
+
+    daemon.start()
+
+if __name__ == '__main__':
+    setup()

docker/Dockerfile

@@ -1,82 +1,54 @@
-# Dockerfile to build the matrixdotorg/synapse docker images.
-#
-# To build the image, run `docker build` command from the root of the
-# synapse repository:
-#
-#    docker build -f docker/Dockerfile .
-#
-# There is an optional PYTHON_VERSION build argument which sets the
-# version of python to build against: for example:
-#
-#    docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.6 .
-#
-
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=2
 
 ###
 ### Stage 0: builder
 ###
-FROM docker.io/python:${PYTHON_VERSION}-slim as builder
+FROM docker.io/python:${PYTHON_VERSION}-alpine3.8 as builder
 
 # install the OS build deps
-RUN apt-get update && apt-get install -y \
-    build-essential \
-    libffi-dev \
-    libjpeg-dev \
-    libpq-dev \
-    libssl-dev \
-    libwebp-dev \
-    libxml++2.6-dev \
-    libxslt1-dev \
-    openssl \
-    rustc \
-    zlib1g-dev \
-    && rm -rf /var/lib/apt/lists/*
 
-# Copy just what we need to pip install
-COPY scripts /synapse/scripts/
-COPY MANIFEST.in README.rst setup.py synctl /synapse/
-COPY synapse/__init__.py /synapse/synapse/__init__.py
-COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py
+RUN apk add \
+        build-base \
+        libffi-dev \
+        libjpeg-turbo-dev \
+        libressl-dev \
+        libxslt-dev \
+        linux-headers \
+        postgresql-dev \
+        zlib-dev
 
-# To speed up rebuilds, install all of the dependencies before we copy over
-# the whole synapse project so that we this layer in the Docker cache can be
-# used while you develop on the source
+# build things which have slow build steps, before we copy synapse, so that
+# the layer can be cached.
 #
-# This is aiming at installing the `install_requires` and `extras_require` from `setup.py`
+# (we really just care about caching a wheel here, as the "pip install" below
+# will install them again.)
+
 RUN pip install --prefix="/install" --no-warn-script-location \
-    /synapse[all]
+        cryptography \
+        msgpack-python \
+        pillow \
+        pynacl
 
-# Copy over the rest of the project
-COPY synapse /synapse/synapse/
+# now install synapse and all of the python deps to /install.
 
-# Install the synapse package itself and all of its children packages.
-#
-# This is aiming at installing only the `packages=find_packages(...)` from `setup.py
-RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
+COPY . /synapse
+RUN pip install --prefix="/install" --no-warn-script-location \
+        /synapse[all]
 
 ###
 ### Stage 1: runtime
 ###
 
-FROM docker.io/python:${PYTHON_VERSION}-slim
+FROM docker.io/python:${PYTHON_VERSION}-alpine3.8
 
-LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
-LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'
-LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git'
-LABEL org.opencontainers.image.licenses='Apache-2.0'
-
-RUN apt-get update && apt-get install -y \
-    curl \
-    gosu \
-    libjpeg62-turbo \
-    libpq5 \
-    libwebp6 \
-    xmlsec1 \
-    libjemalloc2 \
-    libssl-dev \
-    openssl \
-    && rm -rf /var/lib/apt/lists/*
+RUN apk add --no-cache --virtual .runtime_deps \
+        libffi \
+        libjpeg-turbo \
+        libressl \
+        libxslt \
+        libpq \
+        zlib \
+        su-exec
 
 COPY --from=builder /install /usr/local
 COPY ./docker/start.py /start.py
@@ -84,9 +56,6 @@ COPY ./docker/conf /conf
 
 VOLUME ["/data"]
 
-EXPOSE 8008/tcp 8009/tcp 8448/tcp
+EXPOSE 8008/tcp 8448/tcp
 
 ENTRYPOINT ["/start.py"]
-
-HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
-    CMD curl -fSs http://localhost:8008/health || exit 1

docker/Dockerfile-dhvirtualenv

@@ -27,66 +27,38 @@ RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
         wget
 
 # fetch and unpack the package
-# TODO: Upgrade to 1.2.2 once xenial is dropped
-RUN mkdir /dh-virtualenv
-RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/spotify/dh-virtualenv/archive/ac6e1b1.tar.gz
-RUN tar -xv --strip-components=1 -C /dh-virtualenv -f /dh-virtualenv.tar.gz
+RUN wget -q -O /dh-virtuenv-1.1.tar.gz https://github.com/spotify/dh-virtualenv/archive/1.1.tar.gz
+RUN tar xvf /dh-virtuenv-1.1.tar.gz
 
-# install its build deps. We do another apt-cache-update here, because we might
-# be using a stale cache from docker build.
-RUN apt-get update -qq -o Acquire::Languages=none \
-    && cd /dh-virtualenv \
-    && env DEBIAN_FRONTEND=noninteractive mk-build-deps -ri -t "apt-get -y --no-install-recommends"
+# install its build deps
+RUN cd dh-virtualenv-1.1/ \
+    && env DEBIAN_FRONTEND=noninteractive mk-build-deps -ri -t "apt-get -yqq --no-install-recommends"
 
 # build it
-RUN cd /dh-virtualenv && dpkg-buildpackage -us -uc -b
+RUN cd dh-virtualenv-1.1 && dpkg-buildpackage -us -uc -b
 
 ###
 ### Stage 1
 ###
 FROM ${distro}
 
-# Get the distro we want to pull from as a dynamic build variable
-# (We need to define it in each build stage)
-ARG distro=""
-ENV distro ${distro}
-
-# Python < 3.7 assumes LANG="C" means ASCII-only and throws on printing unicode
-# http://bugs.python.org/issue19846
-ENV LANG C.UTF-8
-
 # Install the build dependencies
-#
-# NB: keep this list in sync with the list of build-deps in debian/control
-# TODO: it would be nice to do that automatically.
-# TODO: Remove the dh-systemd stanza after dropping support for Ubuntu xenial
-#       it's a transitional package on all other, more recent releases
 RUN apt-get update -qq -o Acquire::Languages=none \
     && env DEBIAN_FRONTEND=noninteractive apt-get install \
         -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
         build-essential \
         debhelper \
         devscripts \
-        libsystemd-dev \
+        dh-systemd \
         lsb-release \
-        pkg-config \
         python3-dev \
         python3-pip \
         python3-setuptools \
         python3-venv \
-        sqlite3 \
-        libpq-dev \
-        xmlsec1 \
-    && ( env DEBIAN_FRONTEND=noninteractive apt-get install \
-         -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
-         dh-systemd || true )
+        sqlite3
 
-COPY --from=builder /dh-virtualenv_1.2~dev-1_all.deb /
-
-# install dhvirtualenv. Update the apt cache again first, in case we got a
-# cached cache from docker the first time.
-RUN apt-get update -qq -o Acquire::Languages=none \
-    && apt-get install -yq /dh-virtualenv_1.2~dev-1_all.deb
+COPY --from=builder /dh-virtualenv_1.1-1_all.deb /
+RUN apt-get install -yq /dh-virtualenv_1.1-1_all.deb
 
 WORKDIR /synapse/source
 ENTRYPOINT ["bash","/synapse/source/docker/build_debian.sh"]

docker/Dockerfile-pgtests

@@ -3,10 +3,10 @@
 FROM matrixdotorg/sytest:latest
 
 # The Sytest image doesn't come with python, so install that
-RUN apt-get update && apt-get -qq install -y python3 python3-dev python3-pip
+RUN apt-get -qq install -y python python-dev python-pip
 
 # We need tox to run the tests in run_pg_tests.sh
-RUN python3 -m pip install tox
+RUN pip install tox
 
 ADD run_pg_tests.sh /pg_tests.sh
 ENTRYPOINT /pg_tests.sh

docker/Dockerfile-workers

@@ -1,23 +0,0 @@
-# Inherit from the official Synapse docker image
-FROM matrixdotorg/synapse
-
-# Install deps
-RUN apt-get update
-RUN apt-get install -y supervisor redis nginx
-
-# Remove the default nginx sites
-RUN rm /etc/nginx/sites-enabled/default
-
-# Copy Synapse worker, nginx and supervisord configuration template files
-COPY ./docker/conf-workers/* /conf/
-
-# Expose nginx listener port
-EXPOSE 8080/tcp
-
-# Volume for user-editable config files, logs etc.
-VOLUME ["/data"]
-
-# A script to read environment variables and create the necessary
-# files to run the desired worker configuration. Will start supervisord.
-COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
-ENTRYPOINT ["/configure_workers_and_start.py"]

docker/README-testing.md

@@ -1,140 +0,0 @@
-# Running tests against a dockerised Synapse
-
-It's possible to run integration tests against Synapse
-using [Complement](https://github.com/matrix-org/complement). Complement is a Matrix Spec
-compliance test suite for homeservers, and supports any homeserver docker image configured
-to listen on ports 8008/8448. This document contains instructions for building Synapse
-docker images that can be run inside Complement for testing purposes.
-
-Note that running Synapse's unit tests from within the docker image is not supported.
-
-## Testing with SQLite and single-process Synapse
-
-> Note that `scripts-dev/complement.sh` is a script that will automatically build 
-> and run an SQLite-based, single-process of Synapse against Complement.
-
-The instructions below will set up Complement testing for a single-process, 
-SQLite-based Synapse deployment.
-
-Start by building the base Synapse docker image. If you wish to run tests with the latest
-release of Synapse, instead of your current checkout, you can skip this step. From the
-root of the repository:
-
-```sh
-docker build -t matrixdotorg/synapse -f docker/Dockerfile .
-```
-
-This will build an image with the tag `matrixdotorg/synapse`.
-
-Next, build the Synapse image for Complement. You will need a local checkout 
-of Complement. Change to the root of your Complement checkout and run:
-
-```sh
-docker build -t complement-synapse -f "dockerfiles/Synapse.Dockerfile" dockerfiles
-```
-
-This will build an image with the tag `complement-synapse`, which can be handed to 
-Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to 
-[Complement's documentation](https://github.com/matrix-org/complement/#running) for 
-how to run the tests, as well as the various available command line flags.
-
-## Testing with PostgreSQL and single or multi-process Synapse
-
-The above docker image only supports running Synapse with SQLite and in a 
-single-process topology. The following instructions are used to build a Synapse image for 
-Complement that supports either single or multi-process topology with a PostgreSQL 
-database backend.
-
-As with the single-process image, build the base Synapse docker image. If you wish to run
-tests with the latest release of Synapse, instead of your current checkout, you can skip
-this step. From the root of the repository:
-
-```sh
-docker build -t matrixdotorg/synapse -f docker/Dockerfile .
-```
-
-This will build an image with the tag `matrixdotorg/synapse`.
-
-Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`. 
-Again, from the root of the repository:
-
-```sh
-docker build -t matrixdotorg/synapse-workers -f docker/Dockerfile-workers .
-```
-
-This will build an image with the tag` matrixdotorg/synapse-workers`.
-
-It's worth noting at this point that this image is fully functional, and 
-can be used for testing against locally. See instructions for using the container 
-under
-[Running the Dockerfile-worker image standalone](#running-the-dockerfile-worker-image-standalone)
-below.
-
-Finally, build the Synapse image for Complement, which is based on
-`matrixdotorg/synapse-workers`. You will need a local checkout of Complement. Change to
-the root of your Complement checkout and run:
-
-```sh
-docker build -t matrixdotorg/complement-synapse-workers -f dockerfiles/SynapseWorkers.Dockerfile dockerfiles
-```
-
-This will build an image with the tag `complement-synapse`, which can be handed to
-Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
-[Complement's documentation](https://github.com/matrix-org/complement/#running) for
-how to run the tests, as well as the various available command line flags.
-
-## Running the Dockerfile-worker image standalone
-
-For manual testing of a multi-process Synapse instance in Docker,
-[Dockerfile-workers](Dockerfile-workers) is a Dockerfile that will produce an image
-bundling all necessary components together for a workerised homeserver instance.
-
-This includes any desired Synapse worker processes, a nginx to route traffic accordingly,
-a redis for worker communication and a supervisord instance to start up and monitor all
-processes. You will need to provide your own postgres container to connect to, and TLS 
-is not handled by the container.
-
-Once you've built the image using the above instructions, you can run it. Be sure 
-you've set up a volume according to the [usual Synapse docker instructions](README.md).
-Then run something along the lines of:
-
-```
-docker run -d --name synapse \
-    --mount type=volume,src=synapse-data,dst=/data \
-    -p 8008:8008 \
-    -e SYNAPSE_SERVER_NAME=my.matrix.host \
-    -e SYNAPSE_REPORT_STATS=no \
-    -e POSTGRES_HOST=postgres \
-    -e POSTGRES_USER=postgres \
-    -e POSTGRES_PASSWORD=somesecret \
-    -e SYNAPSE_WORKER_TYPES=synchrotron,media_repository,user_dir \
-    -e SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1 \
-    matrixdotorg/synapse-workers
-```
-
-...substituting `POSTGRES*` variables for those that match a postgres host you have 
-available (usually a running postgres docker container).
-
-The `SYNAPSE_WORKER_TYPES` environment variable is a comma-separated list of workers to
-use when running the container. All possible worker names are defined by the keys of the
-`WORKERS_CONFIG` variable in [this script](configure_workers_and_start.py), which the
-Dockerfile makes use of to generate appropriate worker, nginx and supervisord config
-files.
-
-Sharding is supported for a subset of workers, in line with the
-[worker documentation](../docs/workers.md). To run multiple instances of a given worker
-type, simply specify the type multiple times in `SYNAPSE_WORKER_TYPES`
-(e.g `SYNAPSE_WORKER_TYPES=event_creator,event_creator...`).
-
-Otherwise, `SYNAPSE_WORKER_TYPES` can either be left empty or unset to spawn no workers
-(leaving only the main process). The container is configured to use redis-based worker
-mode.
-
-Logs for workers and the main process are logged to stdout and can be viewed with 
-standard `docker logs` tooling. Worker logs contain their worker name 
-after the timestamp.
-
-Setting `SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1` will cause worker logs to be written to
-`<data_dir>/logs/<worker_name>.log`. Logs are kept for 1 week and rotate every day at 00:
-00, according to the container's clock. Logging for the main process must still be 
-configured by modifying the homeserver's log config in your Synapse data volume.

docker/README.md

@@ -1,229 +1,125 @@
 # Synapse Docker
 
-This Docker image will run Synapse as a single process. By default it uses a
-sqlite database; for production use you should connect it to a separate
-postgres database. The image also does *not* provide a TURN server.
+This Docker image will run Synapse as a single process. It does not provide a database
+server or a TURN server, you should run these separately.
 
-This image should work on all platforms that are supported by Docker upstream.
-Note that Docker's WS1-backend Linux Containers on Windows
-platform is [experimental](https://github.com/docker/for-win/issues/6470) and
-is not supported by this image.
+## Run
+
+We do not currently offer a `latest` image, as this has somewhat undefined semantics.
+We instead release only tagged versions so upgrading between releases is entirely
+within your control.
+
+### Using docker-compose (easier)
+
+This image is designed to run either with an automatically generated configuration
+file or with a custom configuration that requires manual editing.
+
+An easy way to make use of this image is via docker-compose. See the
+[contrib/docker](../contrib/docker)
+section of the synapse project for examples.
+
+### Without Compose (harder)
+
+If you do not wish to use Compose, you may still run this image using plain
+Docker commands. Note that the following is just a guideline and you may need
+to add parameters to the docker run command to account for the network situation
+with your postgres database.
+
+```
+docker run \
+    -d \
+    --name synapse \
+    -v ${DATA_PATH}:/data \
+    -e SYNAPSE_SERVER_NAME=my.matrix.host \
+    -e SYNAPSE_REPORT_STATS=yes \
+    docker.io/matrixdotorg/synapse:latest
+```
 
 ## Volumes
 
-By default, the image expects a single volume, located at `/data`, that will hold:
+The image expects a single volume, located at ``/data``, that will hold:
 
-* configuration files;
+* temporary files during uploads;
 * uploaded media and thumbnails;
 * the SQLite database if you do not configure postgres;
 * the appservices configuration.
 
 You are free to use separate volumes depending on storage endpoints at your
-disposal. For instance, `/data/media` could be stored on a large but low
+disposal. For instance, ``/data/media`` coud be stored on a large but low
 performance hdd storage while other files could be stored on high performance
 endpoints.
 
-In order to setup an application service, simply create an `appservices`
+In order to setup an application service, simply create an ``appservices``
 directory in the data volume and write the application service Yaml
 configuration file there. Multiple application services are supported.
 
-## Generating a configuration file
+## Environment
 
-The first step is to generate a valid config file. To do this, you can run the
-image with the `generate` command line option.
+Unless you specify a custom path for the configuration file, a very generic
+file will be generated, based on the following environment settings.
+These are a good starting point for setting up your own deployment.
 
-You will need to specify values for the `SYNAPSE_SERVER_NAME` and
-`SYNAPSE_REPORT_STATS` environment variable, and mount a docker volume to store
-the configuration on. For example:
+Global settings:
+
+* ``UID``, the user id Synapse will run as [default 991]
+* ``GID``, the group id Synapse will run as [default 991]
+* ``SYNAPSE_CONFIG_PATH``, path to a custom config file
+
+If ``SYNAPSE_CONFIG_PATH`` is set, you should generate a configuration file
+then customize it manually. No other environment variable is required.
+
+Otherwise, a dynamic configuration file will be used. The following environment
+variables are available for configuration:
+
+* ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname.
+* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous
+  statistics reporting back to the Matrix project which helps us to get funding.
+* ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if
+  you run your own TLS-capable reverse proxy).
+* ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on
+  the Synapse instance.
+* ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server.
+* ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`].
+* ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`].
+* ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public
+  key in order to enable recaptcha upon registration.
+* ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private
+  key in order to enable recaptcha upon registration.
+* ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN
+  uris to enable TURN for this homeserver.
+* ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required.
+* ``SYNAPSE_MAX_UPLOAD_SIZE``, set this variable to change the max upload size [default `10M`].
+
+Shared secrets, that will be initialized to random values if not set:
+
+* ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if
+  registration is disable.
+* ``SYNAPSE_MACAROON_SECRET_KEY`` secret for signing access tokens
+  to the server.
+
+Database specific values (will use SQLite if not set):
+
+* `POSTGRES_DB` - The database name for the synapse postgres database. [default: `synapse`]
+* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`]
+* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy.
+* `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`]
+
+Mail server specific values (will not send emails if not set):
+
+* ``SYNAPSE_SMTP_HOST``, hostname to the mail server.
+* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default ``25``].
+* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if any.
+* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail server if any.
+
+## Build
+
+Build the docker image with the `docker build` command from the root of the synapse repository.
 
 ```
-docker run -it --rm \
-    --mount type=volume,src=synapse-data,dst=/data \
-    -e SYNAPSE_SERVER_NAME=my.matrix.host \
-    -e SYNAPSE_REPORT_STATS=yes \
-    matrixdotorg/synapse:latest generate
+docker build -t docker.io/matrixdotorg/synapse . -f docker/Dockerfile
 ```
 
-For information on picking a suitable server name, see
-https://github.com/matrix-org/synapse/blob/master/INSTALL.md.
+The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:<version>` where `<version>` is the same as the release tag in the synapse git repository.
 
-The above command will generate a `homeserver.yaml` in (typically)
-`/var/lib/docker/volumes/synapse-data/_data`. You should check this file, and
-customise it to your needs.
-
-The following environment variables are supported in `generate` mode:
-
-* `SYNAPSE_SERVER_NAME` (mandatory): the server public hostname.
-* `SYNAPSE_REPORT_STATS` (mandatory, `yes` or `no`): whether to enable
-  anonymous statistics reporting.
-* `SYNAPSE_HTTP_PORT`: the port Synapse should listen on for http traffic.
-      Defaults to `8008`.
-* `SYNAPSE_CONFIG_DIR`: where additional config files (such as the log config
-  and event signing key) will be stored. Defaults to `/data`.
-* `SYNAPSE_CONFIG_PATH`: path to the file to be generated. Defaults to
-  `<SYNAPSE_CONFIG_DIR>/homeserver.yaml`.
-* `SYNAPSE_DATA_DIR`: where the generated config will put persistent data
-  such as the database and media store. Defaults to `/data`.
-* `UID`, `GID`: the user id and group id to use for creating the data
-  directories. Defaults to `991`, `991`.
-
-## Running synapse
-
-Once you have a valid configuration file, you can start synapse as follows:
-
-```
-docker run -d --name synapse \
-    --mount type=volume,src=synapse-data,dst=/data \
-    -p 8008:8008 \
-    matrixdotorg/synapse:latest
-```
-
-(assuming 8008 is the port Synapse is configured to listen on for http traffic.)
-
-You can then check that it has started correctly with:
-
-```
-docker logs synapse
-```
-
-If all is well, you should now be able to connect to http://localhost:8008 and
-see a confirmation message.
-
-The following environment variables are supported in `run` mode:
-
-* `SYNAPSE_CONFIG_DIR`: where additional config files are stored. Defaults to
-  `/data`.
-* `SYNAPSE_CONFIG_PATH`: path to the config file. Defaults to
-  `<SYNAPSE_CONFIG_DIR>/homeserver.yaml`.
-* `SYNAPSE_WORKER`: module to execute, used when running synapse with workers.
-   Defaults to `synapse.app.homeserver`, which is suitable for non-worker mode.
-* `UID`, `GID`: the user and group id to run Synapse as. Defaults to `991`, `991`.
-* `TZ`: the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) the container will run with. Defaults to `UTC`.
-
-For more complex setups (e.g. for workers) you can also pass your args directly to synapse using `run` mode. For example like this:
-
-```
-docker run -d --name synapse \
-    --mount type=volume,src=synapse-data,dst=/data \
-    -p 8008:8008 \
-    matrixdotorg/synapse:latest run \
-    -m synapse.app.generic_worker \
-    --config-path=/data/homeserver.yaml \
-    --config-path=/data/generic_worker.yaml
-```
-
-If you do not provide `-m`, the value of the `SYNAPSE_WORKER` environment variable is used. If you do not provide at least one `--config-path` or `-c`, the value of the `SYNAPSE_CONFIG_PATH` environment variable is used instead.
-
-## Generating an (admin) user
-
-After synapse is running, you may wish to create a user via `register_new_matrix_user`.
-
-This requires a `registration_shared_secret` to be set in your config file. Synapse
-must be restarted to pick up this change.
-
-You can then call the script:
-
-```
-docker exec -it synapse register_new_matrix_user http://localhost:8008 -c /data/homeserver.yaml --help
-```
-
-Remember to remove the `registration_shared_secret` and restart if you no-longer need it.
-
-## TLS support
-
-The default configuration exposes a single HTTP port: http://localhost:8008. It
-is suitable for local testing, but for any practical use, you will either need
-to use a reverse proxy, or configure Synapse to expose an HTTPS port.
-
-For documentation on using a reverse proxy, see
-https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md.
-
-For more information on enabling TLS support in synapse itself, see
-https://github.com/matrix-org/synapse/blob/master/INSTALL.md#tls-certificates. Of
-course, you will need to expose the TLS port from the container with a `-p`
-argument to `docker run`.
-
-## Legacy dynamic configuration file support
-
-The docker image used to support creating a dynamic configuration file based
-on environment variables. This is no longer supported, and an error will be
-raised if you try to run synapse without a config file.
-
-It is, however, possible to generate a static configuration file based on
-the environment variables that were previously used. To do this, run the docker
-container once with the environment variables set, and `migrate_config`
-command line option. For example:
-
-```
-docker run -it --rm \
-    --mount type=volume,src=synapse-data,dst=/data \
-    -e SYNAPSE_SERVER_NAME=my.matrix.host \
-    -e SYNAPSE_REPORT_STATS=yes \
-    matrixdotorg/synapse:latest migrate_config
-```
-
-This will generate the same configuration file as the legacy mode used, and
-will store it in `/data/homeserver.yaml`. You can then use it as shown above at
-[Running synapse](#running-synapse).
-
-Note that the defaults used in this configuration file may be different to
-those when generating a new config file with `generate`: for example, TLS is
-enabled by default in this mode. You are encouraged to inspect the generated
-configuration file and edit it to ensure it meets your needs.
-
-## Building the image
-
-If you need to build the image from a Synapse checkout, use the following `docker
- build` command from the repo's root:
-
-```
-docker build -t matrixdotorg/synapse -f docker/Dockerfile .
-```
-
-You can choose to build a different docker image by changing the value of the `-f` flag to
-point to another Dockerfile.
-
-## Disabling the healthcheck
-
-If you are using a non-standard port or tls inside docker you can disable the healthcheck
-whilst running the above `docker run` commands. 
-
-```
-   --no-healthcheck
-```
-
-## Disabling the healthcheck in docker-compose file
-
-If you wish to disable the healthcheck via docker-compose, append the following to your service configuration.
-
-```
-  healthcheck:
-    disable: true
-```
-
-## Setting custom healthcheck on docker run
-
-If you wish to point the healthcheck at a different port with docker command, add the following
-
-```
-  --health-cmd 'curl -fSs http://localhost:1234/health'
-```
-
-## Setting the healthcheck in docker-compose file
-
-You can add the following to set a custom healthcheck in a docker compose file.
-You will need docker-compose version >2.1 for this to work. 
-
-```
-healthcheck:
-  test: ["CMD", "curl", "-fSs", "http://localhost:8008/health"]
-  interval: 15s
-  timeout: 5s
-  retries: 3
-  start_period: 5s
-```
-
-## Using jemalloc
-
-Jemalloc is embedded in the image and will be used instead of the default allocator.
-You can read about jemalloc by reading the Synapse [README](../README.md).
+You may have a local Python wheel cache available, in which case copy the relevant
+packages in the ``cache/`` directory at the root of the project.

docker/build_debian.sh

@@ -1,11 +1,10 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # The script to build the Debian package, as ran inside the Docker image.
 
 set -ex
 
-# Get the codename from distro env
-DIST=`cut -d ':' -f2 <<< $distro`
+DIST=`lsb_release -c -s`
 
 # we get a read-only copy of the source: make a writeable copy
 cp -aT /synapse/source /synapse/build

docker/conf-workers/nginx.conf.j2

@@ -1,27 +0,0 @@
-# This file contains the base config for the reverse proxy, as part of ../Dockerfile-workers.
-# configure_workers_and_start.py uses and amends to this file depending on the workers
-# that have been selected.
-
-{{ upstream_directives }}
-
-server {
-    # Listen on an unoccupied port number
-    listen 8008;
-    listen [::]:8008;
-
-    server_name localhost;
-
-    # Nginx by default only allows file uploads up to 1M in size
-    # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
-    client_max_body_size 100M;
-
-{{ worker_locations }}
-
-    # Send all other traffic to the main process
-    location ~* ^(\\/_matrix|\\/_synapse) {
-        proxy_pass http://localhost:8080;
-        proxy_set_header X-Forwarded-For $remote_addr;
-        proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header Host $host;
-    }
-}

docker/conf-workers/shared.yaml.j2

@@ -1,9 +0,0 @@
-# This file contains the base for the shared homeserver config file between Synapse workers,
-# as part of ./Dockerfile-workers.
-# configure_workers_and_start.py uses and amends to this file depending on the workers
-# that have been selected.
-
-redis:
-    enabled: true
-
-{{ shared_worker_config }}

docker/conf-workers/supervisord.conf.j2

@@ -1,41 +0,0 @@
-# This file contains the base config for supervisord, as part of ../Dockerfile-workers.
-# configure_workers_and_start.py uses and amends to this file depending on the workers
-# that have been selected.
-[supervisord]
-nodaemon=true
-user=root
-
-[program:nginx]
-command=/usr/sbin/nginx -g "daemon off;"
-priority=500
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-username=www-data
-autorestart=true
-
-[program:redis]
-command=/usr/bin/redis-server /etc/redis/redis.conf --daemonize no
-priority=1
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-username=redis
-autorestart=true
-
-[program:synapse_main]
-command=/usr/local/bin/python -m synapse.app.homeserver --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml
-priority=10
-# Log startup failures to supervisord's stdout/err
-# Regular synapse logs will still go in the configured data directory
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-autorestart=unexpected
-exitcodes=0
-
-# Additional process blocks
-{{ worker_config }}

docker/conf-workers/worker.yaml.j2

@@ -1,26 +0,0 @@
-# This is a configuration template for a single worker instance, and is
-# used by Dockerfile-workers.
-# Values will be change depending on whichever workers are selected when
-# running that image.
-
-worker_app: "{{ app }}"
-worker_name: "{{ name }}"
-
-# The replication listener on the main synapse process.
-worker_replication_host: 127.0.0.1
-worker_replication_http_port: 9093
-
-worker_listeners:
-  - type: http
-    port: {{ port }}
-{% if listener_resources %}
-    resources:
-      - names:
-{%- for resource in listener_resources %}
-        - {{ resource }}
-{%- endfor %}
-{% endif %}
-
-worker_log_config: {{ worker_log_config_filepath }}
-
-{{ worker_extra_conf }}

docker/conf/homeserver.yaml

@@ -2,18 +2,10 @@
 
 ## TLS ##
 
-{% if not SYNAPSE_NO_TLS %}
-
 tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt"
 tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key"
-
-{% if SYNAPSE_ACME %}
-acme:
-    enabled: true
-    port: 8009
-{% endif %}
-
-{% endif %}
+no_tls: {{ "True" if SYNAPSE_NO_TLS else "False" }}
+tls_fingerprints: []
 
 ## Server ##
 
@@ -21,7 +13,7 @@ server_name: "{{ SYNAPSE_SERVER_NAME }}"
 pid_file: /homeserver.pid
 web_client: False
 soft_file_limit: 0
-log_config: "{{ SYNAPSE_LOG_CONFIG }}"
+log_config: "/compiled/log.config"
 
 ## Ports ##
 
@@ -40,9 +32,7 @@ listeners:
         compress: false
   {% endif %}
 
-  # Allow configuring in case we want to reverse proxy 8008
-  # using another process in the same container
-  - port: {{ SYNAPSE_HTTP_PORT or 8008 }}
+  - port: 8008
     tls: false
     bind_addresses: ['::']
     type: http
@@ -91,7 +81,8 @@ federation_rc_concurrent: 3
 ## Files ##
 
 media_store_path: "/data/media"
-max_upload_size: "{{ SYNAPSE_MAX_UPLOAD_SIZE or "50M" }}"
+uploads_path: "/data/uploads"
+max_upload_size: "{{ SYNAPSE_MAX_UPLOAD_SIZE or "10M" }}"
 max_image_pixels: "32M"
 dynamic_thumbnails: false
 
@@ -175,10 +166,18 @@ report_stats: False
 
 ## API Configuration ##
 
+room_invite_state_types:
+    - "m.room.join_rules"
+    - "m.room.canonical_alias"
+    - "m.room.avatar"
+    - "m.room.name"
+
 {% if SYNAPSE_APPSERVICES %}
 app_service_config_files:
 {% for appservice in SYNAPSE_APPSERVICES %}    - "{{ appservice }}"
 {% endfor %}
+{% else %}
+app_service_config_files: []
 {% endif %}
 
 macaroon_secret_key: "{{ SYNAPSE_MACAROON_SECRET_KEY }}"
@@ -191,10 +190,31 @@ old_signing_keys: {}
 key_refresh_interval: "1d" # 1 Day.
 
 # The trusted servers to download signing keys from.
-trusted_key_servers:
-  - server_name: matrix.org
-    verify_keys:
-      "ed25519:auto": "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"
+perspectives:
+  servers:
+    "matrix.org":
+      verify_keys:
+        "ed25519:auto":
+          key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"
 
 password_config:
    enabled: true
+
+{% if SYNAPSE_SMTP_HOST %}
+email:
+   enable_notifs: false
+   smtp_host: "{{ SYNAPSE_SMTP_HOST }}"
+   smtp_port: {{ SYNAPSE_SMTP_PORT or "25" }}
+   smtp_user: "{{ SYNAPSE_SMTP_USER }}"
+   smtp_pass: "{{ SYNAPSE_SMTP_PASSWORD }}"
+   require_transport_security: False
+   notif_from: "{{ SYNAPSE_SMTP_FROM or "hostmaster@" + SYNAPSE_SERVER_NAME }}"
+   app_name: Matrix
+   # if template_dir is unset, uses the example templates that are part of
+   # the Synapse distribution.
+   #template_dir: res/templates
+   notif_template_html: notif_mail.html
+   notif_template_text: notif_mail.txt
+   notif_for_new_users: True
+   riot_base_url: "https://{{ SYNAPSE_SERVER_NAME }}"
+{% endif %}

docker/conf/log.config

@@ -2,51 +2,28 @@ version: 1
 
 formatters:
   precise:
-{% if worker_name %}
-    format: '%(asctime)s - worker:{{ worker_name }} - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
-{% else %}
-    format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
-{% endif %}
+   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s'
+
+filters:
+  context:
+    (): synapse.util.logcontext.LoggingContextFilter
+    request: ""
 
 handlers:
-  file:
-    class: logging.handlers.TimedRotatingFileHandler
-    formatter: precise
-    filename: {{ LOG_FILE_PATH or "homeserver.log" }}
-    when: "midnight"
-    backupCount: 6  # Does not include the current log file.
-    encoding: utf8
-
-  # Default to buffering writes to log file for efficiency. This means that
-  # there will be a delay for INFO/DEBUG logs to get written, but WARNING/ERROR
-  # logs will still be flushed immediately.
-  buffer:
-    class: logging.handlers.MemoryHandler
-    target: file
-    # The capacity is the number of log lines that are buffered before
-    # being written to disk. Increasing this will lead to better
-    # performance, at the expensive of it taking longer for log lines to
-    # be written to disk.
-    capacity: 10
-    flushLevel: 30  # Flush for WARNING logs as well
-
   console:
     class: logging.StreamHandler
     formatter: precise
+    filters: [context]
 
 loggers:
+    synapse:
+        level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }}
+
     synapse.storage.SQL:
         # beware: increasing this to DEBUG will make synapse log sensitive
         # information such as access tokens.
-        level: INFO
+        level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }}
 
 root:
-    level: {{ SYNAPSE_LOG_LEVEL or "INFO" }}
-
-{% if LOG_FILE_PATH %}
-    handlers: [console, buffer]
-{% else %}
+    level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }}
     handlers: [console]
-{% endif %}
-
-disable_existing_loggers: false

docker/configure_workers_and_start.py

@@ -1,558 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2021 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script reads environment variables and generates a shared Synapse worker,
-# nginx and supervisord configs depending on the workers requested.
-#
-# The environment variables it reads are:
-#   * SYNAPSE_SERVER_NAME: The desired server_name of the homeserver.
-#   * SYNAPSE_REPORT_STATS: Whether to report stats.
-#   * SYNAPSE_WORKER_TYPES: A comma separated list of worker names as specified in WORKER_CONFIG
-#         below. Leave empty for no workers, or set to '*' for all possible workers.
-#
-# NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined
-# in the project's README), this script may be run multiple times, and functionality should
-# continue to work if so.
-
-import os
-import subprocess
-import sys
-
-import jinja2
-import yaml
-
-MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
-
-
-WORKERS_CONFIG = {
-    "pusher": {
-        "app": "synapse.app.pusher",
-        "listener_resources": [],
-        "endpoint_patterns": [],
-        "shared_extra_conf": {"start_pushers": False},
-        "worker_extra_conf": "",
-    },
-    "user_dir": {
-        "app": "synapse.app.user_dir",
-        "listener_resources": ["client"],
-        "endpoint_patterns": [
-            "^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$"
-        ],
-        "shared_extra_conf": {"update_user_directory": False},
-        "worker_extra_conf": "",
-    },
-    "media_repository": {
-        "app": "synapse.app.media_repository",
-        "listener_resources": ["media"],
-        "endpoint_patterns": [
-            "^/_matrix/media/",
-            "^/_synapse/admin/v1/purge_media_cache$",
-            "^/_synapse/admin/v1/room/.*/media.*$",
-            "^/_synapse/admin/v1/user/.*/media.*$",
-            "^/_synapse/admin/v1/media/.*$",
-            "^/_synapse/admin/v1/quarantine_media/.*$",
-        ],
-        "shared_extra_conf": {"enable_media_repo": False},
-        "worker_extra_conf": "enable_media_repo: true",
-    },
-    "appservice": {
-        "app": "synapse.app.appservice",
-        "listener_resources": [],
-        "endpoint_patterns": [],
-        "shared_extra_conf": {"notify_appservices": False},
-        "worker_extra_conf": "",
-    },
-    "federation_sender": {
-        "app": "synapse.app.federation_sender",
-        "listener_resources": [],
-        "endpoint_patterns": [],
-        "shared_extra_conf": {"send_federation": False},
-        "worker_extra_conf": "",
-    },
-    "synchrotron": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": ["client"],
-        "endpoint_patterns": [
-            "^/_matrix/client/(v2_alpha|r0)/sync$",
-            "^/_matrix/client/(api/v1|v2_alpha|r0)/events$",
-            "^/_matrix/client/(api/v1|r0)/initialSync$",
-            "^/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync$",
-        ],
-        "shared_extra_conf": {},
-        "worker_extra_conf": "",
-    },
-    "federation_reader": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": ["federation"],
-        "endpoint_patterns": [
-            "^/_matrix/federation/(v1|v2)/event/",
-            "^/_matrix/federation/(v1|v2)/state/",
-            "^/_matrix/federation/(v1|v2)/state_ids/",
-            "^/_matrix/federation/(v1|v2)/backfill/",
-            "^/_matrix/federation/(v1|v2)/get_missing_events/",
-            "^/_matrix/federation/(v1|v2)/publicRooms",
-            "^/_matrix/federation/(v1|v2)/query/",
-            "^/_matrix/federation/(v1|v2)/make_join/",
-            "^/_matrix/federation/(v1|v2)/make_leave/",
-            "^/_matrix/federation/(v1|v2)/send_join/",
-            "^/_matrix/federation/(v1|v2)/send_leave/",
-            "^/_matrix/federation/(v1|v2)/invite/",
-            "^/_matrix/federation/(v1|v2)/query_auth/",
-            "^/_matrix/federation/(v1|v2)/event_auth/",
-            "^/_matrix/federation/(v1|v2)/exchange_third_party_invite/",
-            "^/_matrix/federation/(v1|v2)/user/devices/",
-            "^/_matrix/federation/(v1|v2)/get_groups_publicised$",
-            "^/_matrix/key/v2/query",
-        ],
-        "shared_extra_conf": {},
-        "worker_extra_conf": "",
-    },
-    "federation_inbound": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": ["federation"],
-        "endpoint_patterns": ["/_matrix/federation/(v1|v2)/send/"],
-        "shared_extra_conf": {},
-        "worker_extra_conf": "",
-    },
-    "event_persister": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": ["replication"],
-        "endpoint_patterns": [],
-        "shared_extra_conf": {},
-        "worker_extra_conf": "",
-    },
-    "background_worker": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": [],
-        "endpoint_patterns": [],
-        # This worker cannot be sharded. Therefore there should only ever be one background
-        # worker, and it should be named background_worker1
-        "shared_extra_conf": {"run_background_tasks_on": "background_worker1"},
-        "worker_extra_conf": "",
-    },
-    "event_creator": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": ["client"],
-        "endpoint_patterns": [
-            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/redact",
-            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send",
-            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$",
-            "^/_matrix/client/(api/v1|r0|unstable)/join/",
-            "^/_matrix/client/(api/v1|r0|unstable)/profile/",
-        ],
-        "shared_extra_conf": {},
-        "worker_extra_conf": "",
-    },
-    "frontend_proxy": {
-        "app": "synapse.app.frontend_proxy",
-        "listener_resources": ["client", "replication"],
-        "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|unstable)/keys/upload"],
-        "shared_extra_conf": {},
-        "worker_extra_conf": (
-            "worker_main_http_uri: http://127.0.0.1:%d"
-            % (MAIN_PROCESS_HTTP_LISTENER_PORT,),
-        ),
-    },
-}
-
-# Templates for sections that may be inserted multiple times in config files
-SUPERVISORD_PROCESS_CONFIG_BLOCK = """
-[program:synapse_{name}]
-command=/usr/local/bin/python -m {app} \
-    --config-path="{config_path}" \
-    --config-path=/conf/workers/shared.yaml \
-    --config-path=/conf/workers/{name}.yaml
-autorestart=unexpected
-priority=500
-exitcodes=0
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-"""
-
-NGINX_LOCATION_CONFIG_BLOCK = """
-    location ~* {endpoint} {
-        proxy_pass {upstream};
-        proxy_set_header X-Forwarded-For $remote_addr;
-        proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header Host $host;
-    }
-"""
-
-NGINX_UPSTREAM_CONFIG_BLOCK = """
-upstream {upstream_worker_type} {
-{body}
-}
-"""
-
-
-# Utility functions
-def log(txt: str):
-    """Log something to the stdout.
-
-    Args:
-        txt: The text to log.
-    """
-    print(txt)
-
-
-def error(txt: str):
-    """Log something and exit with an error code.
-
-    Args:
-        txt: The text to log in error.
-    """
-    log(txt)
-    sys.exit(2)
-
-
-def convert(src: str, dst: str, **template_vars):
-    """Generate a file from a template
-
-    Args:
-        src: Path to the input file.
-        dst: Path to write to.
-        template_vars: The arguments to replace placeholder variables in the template with.
-    """
-    # Read the template file
-    with open(src) as infile:
-        template = infile.read()
-
-    # Generate a string from the template. We disable autoescape to prevent template
-    # variables from being escaped.
-    rendered = jinja2.Template(template, autoescape=False).render(**template_vars)
-
-    # Write the generated contents to a file
-    #
-    # We use append mode in case the files have already been written to by something else
-    # (for instance, as part of the instructions in a dockerfile).
-    with open(dst, "a") as outfile:
-        # In case the existing file doesn't end with a newline
-        outfile.write("\n")
-
-        outfile.write(rendered)
-
-
-def add_sharding_to_shared_config(
-    shared_config: dict,
-    worker_type: str,
-    worker_name: str,
-    worker_port: int,
-) -> None:
-    """Given a dictionary representing a config file shared across all workers,
-    append sharded worker information to it for the current worker_type instance.
-
-    Args:
-        shared_config: The config dict that all worker instances share (after being converted to YAML)
-        worker_type: The type of worker (one of those defined in WORKERS_CONFIG).
-        worker_name: The name of the worker instance.
-        worker_port: The HTTP replication port that the worker instance is listening on.
-    """
-    # The instance_map config field marks the workers that write to various replication streams
-    instance_map = shared_config.setdefault("instance_map", {})
-
-    # Worker-type specific sharding config
-    if worker_type == "pusher":
-        shared_config.setdefault("pusher_instances", []).append(worker_name)
-
-    elif worker_type == "federation_sender":
-        shared_config.setdefault("federation_sender_instances", []).append(worker_name)
-
-    elif worker_type == "event_persister":
-        # Event persisters write to the events stream, so we need to update
-        # the list of event stream writers
-        shared_config.setdefault("stream_writers", {}).setdefault("events", []).append(
-            worker_name
-        )
-
-        # Map of stream writer instance names to host/ports combos
-        instance_map[worker_name] = {
-            "host": "localhost",
-            "port": worker_port,
-        }
-
-    elif worker_type == "media_repository":
-        # The first configured media worker will run the media background jobs
-        shared_config.setdefault("media_instance_running_background_jobs", worker_name)
-
-
-def generate_base_homeserver_config():
-    """Starts Synapse and generates a basic homeserver config, which will later be
-    modified for worker support.
-
-    Raises: CalledProcessError if calling start.py returned a non-zero exit code.
-    """
-    # start.py already does this for us, so just call that.
-    # note that this script is copied in in the official, monolith dockerfile
-    os.environ["SYNAPSE_HTTP_PORT"] = str(MAIN_PROCESS_HTTP_LISTENER_PORT)
-    subprocess.check_output(["/usr/local/bin/python", "/start.py", "migrate_config"])
-
-
-def generate_worker_files(environ, config_path: str, data_dir: str):
-    """Read the desired list of workers from environment variables and generate
-    shared homeserver, nginx and supervisord configs.
-
-    Args:
-        environ: _Environ[str]
-        config_path: Where to output the generated Synapse main worker config file.
-        data_dir: The location of the synapse data directory. Where log and
-            user-facing config files live.
-    """
-    # Note that yaml cares about indentation, so care should be taken to insert lines
-    # into files at the correct indentation below.
-
-    # shared_config is the contents of a Synapse config file that will be shared amongst
-    # the main Synapse process as well as all workers.
-    # It is intended mainly for disabling functionality when certain workers are spun up,
-    # and adding a replication listener.
-
-    # First read the original config file and extract the listeners block. Then we'll add
-    # another listener for replication. Later we'll write out the result.
-    listeners = [
-        {
-            "port": 9093,
-            "bind_address": "127.0.0.1",
-            "type": "http",
-            "resources": [{"names": ["replication"]}],
-        }
-    ]
-    with open(config_path) as file_stream:
-        original_config = yaml.safe_load(file_stream)
-        original_listeners = original_config.get("listeners")
-        if original_listeners:
-            listeners += original_listeners
-
-    # The shared homeserver config. The contents of which will be inserted into the
-    # base shared worker jinja2 template.
-    #
-    # This config file will be passed to all workers, included Synapse's main process.
-    shared_config = {"listeners": listeners}
-
-    # The supervisord config. The contents of which will be inserted into the
-    # base supervisord jinja2 template.
-    #
-    # Supervisord will be in charge of running everything, from redis to nginx to Synapse
-    # and all of its worker processes. Load the config template, which defines a few
-    # services that are necessary to run.
-    supervisord_config = ""
-
-    # Upstreams for load-balancing purposes. This dict takes the form of a worker type to the
-    # ports of each worker. For example:
-    # {
-    #   worker_type: {1234, 1235, ...}}
-    # }
-    # and will be used to construct 'upstream' nginx directives.
-    nginx_upstreams = {}
-
-    # A map of: {"endpoint": "upstream"}, where "upstream" is a str representing what will be
-    # placed after the proxy_pass directive. The main benefit to representing this data as a
-    # dict over a str is that we can easily deduplicate endpoints across multiple instances
-    # of the same worker.
-    #
-    # An nginx site config that will be amended to depending on the workers that are
-    # spun up. To be placed in /etc/nginx/conf.d.
-    nginx_locations = {}
-
-    # Read the desired worker configuration from the environment
-    worker_types = environ.get("SYNAPSE_WORKER_TYPES")
-    if worker_types is None:
-        # No workers, just the main process
-        worker_types = []
-    else:
-        # Split type names by comma
-        worker_types = worker_types.split(",")
-
-    # Create the worker configuration directory if it doesn't already exist
-    os.makedirs("/conf/workers", exist_ok=True)
-
-    # Start worker ports from this arbitrary port
-    worker_port = 18009
-
-    # A counter of worker_type -> int. Used for determining the name for a given
-    # worker type when generating its config file, as each worker's name is just
-    # worker_type + instance #
-    worker_type_counter = {}
-
-    # For each worker type specified by the user, create config values
-    for worker_type in worker_types:
-        worker_type = worker_type.strip()
-
-        worker_config = WORKERS_CONFIG.get(worker_type)
-        if worker_config:
-            worker_config = worker_config.copy()
-        else:
-            log(worker_type + " is an unknown worker type! It will be ignored")
-            continue
-
-        new_worker_count = worker_type_counter.setdefault(worker_type, 0) + 1
-        worker_type_counter[worker_type] = new_worker_count
-
-        # Name workers by their type concatenated with an incrementing number
-        # e.g. federation_reader1
-        worker_name = worker_type + str(new_worker_count)
-        worker_config.update(
-            {"name": worker_name, "port": worker_port, "config_path": config_path}
-        )
-
-        # Update the shared config with any worker-type specific options
-        shared_config.update(worker_config["shared_extra_conf"])
-
-        # Check if more than one instance of this worker type has been specified
-        worker_type_total_count = worker_types.count(worker_type)
-        if worker_type_total_count > 1:
-            # Update the shared config with sharding-related options if necessary
-            add_sharding_to_shared_config(
-                shared_config, worker_type, worker_name, worker_port
-            )
-
-        # Enable the worker in supervisord
-        supervisord_config += SUPERVISORD_PROCESS_CONFIG_BLOCK.format_map(worker_config)
-
-        # Add nginx location blocks for this worker's endpoints (if any are defined)
-        for pattern in worker_config["endpoint_patterns"]:
-            # Determine whether we need to load-balance this worker
-            if worker_type_total_count > 1:
-                # Create or add to a load-balanced upstream for this worker
-                nginx_upstreams.setdefault(worker_type, set()).add(worker_port)
-
-                # Upstreams are named after the worker_type
-                upstream = "http://" + worker_type
-            else:
-                upstream = "http://localhost:%d" % (worker_port,)
-
-            # Note that this endpoint should proxy to this upstream
-            nginx_locations[pattern] = upstream
-
-        # Write out the worker's logging config file
-
-        # Check whether we should write worker logs to disk, in addition to the console
-        extra_log_template_args = {}
-        if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
-            extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
-                dir=data_dir, name=worker_name
-            )
-
-        # Render and write the file
-        log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
-        convert(
-            "/conf/log.config",
-            log_config_filepath,
-            worker_name=worker_name,
-            **extra_log_template_args,
-        )
-
-        # Then a worker config file
-        convert(
-            "/conf/worker.yaml.j2",
-            "/conf/workers/{name}.yaml".format(name=worker_name),
-            **worker_config,
-            worker_log_config_filepath=log_config_filepath,
-        )
-
-        worker_port += 1
-
-    # Build the nginx location config blocks
-    nginx_location_config = ""
-    for endpoint, upstream in nginx_locations.items():
-        nginx_location_config += NGINX_LOCATION_CONFIG_BLOCK.format(
-            endpoint=endpoint,
-            upstream=upstream,
-        )
-
-    # Determine the load-balancing upstreams to configure
-    nginx_upstream_config = ""
-    for upstream_worker_type, upstream_worker_ports in nginx_upstreams.items():
-        body = ""
-        for port in upstream_worker_ports:
-            body += "    server localhost:%d;\n" % (port,)
-
-        # Add to the list of configured upstreams
-        nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
-            upstream_worker_type=upstream_worker_type,
-            body=body,
-        )
-
-    # Finally, we'll write out the config files.
-
-    # Shared homeserver config
-    convert(
-        "/conf/shared.yaml.j2",
-        "/conf/workers/shared.yaml",
-        shared_worker_config=yaml.dump(shared_config),
-    )
-
-    # Nginx config
-    convert(
-        "/conf/nginx.conf.j2",
-        "/etc/nginx/conf.d/matrix-synapse.conf",
-        worker_locations=nginx_location_config,
-        upstream_directives=nginx_upstream_config,
-    )
-
-    # Supervisord config
-    convert(
-        "/conf/supervisord.conf.j2",
-        "/etc/supervisor/conf.d/supervisord.conf",
-        main_config_path=config_path,
-        worker_config=supervisord_config,
-    )
-
-    # Ensure the logging directory exists
-    log_dir = data_dir + "/logs"
-    if not os.path.exists(log_dir):
-        os.mkdir(log_dir)
-
-
-def start_supervisord():
-    """Starts up supervisord which then starts and monitors all other necessary processes
-
-    Raises: CalledProcessError if calling start.py return a non-zero exit code.
-    """
-    subprocess.run(["/usr/bin/supervisord"], stdin=subprocess.PIPE)
-
-
-def main(args, environ):
-    config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
-    config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
-    data_dir = environ.get("SYNAPSE_DATA_DIR", "/data")
-
-    # override SYNAPSE_NO_TLS, we don't support TLS in worker mode,
-    # this needs to be handled by a frontend proxy
-    environ["SYNAPSE_NO_TLS"] = "yes"
-
-    # Generate the base homeserver config if one does not yet exist
-    if not os.path.exists(config_path):
-        log("Generating base homeserver config")
-        generate_base_homeserver_config()
-
-    # This script may be run multiple times (mostly by Complement, see note at top of file).
-    # Don't re-configure workers in this instance.
-    mark_filepath = "/conf/workers_have_been_configured"
-    if not os.path.exists(mark_filepath):
-        # Always regenerate all other config files
-        generate_worker_files(environ, config_path, data_dir)
-
-        # Mark workers as being configured
-        with open(mark_filepath, "w") as f:
-            f.write("")
-
-    # Start supervisord, which will start Synapse, all of the configured worker
-    # processes, redis, nginx etc. according to the config we created above.
-    start_supervisord()
-
-
-if __name__ == "__main__":
-    main(sys.argv, os.environ)

docker/run_pg_tests.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # This script runs the PostgreSQL tests inside a Docker container. It expects
 # the relevant source files to be mounted into /src (done automatically by the
@@ -17,4 +17,4 @@ su -c '/usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start'
 # Run the tests
 cd /src
 export TRIAL_FLAGS="-j 4"
-tox --workdir=/tmp -e py35-postgres
+tox --workdir=/tmp -e py27-postgres

docker/start.py

@@ -1,266 +1,67 @@
 #!/usr/local/bin/python
 
-import codecs
-import glob
-import os
-import platform
-import subprocess
-import sys
-
 import jinja2
-
+import os
+import sys
+import subprocess
+import glob
+import codecs
 
 # Utility functions
-def log(txt):
-    print(txt, file=sys.stderr)
+convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ))
 
+def check_arguments(environ, args):
+    for argument in args:
+        if argument not in environ:
+            print("Environment variable %s is mandatory, exiting." % argument)
+            sys.exit(2)
 
-def error(txt):
-    log(txt)
-    sys.exit(2)
-
-
-def convert(src, dst, environ):
-    """Generate a file from a template
-
-    Args:
-        src (str): path to input file
-        dst (str): path to file to write
-        environ (dict): environment dictionary, for replacement mappings.
-    """
-    with open(src) as infile:
-        template = infile.read()
-    rendered = jinja2.Template(template).render(**environ)
-    with open(dst, "w") as outfile:
-        outfile.write(rendered)
-
-
-def generate_config_from_template(config_dir, config_path, environ, ownership):
-    """Generate a homeserver.yaml from environment variables
-
-    Args:
-        config_dir (str): where to put generated config files
-        config_path (str): where to put the main config file
-        environ (dict): environment dictionary
-        ownership (str|None): "<user>:<group>" string which will be used to set
-            ownership of the generated configs. If None, ownership will not change.
-    """
-    for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"):
-        if v not in environ:
-            error(
-                "Environment variable '%s' is mandatory when generating a config file."
-                % (v,)
-            )
-
-    # populate some params from data files (if they exist, else create new ones)
-    environ = environ.copy()
-    secrets = {
-        "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET",
-        "macaroon": "SYNAPSE_MACAROON_SECRET_KEY",
-    }
-
+def generate_secrets(environ, secrets):
     for name, secret in secrets.items():
         if secret not in environ:
             filename = "/data/%s.%s.key" % (environ["SYNAPSE_SERVER_NAME"], name)
-
-            # if the file already exists, load in the existing value; otherwise,
-            # generate a new secret and write it to a file
-
             if os.path.exists(filename):
-                log("Reading %s from %s" % (secret, filename))
-                with open(filename) as handle:
-                    value = handle.read()
+                with open(filename) as handle: value = handle.read()
             else:
-                log("Generating a random secret for {}".format(secret))
+                print("Generating a random secret for {}".format(name))
                 value = codecs.encode(os.urandom(32), "hex").decode()
-                with open(filename, "w") as handle:
-                    handle.write(value)
+                with open(filename, "w") as handle: handle.write(value)
             environ[secret] = value
 
-    environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml")
-    if not os.path.exists(config_dir):
-        os.mkdir(config_dir)
+# Prepare the configuration
+mode = sys.argv[1] if len(sys.argv) > 1 else None
+environ = os.environ.copy()
+ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991))
+args = ["python", "-m", "synapse.app.homeserver"]
 
-    # Convert SYNAPSE_NO_TLS to boolean if exists
-    if "SYNAPSE_NO_TLS" in environ:
-        tlsanswerstring = str.lower(environ["SYNAPSE_NO_TLS"])
-        if tlsanswerstring in ("true", "on", "1", "yes"):
-            environ["SYNAPSE_NO_TLS"] = True
-        else:
-            if tlsanswerstring in ("false", "off", "0", "no"):
-                environ["SYNAPSE_NO_TLS"] = False
-            else:
-                error(
-                    'Environment variable "SYNAPSE_NO_TLS" found but value "'
-                    + tlsanswerstring
-                    + '" unrecognized; exiting.'
-                )
-
-    if "SYNAPSE_LOG_CONFIG" not in environ:
-        environ["SYNAPSE_LOG_CONFIG"] = config_dir + "/log.config"
-
-    log("Generating synapse config file " + config_path)
-    convert("/conf/homeserver.yaml", config_path, environ)
-
-    log_config_file = environ["SYNAPSE_LOG_CONFIG"]
-    log("Generating log config file " + log_config_file)
-    convert("/conf/log.config", log_config_file, environ)
-
-    # Hopefully we already have a signing key, but generate one if not.
-    args = [
-        "python",
-        "-m",
-        "synapse.app.homeserver",
-        "--config-path",
-        config_path,
-        # tell synapse to put generated keys in /data rather than /compiled
-        "--keys-directory",
-        config_dir,
-        "--generate-keys",
+# In generate mode, generate a configuration, missing keys, then exit
+if mode == "generate":
+    check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_CONFIG_PATH"))
+    args += [
+        "--server-name", environ["SYNAPSE_SERVER_NAME"],
+        "--report-stats", environ["SYNAPSE_REPORT_STATS"],
+        "--config-path", environ["SYNAPSE_CONFIG_PATH"],
+        "--generate-config"
     ]
+    os.execv("/usr/local/bin/python", args)
 
-    if ownership is not None:
+# In normal mode, generate missing keys if any, then run synapse
+else:
+    # Parse the configuration file
+    if "SYNAPSE_CONFIG_PATH" in environ:
+        args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]]
+    else:
+        check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"))
+        generate_secrets(environ, {
+            "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET",
+            "macaroon": "SYNAPSE_MACAROON_SECRET_KEY"
+        })
+        environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml")
+        if not os.path.exists("/compiled"): os.mkdir("/compiled")
+        convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ)
+        convert("/conf/log.config", "/compiled/log.config", environ)
         subprocess.check_output(["chown", "-R", ownership, "/data"])
-        args = ["gosu", ownership] + args
-
-    subprocess.check_output(args)
-
-
-def run_generate_config(environ, ownership):
-    """Run synapse with a --generate-config param to generate a template config file
-
-    Args:
-        environ (dict): env var dict
-        ownership (str|None): "userid:groupid" arg for chmod. If None, ownership will not change.
-
-    Never returns.
-    """
-    for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"):
-        if v not in environ:
-            error("Environment variable '%s' is mandatory in `generate` mode." % (v,))
-
-    server_name = environ["SYNAPSE_SERVER_NAME"]
-    config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
-    config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
-    data_dir = environ.get("SYNAPSE_DATA_DIR", "/data")
-
-    # create a suitable log config from our template
-    log_config_file = "%s/%s.log.config" % (config_dir, server_name)
-    if not os.path.exists(log_config_file):
-        log("Creating log config %s" % (log_config_file,))
-        convert("/conf/log.config", log_config_file, environ)
-
-    args = [
-        "python",
-        "-m",
-        "synapse.app.homeserver",
-        "--server-name",
-        server_name,
-        "--report-stats",
-        environ["SYNAPSE_REPORT_STATS"],
-        "--config-path",
-        config_path,
-        "--config-directory",
-        config_dir,
-        "--data-directory",
-        data_dir,
-        "--generate-config",
-        "--open-private-ports",
-    ]
-    # log("running %s" % (args, ))
-
-    if ownership is not None:
-        # make sure that synapse has perms to write to the data dir.
-        subprocess.check_output(["chown", ownership, data_dir])
-
-        args = ["gosu", ownership] + args
-        os.execv("/usr/sbin/gosu", args)
-    else:
-        os.execv("/usr/local/bin/python", args)
-
-
-def main(args, environ):
-    mode = args[1] if len(args) > 1 else "run"
-    desired_uid = int(environ.get("UID", "991"))
-    desired_gid = int(environ.get("GID", "991"))
-    synapse_worker = environ.get("SYNAPSE_WORKER", "synapse.app.homeserver")
-    if (desired_uid == os.getuid()) and (desired_gid == os.getgid()):
-        ownership = None
-    else:
-        ownership = "{}:{}".format(desired_uid, desired_gid)
-
-    if ownership is None:
-        log("Will not perform chmod/gosu as UserID already matches request")
-
-    # In generate mode, generate a configuration and missing keys, then exit
-    if mode == "generate":
-        return run_generate_config(environ, ownership)
-
-    if mode == "migrate_config":
-        # generate a config based on environment vars.
-        config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
-        config_path = environ.get(
-            "SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml"
-        )
-        return generate_config_from_template(
-            config_dir, config_path, environ, ownership
-        )
-
-    if mode != "run":
-        error("Unknown execution mode '%s'" % (mode,))
-
-    args = args[2:]
-
-    if "-m" not in args:
-        args = ["-m", synapse_worker] + args
-
-    jemallocpath = "/usr/lib/%s-linux-gnu/libjemalloc.so.2" % (platform.machine(),)
-
-    if os.path.isfile(jemallocpath):
-        environ["LD_PRELOAD"] = jemallocpath
-        environ["PYTHONMALLOC"] = "malloc"
-    else:
-        log("Could not find %s, will not use" % (jemallocpath,))
-
-    # if there are no config files passed to synapse, try adding the default file
-    if not any(p.startswith("--config-path") or p.startswith("-c") for p in args):
-        config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
-        config_path = environ.get(
-            "SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml"
-        )
-
-        if not os.path.exists(config_path):
-            if "SYNAPSE_SERVER_NAME" in environ:
-                error(
-                    """\
-Config file '%s' does not exist.
-
-The synapse docker image no longer supports generating a config file on-the-fly
-based on environment variables. You can migrate to a static config file by
-running with 'migrate_config'. See the README for more details.
-"""
-                    % (config_path,)
-                )
-
-            error(
-                "Config file '%s' does not exist. You should either create a new "
-                "config file by running with the `generate` argument (and then edit "
-                "the resulting file before restarting) or specify the path to an "
-                "existing config file with the SYNAPSE_CONFIG_PATH variable."
-                % (config_path,)
-            )
-
-        args += ["--config-path", config_path]
-
-    log("Starting synapse with args " + " ".join(args))
-
-    args = ["python"] + args
-    if ownership is not None:
-        args = ["gosu", ownership] + args
-        os.execve("/usr/sbin/gosu", args, environ)
-    else:
-        os.execve("/usr/local/bin/python", args, environ)
-
-
-if __name__ == "__main__":
-    main(sys.argv, os.environ)
+        args += ["--config-path", "/compiled/homeserver.yaml"]
+    # Generate missing keys and start synapse
+    subprocess.check_output(args + ["--generate-keys"])
+    os.execv("/sbin/su-exec", ["su-exec", ownership] + args)

docs/.sample_config_header.yaml

@@ -1,25 +0,0 @@
-# This file is maintained as an up-to-date snapshot of the default
-# homeserver.yaml configuration generated by Synapse.
-#
-# It is intended to act as a reference for the default configuration,
-# helping admins keep track of new options and other changes, and compare
-# their configs with the current default.  As such, many of the actual
-# config values shown are placeholders.
-#
-# It is *not* intended to be copied and used as the basis for a real
-# homeserver.yaml. Instead, if you are starting from scratch, please generate
-# a fresh config using Synapse by following the instructions in INSTALL.md.
-
-# Configuration options that take a time period can be set using a number
-# followed by a letter. Letters have the following meanings:
-# s = second
-# m = minute
-# h = hour
-# d = day
-# w = week
-# y = year
-# For example, setting redaction_retention_period: 5m would remove redacted
-# messages from the database after 5 minutes, rather than 5 months.
-
-################################################################################
-