Prevent curly braces from breaking in format strings

2021-04-27 11:40:21 +01:00
989 changed files with 18173 additions and 44210 deletions
@@ -0,0 +1,13 @@
+CI
+BUILDKITE
+BUILDKITE_BUILD_NUMBER
+BUILDKITE_BRANCH
+BUILDKITE_BUILD_NUMBER
+BUILDKITE_JOB_ID
+BUILDKITE_BUILD_URL
+BUILDKITE_PROJECT_SLUG
+BUILDKITE_COMMIT
+BUILDKITE_PULL_REQUEST
+BUILDKITE_TAG
+CODECOV_TOKEN
+TRIAL_FLAGS
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs|release-.*)$ ]]; then
+    echo "Not merging forward, as this is a release branch"
+    exit 0
+fi
+
+if [[ -z $BUILDKITE_PULL_REQUEST_BASE_BRANCH ]]; then
+    echo "Not a pull request, or hasn't had a PR opened yet..."
+
+    # It probably hasn't had a PR opened yet. Since all PRs land on develop, we
+    # can probably assume it's based on it and will be merged into it.
+    GITBASE="develop"
+else
+    # Get the reference, using the GitHub API
+    GITBASE=$BUILDKITE_PULL_REQUEST_BASE_BRANCH
+fi
+
+echo "--- merge_base_branch $GITBASE"
+
+# Show what we are before
+git --no-pager show -s
+
+# Set up username so it can do a merge
+git config --global user.email bot@matrix.org
+git config --global user.name "A robot"
+
+# Fetch and merge. If it doesn't work, it will raise due to set -e.
+git fetch -u origin $GITBASE
+git merge --no-edit --no-commit origin/$GITBASE
+
+# Show what we are after.
+git --no-pager show -s
@@ -3,7 +3,7 @@
 # CI's Docker setup at the point where this file is considered.
 server_name: "localhost:8800"

-signing_key_path: ".ci/test.signing.key"
+signing_key_path: "/src/.buildkite/test.signing.key"

 report_stats: false

@@ -11,9 +11,11 @@ database:
  name: "psycopg2"
  args:
    user: postgres
-    host: localhost
+    host: postgres
    password: postgres
    database: synapse

 # Suppress the key server warning.
-trusted_key_servers: []
+trusted_key_servers:
+  - server_name: "matrix.org"
+suppress_key_server_warning: true
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from synapse.storage.engines import create_engine
+
+logger = logging.getLogger("create_postgres_db")
+
+if __name__ == "__main__":
+    # Create a PostgresEngine.
+    db_engine = create_engine({"name": "psycopg2", "args": {}})
+
+    # Connect to postgres to create the base database.
+    # We use "postgres" as a database because it's bound to exist and the "synapse" one
+    # doesn't exist yet.
+    db_conn = db_engine.module.connect(
+        user="postgres", host="postgres", password="postgres", dbname="postgres"
+    )
+    db_conn.autocommit = True
+    cur = db_conn.cursor()
+    cur.execute("CREATE DATABASE synapse;")
+    cur.close()
+    db_conn.close()
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash

-# this script is run by GitHub Actions in a plain `bionic` container; it installs the
+# this script is run by buildkite in a plain `bionic` container; it installs the
 # minimal requirements for tox and hands over to the py3-old tox environment.

 set -ex
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Test script for 'synapse_port_db', which creates a virtualenv, installs Synapse along
+# with additional dependencies needed for the test (such as coverage or the PostgreSQL
+# driver), update the schema of the test SQLite database and run background updates on it,
+# create an empty test database in PostgreSQL, then run the 'synapse_port_db' script to
+# test porting the SQLite database to the PostgreSQL database (with coverage).
+
+set -xe
+cd `dirname $0`/../..
+
+echo "--- Install dependencies"
+
+# Install dependencies for this test.
+pip install psycopg2 coverage coverage-enable-subprocess
+
+# Install Synapse itself. This won't update any libraries.
+pip install -e .
+
+echo "--- Generate the signing key"
+
+# Generate the server's signing key.
+python -m synapse.app.homeserver --generate-keys -c .buildkite/sqlite-config.yaml
+
+echo "--- Prepare the databases"
+
+# Make sure the SQLite3 database is using the latest schema and has no pending background update.
+scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
+
+# Create the PostgreSQL database.
+./.buildkite/scripts/create_postgres_db.py
+
+echo "+++ Run synapse_port_db"
+
+# Run the script
+coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
@@ -3,14 +3,16 @@
 # schema and run background updates on it.
 server_name: "localhost:8800"

-signing_key_path: ".ci/test.signing.key"
+signing_key_path: "/src/.buildkite/test.signing.key"

 report_stats: false

 database:
  name: "sqlite3"
  args:
-    database: ".ci/test_db.db"
+    database: ".buildkite/test_db.db"

 # Suppress the key server warning.
-trusted_key_servers: []
+trusted_key_servers:
+  - server_name: "matrix.org"
+suppress_key_server_warning: true
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-# replaces the dependency on Twisted in `python_dependencies` with trunk.
-
-set -e
-cd "$(dirname "$0")"/..
-
-sed -i -e 's#"Twisted.*"#"Twisted @ git+https://github.com/twisted/twisted"#' synapse/python_dependencies.py
@@ -1,57 +0,0 @@
-#!/usr/bin/env bash
-#
-# Test script for 'synapse_port_db'.
-#   - sets up synapse and deps
-#   - runs the port script on a prepopulated test sqlite db
-#   - also runs it against an new sqlite db
-
-
-set -xe
-cd `dirname $0`/../..
-
-echo "--- Install dependencies"
-
-# Install dependencies for this test.
-pip install psycopg2 coverage coverage-enable-subprocess
-
-# Install Synapse itself. This won't update any libraries.
-pip install -e .
-
-echo "--- Generate the signing key"
-
-# Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
-
-echo "--- Prepare test database"
-
-# Make sure the SQLite3 database is using the latest schema and has no pending background update.
-scripts-dev/update_database --database-config .ci/sqlite-config.yaml
-
-# Create the PostgreSQL database.
-.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
-
-echo "+++ Run synapse_port_db against test database"
-coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
-
-# We should be able to run twice against the same database.
-echo "+++ Run synapse_port_db a second time"
-coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
-
-#####
-
-# Now do the same again, on an empty database.
-
-echo "--- Prepare empty SQLite database"
-
-# we do this by deleting the sqlite db, and then doing the same again.
-rm .ci/test_db.db
-
-scripts-dev/update_database --database-config .ci/sqlite-config.yaml
-
-# re-create the PostgreSQL database.
-.ci/scripts/postgres_exec.py \
-  "DROP DATABASE synapse" \
-  "CREATE DATABASE synapse"
-
-echo "+++ Run synapse_port_db against empty database"
-coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
@@ -1,4 +0,0 @@
---
-title: CI run against Twisted trunk is failing
---
-See https://github.com/{{env.GITHUB_REPOSITORY}}/actions/runs/{{env.GITHUB_RUN_ID}}
@@ -0,0 +1,78 @@
+version: 2.1
+jobs:
+  dockerhubuploadrelease:
+    docker:
+      - image: docker:git
+    steps:
+      - checkout
+      - docker_prepare
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      # for release builds, we want to get the amd64 image out asap, so first
+      # we do an amd64-only build, before following up with a multiarch build.
+      - docker_build:
+          tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
+          platforms: linux/amd64
+      - docker_build:
+          tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
+          platforms: linux/amd64,linux/arm64
+
+  dockerhubuploadlatest:
+    docker:
+      - image: docker:git
+    steps:
+      - checkout
+      - docker_prepare
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      # for `latest`, we don't want the arm images to disappear, so don't update the tag
+      # until all of the platforms are built.
+      - docker_build:
+          tag: -t matrixdotorg/synapse:latest
+          platforms: linux/amd64,linux/arm64
+
+workflows:
+  build:
+    jobs:
+      - dockerhubuploadrelease:
+          filters:
+            tags:
+              only: /v[0-9].[0-9]+.[0-9]+.*/
+            branches:
+              ignore: /.*/
+      - dockerhubuploadlatest:
+          filters:
+            branches:
+              only: master
+
+commands:
+  docker_prepare:
+    description: Sets up a remote docker server, downloads the buildx cli plugin, and enables multiarch images
+    parameters:
+      buildx_version:
+        type: string
+        default: "v0.4.1"
+    steps:
+      - setup_remote_docker:
+          # 19.03.13 was the most recent available on circleci at the time of
+          # writing.
+          version: 19.03.13
+      - run: apk add --no-cache curl
+      - run: mkdir -vp ~/.docker/cli-plugins/ ~/dockercache
+      - run: curl --silent -L "https://github.com/docker/buildx/releases/download/<< parameters.buildx_version >>/buildx-<< parameters.buildx_version >>.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
+      - run: chmod a+x ~/.docker/cli-plugins/docker-buildx
+      # install qemu links in /proc/sys/fs/binfmt_misc on the docker instance running the circleci job
+      - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+      # create a context named `builder` for the builds
+      - run: docker context create builder
+      # create a buildx builder using the new context, and set it as the default
+      - run: docker buildx create builder --use
+
+  docker_build:
+    description: Builds and pushed images to dockerhub using buildx
+    parameters:
+      platforms:
+        type: string
+        default: linux/amd64
+      tag:
+        type: string
+    steps:
+      - run: docker buildx build -f docker/Dockerfile --push --platform << parameters.platforms >> --label gitsha1=${CIRCLE_SHA1} << parameters.tag >> --progress=plain .
@@ -1,72 +0,0 @@
-# GitHub actions workflow which builds and publishes the docker images.
-
-name: Build docker images
-
-on:
-  push:
-    tags: ["v*"]
-    branches: [ master, main ]
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up QEMU
-        id: qemu
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: arm64
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Inspect builder
-        run: docker buildx inspect
-          
-      - name: Log in to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Calculate docker image tag
-        id: set-tag
-        run: |
-          case "${GITHUB_REF}" in
-              refs/heads/master|refs/heads/main)
-                  tag=latest
-                  ;;
-              refs/tags/*)
-                  tag=${GITHUB_REF#refs/tags/}
-                  ;;
-              *)
-                  tag=${GITHUB_SHA}
-                  ;;
-          esac
-          echo "::set-output name=tag::$tag"
-
-        # for release builds, we want to get the amd64 image out asap, so first
-        # we do an amd64-only build, before following up with a multiarch build.
-      - name: Build and push amd64
-        uses: docker/build-push-action@v2
-        if: "${{ startsWith(github.ref, 'refs/tags/v') }}"
-        with:
-          push: true
-          labels: "gitsha1=${{ github.sha }}"
-          tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
-          file: "docker/Dockerfile"
-          platforms: linux/amd64
-
-      - name: Build and push all platforms
-        uses: docker/build-push-action@v2
-        with:
-          push: true
-          labels: "gitsha1=${{ github.sha }}"
-          tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
-          file: "docker/Dockerfile"
-          platforms: linux/amd64,linux/arm64
@@ -1,66 +0,0 @@
-name: Deploy the documentation
-
-on:
-  push:
-    branches:
-      # For bleeding-edge documentation
-      - develop
-      # For documentation specific to a release
-      - 'release-v*'
-      # stable docs
-      - master
-
-  workflow_dispatch:
-
-jobs:
-  pages:
-    name: GitHub Pages
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Setup mdbook
-        uses: peaceiris/actions-mdbook@4b5ef36b314c2599664ca107bb8c02412548d79d # v1.1.14
-        with:
-          mdbook-version: '0.4.9'
-
-      - name: Build the documentation
-        # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
-        # However, we're using docs/README.md for other purposes and need to pick a new page
-        # as the default. Let's opt for the welcome page instead.
-        run: |
-          mdbook build
-          cp book/welcome_and_overview.html book/index.html
-
-      # Figure out the target directory.
-      #
-      # The target directory depends on the name of the branch
-      #
-      - name: Get the target directory name
-        id: vars
-        run: |
-          # first strip the 'refs/heads/' prefix with some shell foo
-          branch="${GITHUB_REF#refs/heads/}"
-
-          case $branch in
-              release-*)
-                  # strip 'release-' from the name for release branches.
-                  branch="${branch#release-}"
-                  ;;
-              master)
-                  # deploy to "latest" for the master branch.
-                  branch="latest"
-                  ;;
-          esac
-
-          # finally, set the 'branch-version' var.
-          echo "::set-output name=branch-version::$branch"
-          
-      # Deploy to the target directory.
-      - name: Deploy to gh pages
-        uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          keep_files: true
-          publish_dir: ./book
-          destination_dir: ./${{ steps.vars.outputs.branch-version }}
@@ -1,130 +0,0 @@
-# GitHub actions workflow which builds the release artifacts.
-
-name: Build release artifacts
-
-on:
-  # we build on PRs and develop to (hopefully) get early warning
-  # of things breaking (but only build one set of debs)
-  pull_request:
-  push:
-    branches: ["develop"]
-
-    # we do the full build on tags.
-    tags: ["v*"]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-  
-permissions:
-  contents: write
-
-jobs:
-  get-distros:
-    name: "Calculate list of debian distros"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - id: set-distros
-        run: |
-          # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
-          dists='["debian:sid"]'
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-              dists=$(scripts-dev/build_debian_packages --show-dists-json)
-          fi
-          echo "::set-output name=distros::$dists"
-    # map the step outputs to job outputs
-    outputs:
-      distros: ${{ steps.set-distros.outputs.distros }}
-
-  # now build the packages with a matrix build.
-  build-debs:
-    needs: get-distros
-    name: "Build .deb packages"
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        distro: ${{ fromJson(needs.get-distros.outputs.distros) }}
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          path: src
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          install: true
-
-      - name: Set up docker layer caching
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
-
-      - name: Set up python
-        uses: actions/setup-python@v2
-
-      - name: Build the packages
-        # see https://github.com/docker/build-push-action/issues/252
-        # for the cache magic here
-        run: |
-          ./src/scripts-dev/build_debian_packages \
-            --docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \
-            --docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
-            --docker-build-arg=--progress=plain \
-            --docker-build-arg=--load \
-            "${{ matrix.distro }}"
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
-
-      - name: Upload debs as artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: debs
-          path: debs/*
-
-  build-sdist:
-    name: "Build pypi distribution files"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install wheel
-      - run: |
-          python setup.py sdist bdist_wheel
-      - uses: actions/upload-artifact@v2
-        with:
-          name: python-dist
-          path: dist/*
-
-  # if it's a tag, create a release and attach the artifacts to it
-  attach-assets:
-    name: "Attach assets to release"
-    if: ${{ !failure() && !cancelled() && startsWith(github.ref, 'refs/tags/') }}
-    needs:
-      - build-debs
-      - build-sdist
-    runs-on: ubuntu-latest
-    steps:
-      - name: Download all workflow run artifacts
-        uses: actions/download-artifact@v2
-      - name: Build a tarball for the debs
-        run: tar -cvJf debs.tar.xz debs
-      - name: Attach to release
-        uses: softprops/action-gh-release@a929a66f232c1b11af63782948aa2210f981808a  # PR#109
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          files: |
-            python-dist/*
-            debs.tar.xz
-          # if it's not already published, keep the release as a draft.
-          draft: true
-          # mark it as a prerelease if the tag contains 'rc'.
-          prerelease: ${{ contains(github.ref, 'rc') }}
@@ -5,10 +5,6 @@ on:
    branches: ["develop", "release-*"]
  pull_request:

-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
 jobs:
  lint:
    runs-on: ubuntu-latest
@@ -39,14 +35,13 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
      - uses: actions/setup-python@v2
      - run: pip install tox
+      - name: Patch Buildkite-specific test script
+        run: |
+          sed -i -e 's/\$BUILDKITE_PULL_REQUEST/${{ github.event.number }}/' \
+            scripts-dev/check-newsfragment
      - run: scripts-dev/check-newsfragment
-        env:
-          PULL_REQUEST_NUMBER: ${{ github.event.number }}

  lint-sdist:
    runs-on: ubuntu-latest
@@ -64,14 +59,14 @@ jobs:

  # Dummy step to gate other tests on without repeating the whole list
  linting-done:
-    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
+    if: ${{ always() }} # Run this even if prior jobs were skipped
    needs: [lint, lint-crlf, lint-newsfile, lint-sdist]
    runs-on: ubuntu-latest
    steps:
      - run: "true"

  trial:
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
+    if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail
    needs: linting-done
    runs-on: ubuntu-latest
    strategy:
@@ -130,7 +125,7 @@ jobs:
          || true

  trial-olddeps:
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
+    if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail
    needs: linting-done
    runs-on: ubuntu-latest
    steps:
@@ -139,7 +134,7 @@ jobs:
        uses: docker://ubuntu:bionic # For old python and sqlite
        with:
          workdir: /github/workspace
-          entrypoint: .ci/scripts/test_old_deps.sh
+          entrypoint: .buildkite/scripts/test_old_deps.sh
        env:
          TRIAL_FLAGS: "--jobs=2"
      - name: Dump logs
@@ -155,7 +150,7 @@ jobs:

  trial-pypy:
    # Very slow; only run if the branch name includes 'pypy'
-    if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
+    if: ${{ contains(github.ref, 'pypy') && !failure() }}
    needs: linting-done
    runs-on: ubuntu-latest
    strategy:
@@ -184,7 +179,7 @@ jobs:
          || true

  sytest:
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() }}
    needs: linting-done
    runs-on: ubuntu-latest
    container:
@@ -192,12 +187,12 @@ jobs:
      volumes:
        - ${{ github.workspace }}:/src
      env:
+        BUILDKITE_BRANCH: ${{ github.head_ref }}
        POSTGRES: ${{ matrix.postgres && 1}}
        MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
        WORKERS: ${{ matrix.workers && 1 }}
        REDIS: ${{ matrix.redis && 1 }}
        BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
-        TOP: ${{ github.workspace }}

    strategy:
      fail-fast: false
@@ -227,13 +222,13 @@ jobs:
    steps:
      - uses: actions/checkout@v2
      - name: Prepare test blacklist
-        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
+        run: cat sytest-blacklist .buildkite/worker-blacklist > synapse-blacklist-with-workers
      - name: Run SyTest
        run: /bootstrap.sh synapse
        working-directory: /src
-      - name: Summarise results.tap
+      - name: Dump results.tap
        if: ${{ always() }}
-        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
+        run: cat /logs/results.tap
      - name: Upload SyTest logs
        uses: actions/upload-artifact@v2
        if: ${{ always() }}
@@ -244,11 +239,9 @@ jobs:
            /logs/**/*.log*

  portdb:
-    if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
+    if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail
    needs: linting-done
    runs-on: ubuntu-latest
-    env:
-      TOP: ${{ github.workspace }}
    strategy:
      matrix:
        include:
@@ -278,10 +271,16 @@ jobs:
      - uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
-      - run: .ci/scripts/test_synapse_port_db.sh
+      - name: Patch Buildkite-specific test scripts
+        run: |
+          sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/create_postgres_db.py
+          sed -i -e 's/host: postgres/host: localhost/' .buildkite/postgres-config.yaml
+          sed -i -e 's|/src/||' .buildkite/{sqlite,postgres}-config.yaml
+          sed -i -e 's/\$TOP/\$GITHUB_WORKSPACE/' .coveragerc
+      - run: .buildkite/scripts/test_synapse_port_db.sh

  complement:
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() }}
    needs: linting-done
    runs-on: ubuntu-latest
    container:
@@ -300,29 +299,11 @@ jobs:
        with:
          path: synapse

-      # Attempt to check out the same branch of Complement as the PR. If it
-      # doesn't exist, fallback to master.
-      - name: Checkout complement
-        shell: bash
-        run: |
-          mkdir -p complement
-          # Attempt to use the version of complement which best matches the current
-          # build. Depending on whether this is a PR or release, etc. we need to
-          # use different fallbacks.
-          #
-          # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
-          #    for pull requests, otherwise GITHUB_REF).
-          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
-          #    (GITHUB_BASE_REF for pull requests).
-          # 3. Use the default complement branch ("master").
-          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
-            # Skip empty branch names and merge commits.
-            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
-              continue
-            fi
-
-            (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
-          done
+      - name: Run actions/checkout@v2 for complement
+        uses: actions/checkout@v2
+        with:
+          repository: "matrix-org/complement"
+          path: complement

      # Build initial Synapse image
      - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
@@ -335,44 +316,7 @@ jobs:
        working-directory: complement/dockerfiles

      # Run Complement
-      - run: go test -v -tags synapse_blacklist,msc2403,msc2946,msc3083 ./tests/...
+      - run: go test -v -tags synapse_blacklist ./tests
        env:
          COMPLEMENT_BASE_IMAGE: complement-synapse:latest
        working-directory: complement
-
-  # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
-  tests-done:
-    if: ${{ always() }}
-    needs:
-      - lint
-      - lint-crlf
-      - lint-newsfile
-      - lint-sdist
-      - trial
-      - trial-olddeps
-      - sytest
-      - portdb
-      - complement
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set build result
-        env:
-          NEEDS_CONTEXT: ${{ toJSON(needs) }}
-        # the `jq` incantation dumps out a series of "<job> <result>" lines.
-        # we set it to an intermediate variable to avoid a pipe, which makes it
-        # hard to set $rc.
-        run: |
-          rc=0
-          results=$(jq -r 'to_entries[] | [.key,.value.result] | join(" ")' <<< $NEEDS_CONTEXT)
-          while read job result ; do
-              # The newsfile lint may be skipped on non PR builds
-              if [ $result == "skipped" ] && [ $job == "lint-newsfile" ]; then
-                continue
-              fi
-
-              if [ "$result" != "success" ]; then
-                  echo "::set-failed ::Job $job returned $result"
-                  rc=1
-              fi
-          done <<< $results
-          exit $rc
@@ -1,90 +0,0 @@
-name: Twisted Trunk
-
-on:
-  schedule:
-    - cron: 0 8 * * *
-
-  workflow_dispatch:
-
-jobs:
-  mypy:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e mypy
-
-  trial:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.6
-      - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e py
-        env:
-          TRIAL_FLAGS: "--jobs=2"
-
-      - name: Dump logs
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  sytest:
-    runs-on: ubuntu-latest
-    container:
-      image: matrixdotorg/sytest-synapse:buster
-      volumes:
-        - ${{ github.workspace }}:/src
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Patch dependencies
-        run: .ci/patch_for_twisted_trunk.sh
-        working-directory: /src
-      - name: Run SyTest
-        run: /bootstrap.sh synapse
-        working-directory: /src
-      - name: Summarise results.tap
-        if: ${{ always() }}
-        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
-      - name: Upload SyTest logs
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
-        with:
-          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
-          path: |
-            /logs/results.tap
-            /logs/**/*.log*
-
-  # open an issue if the build fails, so we know about it.
-  open-issue:
-    if: failure()
-    needs:
-      - mypy
-      - trial
-      - sytest
-
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          update_existing: true
-          filename: .ci/twisted_trunk_build_failed_issue_template.md
@@ -46,6 +46,3 @@ __pycache__/
 /docs/build/
 /htmlcov
 /pip-wheel-metadata/
-
-# docs
-book/
@@ -13,9 +13,8 @@ This document aims to get you started with contributing to this repo!
 - [7. Turn coffee and documentation into code and documentation!](#7-turn-coffee-and-documentation-into-code-and-documentation)
 - [8. Test, test, test!](#8-test-test-test)
  * [Run the linters.](#run-the-linters)
-  * [Run the unit tests.](#run-the-unit-tests-twisted-trial)
-  * [Run the integration tests (SyTest).](#run-the-integration-tests-sytest)
-  * [Run the integration tests (Complement).](#run-the-integration-tests-complement)
+  * [Run the unit tests.](#run-the-unit-tests)
+  * [Run the integration tests.](#run-the-integration-tests)
 - [9. Submit your patch.](#9-submit-your-patch)
  * [Changelog](#changelog)
    + [How do I know what to call the changelog file before I create the PR?](#how-do-i-know-what-to-call-the-changelog-file-before-i-create-the-pr)
@@ -156,7 +155,7 @@ source ./env/bin/activate
 ./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder
 ```

-## Run the unit tests (Twisted trial).
+## Run the unit tests.

 The unit tests run parts of Synapse, including your changes, to see if anything
 was broken. They are slower than the linters but will typically catch more errors.
@@ -174,20 +173,13 @@ source ./env/bin/activate
 trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite
 ```

-If your tests fail, you may wish to look at the logs (the default log level is `ERROR`):
+If your tests fail, you may wish to look at the logs:

 ```sh
 less _trial_temp/test.log
 ```

-To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`:
-
-```sh
-SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests
-```
-
-
-## Run the integration tests ([Sytest](https://github.com/matrix-org/sytest)).
+## Run the integration tests.

 The integration tests are a more comprehensive suite of tests. They
 run a full version of Synapse, including your changes, to check if
@@ -198,49 +190,12 @@ The following command will let you run the integration test with the most common
 configuration:

 ```sh
-$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:buster
+$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:py37
 ```

 This configuration should generally cover  your needs. For more details about other configurations, see [documentation in the SyTest repo](https://github.com/matrix-org/sytest/blob/develop/docker/README.md).


-## Run the integration tests ([Complement](https://github.com/matrix-org/complement)).
-
-[Complement](https://github.com/matrix-org/complement) is a suite of black box tests that can be run on any homeserver implementation. It can also be thought of as end-to-end (e2e) tests.
-
-It's often nice to develop on Synapse and write Complement tests at the same time.
-Here is how to run your local Synapse checkout against your local Complement checkout.
-
-(checkout [`complement`](https://github.com/matrix-org/complement) alongside your `synapse` checkout)
-```sh
-COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh
-```
-
-To run a specific test file, you can pass the test name at the end of the command. The name passed comes from the naming structure in your Complement tests. If you're unsure of the name, you can do a full run and copy it from the test output:
-
-```sh
-COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh TestBackfillingHistory
-```
-
-To run a specific test, you can specify the whole name structure:
-
-```sh
-COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh TestBackfillingHistory/parallel/Backfilled_historical_events_resolve_with_proper_state_in_correct_order
-```
-
-
-### Access database for homeserver after Complement test runs.
-
-If you're curious what the database looks like after you run some tests, here are some steps to get you going in Synapse:
-
- 1. In your Complement test comment out `defer deployment.Destroy(t)` and replace with `defer time.Sleep(2 * time.Hour)` to keep the homeserver running after the tests complete
- 1. Start the Complement tests
- 1. Find the name of the container, `docker ps -f name=complement_` (this will filter for just the Compelement related Docker containers)
- 1. Access the container replacing the name with what you found in the previous step: `docker exec -it complement_1_hs_with_application_service.hs1_2 /bin/bash`
- 1. Install sqlite (database driver), `apt-get update && apt-get install -y sqlite3`
- 1. Then run `sqlite3` and open the database `.open /conf/homeserver.db` (this db path comes from the Synapse homeserver.yaml)
-
-
 # 9. Submit your patch.

 Once you're happy with your patch, it's time to prepare a Pull Request.
@@ -253,7 +208,6 @@ To prepare a Pull Request, please:
 4. on GitHub, [create the Pull Request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request);
 5. add a [changelog entry](#changelog) and push it to your Pull Request;
 6. for most contributors, that's all - however, if you are a member of the organization `matrix-org`, on GitHub, please request a review from `matrix.org / Synapse Core`.
-7. if you need to update your PR, please avoid rebasing and just add new commits to your branch.


 ## Changelog
@@ -431,7 +385,7 @@ By now, you know the drill!
 # Notes for maintainers on merging PRs etc

 There are some notes for those with commit access to the project on how we
-manage git [here](docs/development/git.md).
+manage git [here](docs/dev/git.md).

 # Conclusion

@@ -1,7 +1,594 @@
 # Installation Instructions

-This document has moved to the
-[Synapse documentation website](https://matrix-org.github.io/synapse/latest/setup/installation.html).
-Please update your links.
+There are 3 steps to follow under **Installation Instructions**.

-The markdown source is available in [docs/setup/installation.md](docs/setup/installation.md).
+- [Installation Instructions](#installation-instructions)
+  - [Choosing your server name](#choosing-your-server-name)
+  - [Installing Synapse](#installing-synapse)
+    - [Installing from source](#installing-from-source)
+      - [Platform-specific prerequisites](#platform-specific-prerequisites)
+        - [Debian/Ubuntu/Raspbian](#debianubunturaspbian)
+        - [ArchLinux](#archlinux)
+        - [CentOS/Fedora](#centosfedora)
+        - [macOS](#macos)
+        - [OpenSUSE](#opensuse)
+        - [OpenBSD](#openbsd)
+        - [Windows](#windows)
+    - [Prebuilt packages](#prebuilt-packages)
+      - [Docker images and Ansible playbooks](#docker-images-and-ansible-playbooks)
+      - [Debian/Ubuntu](#debianubuntu)
+        - [Matrix.org packages](#matrixorg-packages)
+        - [Downstream Debian packages](#downstream-debian-packages)
+        - [Downstream Ubuntu packages](#downstream-ubuntu-packages)
+      - [Fedora](#fedora)
+      - [OpenSUSE](#opensuse-1)
+      - [SUSE Linux Enterprise Server](#suse-linux-enterprise-server)
+      - [ArchLinux](#archlinux-1)
+      - [Void Linux](#void-linux)
+      - [FreeBSD](#freebsd)
+      - [OpenBSD](#openbsd-1)
+      - [NixOS](#nixos)
+  - [Setting up Synapse](#setting-up-synapse)
+    - [Using PostgreSQL](#using-postgresql)
+    - [TLS certificates](#tls-certificates)
+    - [Client Well-Known URI](#client-well-known-uri)
+    - [Email](#email)
+    - [Registering a user](#registering-a-user)
+    - [Setting up a TURN server](#setting-up-a-turn-server)
+    - [URL previews](#url-previews)
+    - [Troubleshooting Installation](#troubleshooting-installation)
+
+
+## Choosing your server name
+
+It is important to choose the name for your server before you install Synapse,
+because it cannot be changed later.
+
+The server name determines the "domain" part of user-ids for users on your
+server: these will all be of the format `@user:my.domain.name`. It also
+determines how other matrix servers will reach yours for federation.
+
+For a test configuration, set this to the hostname of your server. For a more
+production-ready setup, you will probably want to specify your domain
+(`example.com`) rather than a matrix-specific hostname here (in the same way
+that your email address is probably `user@example.com` rather than
+`user@email.example.com`) - but doing so may require more advanced setup: see
+[Setting up Federation](docs/federate.md).
+
+## Installing Synapse
+
+### Installing from source
+
+(Prebuilt packages are available for some platforms - see [Prebuilt packages](#prebuilt-packages).)
+
+When installing from source please make sure that the [Platform-specific prerequisites](#platform-specific-prerequisites) are already installed.
+
+System requirements:
+
+- POSIX-compliant system (tested on Linux & OS X)
+- Python 3.5.2 or later, up to Python 3.9.
+- At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
+
+
+To install the Synapse homeserver run:
+
+```sh
+mkdir -p ~/synapse
+virtualenv -p python3 ~/synapse/env
+source ~/synapse/env/bin/activate
+pip install --upgrade pip
+pip install --upgrade setuptools
+pip install matrix-synapse
+```
+
+This will download Synapse from [PyPI](https://pypi.org/project/matrix-synapse)
+and install it, along with the python libraries it uses, into a virtual environment
+under `~/synapse/env`.  Feel free to pick a different directory if you
+prefer.
+
+This Synapse installation can then be later upgraded by using pip again with the
+update flag:
+
+```sh
+source ~/synapse/env/bin/activate
+pip install -U matrix-synapse
+```
+
+Before you can start Synapse, you will need to generate a configuration
+file. To do this, run (in your virtualenv, as before):
+
+```sh
+cd ~/synapse
+python -m synapse.app.homeserver \
+    --server-name my.domain.name \
+    --config-path homeserver.yaml \
+    --generate-config \
+    --report-stats=[yes|no]
+```
+
+... substituting an appropriate value for `--server-name`.
+
+This command will generate you a config file that you can then customise, but it will
+also generate a set of keys for you. These keys will allow your homeserver to
+identify itself to other homeserver, so don't lose or delete them. It would be
+wise to back them up somewhere safe. (If, for whatever reason, you do need to
+change your homeserver's keys, you may find that other homeserver have the
+old key cached. If you update the signing key, you should change the name of the
+key in the `<server name>.signing.key` file (the second word) to something
+different. See the [spec](https://matrix.org/docs/spec/server_server/latest.html#retrieving-server-keys) for more information on key management).
+
+To actually run your new homeserver, pick a working directory for Synapse to
+run (e.g. `~/synapse`), and:
+
+```sh
+cd ~/synapse
+source env/bin/activate
+synctl start
+```
+
+#### Platform-specific prerequisites
+
+Synapse is written in Python but some of the libraries it uses are written in
+C. So before we can install Synapse itself we need a working C compiler and the
+header files for Python C extensions.
+
+##### Debian/Ubuntu/Raspbian
+
+Installing prerequisites on Ubuntu or Debian:
+
+```sh
+sudo apt install build-essential python3-dev libffi-dev \
+                     python3-pip python3-setuptools sqlite3 \
+                     libssl-dev virtualenv libjpeg-dev libxslt1-dev
+```
+
+##### ArchLinux
+
+Installing prerequisites on ArchLinux:
+
+```sh
+sudo pacman -S base-devel python python-pip \
+               python-setuptools python-virtualenv sqlite3
+```
+
+##### CentOS/Fedora
+
+Installing prerequisites on CentOS or Fedora Linux:
+
+```sh
+sudo dnf install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
+                 libwebp-devel libxml2-devel libxslt-devel libpq-devel \
+                 python3-virtualenv libffi-devel openssl-devel python3-devel
+sudo dnf groupinstall "Development Tools"
+```
+
+##### macOS
+
+Installing prerequisites on macOS:
+
+```sh
+xcode-select --install
+sudo easy_install pip
+sudo pip install virtualenv
+brew install pkg-config libffi
+```
+
+On macOS Catalina (10.15) you may need to explicitly install OpenSSL
+via brew and inform `pip` about it so that `psycopg2` builds:
+
+```sh
+brew install openssl@1.1
+export LDFLAGS="-L/usr/local/opt/openssl/lib"
+export CPPFLAGS="-I/usr/local/opt/openssl/include"
+```
+
+##### OpenSUSE
+
+Installing prerequisites on openSUSE:
+
+```sh
+sudo zypper in -t pattern devel_basis
+sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \
+               python-devel libffi-devel libopenssl-devel libjpeg62-devel
+```
+
+##### OpenBSD
+
+A port of Synapse is available under `net/synapse`. The filesystem
+underlying the homeserver directory (defaults to `/var/synapse`) has to be
+mounted with `wxallowed` (cf. `mount(8)`), so creating a separate filesystem
+and mounting it to `/var/synapse` should be taken into consideration.
+
+To be able to build Synapse's dependency on python the `WRKOBJDIR`
+(cf. `bsd.port.mk(5)`) for building python, too, needs to be on a filesystem
+mounted with `wxallowed` (cf. `mount(8)`).
+
+Creating a `WRKOBJDIR` for building python under `/usr/local` (which on a
+default OpenBSD installation is mounted with `wxallowed`):
+
+```sh
+doas mkdir /usr/local/pobj_wxallowed
+```
+
+Assuming `PORTS_PRIVSEP=Yes` (cf. `bsd.port.mk(5)`) and `SUDO=doas` are
+configured in `/etc/mk.conf`:
+
+```sh
+doas chown _pbuild:_pbuild /usr/local/pobj_wxallowed
+```
+
+Setting the `WRKOBJDIR` for building python:
+
+```sh
+echo WRKOBJDIR_lang/python/3.7=/usr/local/pobj_wxallowed  \\nWRKOBJDIR_lang/python/2.7=/usr/local/pobj_wxallowed >> /etc/mk.conf
+```
+
+Building Synapse:
+
+```sh
+cd /usr/ports/net/synapse
+make install
+```
+
+##### Windows
+
+If you wish to run or develop Synapse on Windows, the Windows Subsystem For
+Linux provides a Linux environment on Windows 10 which is capable of using the
+Debian, Fedora, or source installation methods. More information about WSL can
+be found at <https://docs.microsoft.com/en-us/windows/wsl/install-win10> for
+Windows 10 and <https://docs.microsoft.com/en-us/windows/wsl/install-on-server>
+for Windows Server.
+
+### Prebuilt packages
+
+As an alternative to installing from source, prebuilt packages are available
+for a number of platforms.
+
+#### Docker images and Ansible playbooks
+
+There is an official synapse image available at
+<https://hub.docker.com/r/matrixdotorg/synapse> which can be used with
+the docker-compose file available at [contrib/docker](contrib/docker). Further
+information on this including configuration options is available in the README
+on hub.docker.com.
+
+Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
+Dockerfile to automate a synapse server in a single Docker image, at
+<https://hub.docker.com/r/avhost/docker-matrix/tags/>
+
+Slavi Pantaleev has created an Ansible playbook,
+which installs the offical Docker image of Matrix Synapse
+along with many other Matrix-related services (Postgres database, Element, coturn,
+ma1sd, SSL support, etc.).
+For more details, see
+<https://github.com/spantaleev/matrix-docker-ansible-deploy>
+
+#### Debian/Ubuntu
+
+##### Matrix.org packages
+
+Matrix.org provides Debian/Ubuntu packages of the latest stable version of
+Synapse via <https://packages.matrix.org/debian/>. They are available for Debian
+9 (Stretch), Ubuntu 16.04 (Xenial), and later. To use them:
+
+```sh
+sudo apt install -y lsb-release wget apt-transport-https
+sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
+echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" |
+    sudo tee /etc/apt/sources.list.d/matrix-org.list
+sudo apt update
+sudo apt install matrix-synapse-py3
+```
+
+**Note**: if you followed a previous version of these instructions which
+recommended using `apt-key add` to add an old key from
+`https://matrix.org/packages/debian/`, you should note that this key has been
+revoked. You should remove the old key with `sudo apt-key remove
+C35EB17E1EAE708E6603A9B3AD0592FE47F0DF61`, and follow the above instructions to
+update your configuration.
+
+The fingerprint of the repository signing key (as shown by `gpg
+/usr/share/keyrings/matrix-org-archive-keyring.gpg`) is
+`AAF9AE843A7584B5A3E4CD2BCF45A512DE2DA058`.
+
+##### Downstream Debian packages
+
+We do not recommend using the packages from the default Debian `buster`
+repository at this time, as they are old and suffer from known security
+vulnerabilities. You can install the latest version of Synapse from
+[our repository](#matrixorg-packages) or from `buster-backports`. Please
+see the [Debian documentation](https://backports.debian.org/Instructions/)
+for information on how to use backports.
+
+If you are using Debian `sid` or testing, Synapse is available in the default
+repositories and it should be possible to install it simply with:
+
+```sh
+sudo apt install matrix-synapse
+```
+
+##### Downstream Ubuntu packages
+
+We do not recommend using the packages in the default Ubuntu repository
+at this time, as they are old and suffer from known security vulnerabilities.
+The latest version of Synapse can be installed from [our repository](#matrixorg-packages).
+
+#### Fedora
+
+Synapse is in the Fedora repositories as `matrix-synapse`:
+
+```sh
+sudo dnf install matrix-synapse
+```
+
+Oleg Girko provides Fedora RPMs at
+<https://obs.infoserver.lv/project/monitor/matrix-synapse>
+
+#### OpenSUSE
+
+Synapse is in the OpenSUSE repositories as `matrix-synapse`:
+
+```sh
+sudo zypper install matrix-synapse
+```
+
+#### SUSE Linux Enterprise Server
+
+Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 repository at
+<https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/>
+
+#### ArchLinux
+
+The quickest way to get up and running with ArchLinux is probably with the community package
+<https://www.archlinux.org/packages/community/any/matrix-synapse/>, which should pull in most of
+the necessary dependencies.
+
+pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):
+
+```sh
+sudo pip install --upgrade pip
+```
+
+If you encounter an error with lib bcrypt causing an Wrong ELF Class:
+ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly
+compile it under the right architecture. (This should not be needed if
+installing under virtualenv):
+
+```sh
+sudo pip uninstall py-bcrypt
+sudo pip install py-bcrypt
+```
+
+#### Void Linux
+
+Synapse can be found in the void repositories as 'synapse':
+
+```sh
+xbps-install -Su
+xbps-install -S synapse
+```
+
+#### FreeBSD
+
+Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from:
+
+- Ports: `cd /usr/ports/net-im/py-matrix-synapse && make install clean`
+- Packages: `pkg install py37-matrix-synapse`
+
+#### OpenBSD
+
+As of OpenBSD 6.7 Synapse is available as a pre-compiled binary. The filesystem
+underlying the homeserver directory (defaults to `/var/synapse`) has to be
+mounted with `wxallowed` (cf. `mount(8)`), so creating a separate filesystem
+and mounting it to `/var/synapse` should be taken into consideration.
+
+Installing Synapse:
+
+```sh
+doas pkg_add synapse
+```
+
+#### NixOS
+
+Robin Lambertz has packaged Synapse for NixOS at:
+<https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-synapse.nix>
+
+## Setting up Synapse
+
+Once you have installed synapse as above, you will need to configure it.
+
+### Using PostgreSQL
+
+By default Synapse uses [SQLite](https://sqlite.org/) and in doing so trades performance for convenience.
+SQLite is only recommended in Synapse for testing purposes or for servers with
+very light workloads.
+
+Almost all installations should opt to use [PostgreSQL](https://www.postgresql.org). Advantages include:
+
+- significant performance improvements due to the superior threading and
+  caching model, smarter query optimiser
+- allowing the DB to be run on separate hardware
+
+For information on how to install and use PostgreSQL in Synapse, please see
+[docs/postgres.md](docs/postgres.md)
+
+### TLS certificates
+
+The default configuration exposes a single HTTP port on the local
+interface: `http://localhost:8008`. It is suitable for local testing,
+but for any practical use, you will need Synapse's APIs to be served
+over HTTPS.
+
+The recommended way to do so is to set up a reverse proxy on port
+`8448`. You can find documentation on doing so in
+[docs/reverse_proxy.md](docs/reverse_proxy.md).
+
+Alternatively, you can configure Synapse to expose an HTTPS port. To do
+so, you will need to edit `homeserver.yaml`, as follows:
+
+- First, under the `listeners` section, uncomment the configuration for the
+  TLS-enabled listener. (Remove the hash sign (`#`) at the start of
+  each line). The relevant lines are like this:
+
+```yaml
+  - port: 8448
+    type: http
+    tls: true
+    resources:
+      - names: [client, federation]
+  ```
+
+- You will also need to uncomment the `tls_certificate_path` and
+  `tls_private_key_path` lines under the `TLS` section. You will need to manage
+  provisioning of these certificates yourself — Synapse had built-in ACME
+  support, but the ACMEv1 protocol Synapse implements is deprecated, not
+  allowed by LetsEncrypt for new sites, and will break for existing sites in
+  late 2020. See [ACME.md](docs/ACME.md).
+
+  If you are using your own certificate, be sure to use a `.pem` file that
+  includes the full certificate chain including any intermediate certificates
+  (for instance, if using certbot, use `fullchain.pem` as your certificate, not
+  `cert.pem`).
+
+For a more detailed guide to configuring your server for federation, see
+[federate.md](docs/federate.md).
+
+### Client Well-Known URI
+
+Setting up the client Well-Known URI is optional but if you set it up, it will
+allow users to enter their full username (e.g. `@user:<server_name>`) into clients
+which support well-known lookup to automatically configure the homeserver and
+identity server URLs. This is useful so that users don't have to memorize or think
+about the actual homeserver URL you are using.
+
+The URL `https://<server_name>/.well-known/matrix/client` should return JSON in
+the following format.
+
+```json
+{
+  "m.homeserver": {
+    "base_url": "https://<matrix.example.com>"
+  }
+}
+```
+
+It can optionally contain identity server information as well.
+
+```json
+{
+  "m.homeserver": {
+    "base_url": "https://<matrix.example.com>"
+  },
+  "m.identity_server": {
+    "base_url": "https://<identity.example.com>"
+  }
+}
+```
+
+To work in browser based clients, the file must be served with the appropriate
+Cross-Origin Resource Sharing (CORS) headers. A recommended value would be
+`Access-Control-Allow-Origin: *` which would allow all browser based clients to
+view it.
+
+In nginx this would be something like:
+
+```nginx
+location /.well-known/matrix/client {
+    return 200 '{"m.homeserver": {"base_url": "https://<matrix.example.com>"}}';
+    default_type application/json;
+    add_header Access-Control-Allow-Origin *;
+}
+```
+
+You should also ensure the `public_baseurl` option in `homeserver.yaml` is set
+correctly. `public_baseurl` should be set to the URL that clients will use to
+connect to your server. This is the same URL you put for the `m.homeserver`
+`base_url` above.
+
+```yaml
+public_baseurl: "https://<matrix.example.com>"
+```
+
+### Email
+
+It is desirable for Synapse to have the capability to send email. This allows
+Synapse to send password reset emails, send verifications when an email address
+is added to a user's account, and send email notifications to users when they
+receive new messages.
+
+To configure an SMTP server for Synapse, modify the configuration section
+headed `email`, and be sure to have at least the `smtp_host`, `smtp_port`
+and `notif_from` fields filled out.  You may also need to set `smtp_user`,
+`smtp_pass`, and `require_transport_security`.
+
+If email is not configured, password reset, registration and notifications via
+email will be disabled.
+
+### Registering a user
+
+The easiest way to create a new user is to do so from a client like [Element](https://element.io/).
+
+Alternatively, you can do so from the command line. This can be done as follows:
+
+ 1. If synapse was installed via pip, activate the virtualenv as follows (if Synapse was
+    installed via a prebuilt package, `register_new_matrix_user` should already be
+    on the search path):
+    ```sh
+    cd ~/synapse
+    source env/bin/activate
+    synctl start # if not already running
+    ```
+ 2. Run the following command:
+    ```sh
+    register_new_matrix_user -c homeserver.yaml http://localhost:8008
+    ```
+
+This will prompt you to add details for the new user, and will then connect to
+the running Synapse to create the new user. For example:
+```
+New user localpart: erikj
+Password:
+Confirm password:
+Make admin [no]:
+Success!
+```
+
+This process uses a setting `registration_shared_secret` in
+`homeserver.yaml`, which is shared between Synapse itself and the
+`register_new_matrix_user` script. It doesn't matter what it is (a random
+value is generated by `--generate-config`), but it should be kept secret, as
+anyone with knowledge of it can register users, including admin accounts,
+on your server even if `enable_registration` is `false`.
+
+### Setting up a TURN server
+
+For reliable VoIP calls to be routed via this homeserver, you MUST configure
+a TURN server. See [docs/turn-howto.md](docs/turn-howto.md) for details.
+
+### URL previews
+
+Synapse includes support for previewing URLs, which is disabled by default.  To
+turn it on you must enable the `url_preview_enabled: True` config parameter
+and explicitly specify the IP ranges that Synapse is not allowed to spider for
+previewing in the `url_preview_ip_range_blacklist` configuration parameter.
+This is critical from a security perspective to stop arbitrary Matrix users
+spidering 'internal' URLs on your network. At the very least we recommend that
+your loopback and RFC1918 IP addresses are blacklisted.
+
+This also requires the optional `lxml` python dependency to be  installed. This
+in turn requires the `libxml2` library to be available - on  Debian/Ubuntu this
+means `apt-get install libxml2-dev`, or equivalent for your OS.
+
+### Troubleshooting Installation
+
+`pip` seems to leak *lots* of memory during installation. For instance, a Linux
+host with 512MB of RAM may run out of memory whilst installing Twisted. If this
+happens, you will have to individually install the dependencies which are
+failing, e.g.:
+
+```sh
+pip install twisted
+```
+
+If you have any other problems, feel free to ask in
+[#synapse:matrix.org](https://matrix.to/#/#synapse:matrix.org).
@@ -40,13 +40,12 @@ exclude mypy.ini
 exclude sytest-blacklist
 exclude test_postgresql.sh

-include book.toml
 include pyproject.toml
 recursive-include changelog.d *

+prune .buildkite
 prune .circleci
 prune .github
-prune .ci
 prune contrib
 prune debian
 prune demo/etc
@@ -25,7 +25,7 @@ The overall architecture is::

 ``#matrix:matrix.org`` is the official support room for Matrix, and can be
 accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html or
-via IRC bridge at irc://irc.libera.chat/matrix.
+via IRC bridge at irc://irc.freenode.net/matrix.

 Synapse is currently in rapid development, but as of version 0.5 we believe it
 is sufficiently stable to be run as an internet-facing service for real usage!
@@ -94,8 +94,7 @@ Synapse Installation

 .. _federation:

-* For details on how to install synapse, see
-  `Installation Instructions <https://matrix-org.github.io/synapse/latest/setup/installation.html>`_.
+* For details on how to install synapse, see `<INSTALL.md>`_.
 * For specific details on how to configure Synapse for federation see `docs/federate.md <docs/federate.md>`_


@@ -107,8 +106,7 @@ from a web client.

 Unless you are running a test instance of Synapse on your local machine, in
 general, you will need to enable TLS support before you can successfully
-connect from a client: see
-`TLS certificates <https://matrix-org.github.io/synapse/latest/setup/installation.html#tls-certificates>`_.
+connect from a client: see `<INSTALL.md#tls-certificates>`_.

 An easy way to get started is to login or register via Element at
 https://app.element.io/#/login or https://app.element.io/#/register respectively.
@@ -144,55 +142,38 @@ the form of::
 As when logging in, you will need to specify a "Custom server".  Specify your
 desired ``localpart`` in the 'User name' box.

-Security note
+ACME setup
+==========
+
+For details on having Synapse manage your federation TLS certificates
+automatically, please see `<docs/ACME.md>`_.
+
+
+Security Note
 =============

-Matrix serves raw, user-supplied data in some APIs -- specifically the `content
-repository endpoints`_.
+Matrix serves raw user generated data in some APIs - specifically the `content
+repository endpoints <https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid>`_.

-.. _content repository endpoints: https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid
+Whilst we have tried to mitigate against possible XSS attacks (e.g.
+https://github.com/matrix-org/synapse/pull/1021) we recommend running
+matrix homeservers on a dedicated domain name, to limit any malicious user generated
+content served to web browsers a matrix API from being able to attack webapps hosted
+on the same domain.  This is particularly true of sharing a matrix webclient and
+server on the same domain.

-Whilst we make a reasonable effort to mitigate against XSS attacks (for
-instance, by using `CSP`_), a Matrix homeserver should not be hosted on a
-domain hosting other web applications. This especially applies to sharing
-the domain with Matrix web clients and other sensitive applications like
-webmail. See
-https://developer.github.com/changes/2014-04-25-user-content-security for more
-information.
-
-.. _CSP: https://github.com/matrix-org/synapse/pull/1021
-
-Ideally, the homeserver should not simply be on a different subdomain, but on
-a completely different `registered domain`_ (also known as top-level site or
-eTLD+1). This is because `some attacks`_ are still possible as long as the two
-applications share the same registered domain.
-
-.. _registered domain: https://tools.ietf.org/html/draft-ietf-httpbis-rfc6265bis-03#section-2.3
-
-.. _some attacks: https://en.wikipedia.org/wiki/Session_fixation#Attacks_using_cross-subdomain_cookie
-
-To illustrate this with an example, if your Element Web or other sensitive web
-application is hosted on ``A.example1.com``, you should ideally host Synapse on
-``example2.com``. Some amount of protection is offered by hosting on
-``B.example1.com`` instead, so this is also acceptable in some scenarios.
-However, you should *not* host your Synapse on ``A.example1.com``.
-
-Note that all of the above refers exclusively to the domain used in Synapse's
-``public_baseurl`` setting. In particular, it has no bearing on the domain
-mentioned in MXIDs hosted on that server.
-
-Following this advice ensures that even if an XSS is found in Synapse, the
-impact to other applications will be minimal.
+See https://github.com/vector-im/riot-web/issues/1977 and
+https://developer.github.com/changes/2014-04-25-user-content-security for more details.


 Upgrading an existing Synapse
 =============================

-The instructions for upgrading synapse are in `the upgrade notes`_.
+The instructions for upgrading synapse are in `UPGRADE.rst`_.
 Please check these instructions as upgrading may require extra steps for some
 versions of synapse.

-.. _the upgrade notes: https://matrix-org.github.io/synapse/develop/upgrade.html
+.. _UPGRADE.rst: UPGRADE.rst

 .. _reverse-proxy:

@@ -267,7 +248,7 @@ Join our developer community on Matrix: `#synapse-dev:matrix.org <https://matrix

 Before setting up a development environment for synapse, make sure you have the
 system dependencies (such as the python header files) installed - see
-`Installing from source <https://matrix-org.github.io/synapse/latest/setup/installation.html#installing-from-source>`_.
+`Installing from source <INSTALL.md#installing-from-source>`_.

 To check out a synapse for development, clone the git repo into a working
 directory of your choice::
@@ -288,6 +269,18 @@ try installing the failing modules individually::

    pip install -e "module-name"

+Once this is done, you may wish to run Synapse's unit tests to
+check that everything is installed correctly::
+
+    python -m twisted.trial tests
+
+This should end with a 'PASSED' result (note that exact numbers will
+differ)::
+
+    Ran 1337 tests in 716.064s
+
+    PASSED (skips=15, successes=1322)
+
 We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`

    ./demo/start.sh
@@ -307,23 +300,6 @@ If you just want to start a single instance of the app and run it directly::
    python -m synapse.app.homeserver --config-path homeserver.yaml


-Running the unit tests
-======================
-
-After getting up and running, you may wish to run Synapse's unit tests to
-check that everything is installed correctly::
-
-    trial tests
-
-This should end with a 'PASSED' result (note that exact numbers will
-differ)::
-
-    Ran 1337 tests in 716.064s
-
-    PASSED (skips=15, successes=1322)
-
-For more tips on running the unit tests, like running a specific test or
-to see the logging output, see the `CONTRIBUTING doc <CONTRIBUTING.md#run-the-unit-tests>`_.


 Running the Integration Tests
@@ -335,8 +311,8 @@ access the API as a Matrix client would. It is able to run Synapse directly from
 the source tree, so installation of the server is not required.

 Testing with SyTest is recommended for verifying that changes related to the
-Client-Server API are functioning correctly. See the `SyTest installation
-instructions <https://github.com/matrix-org/sytest#installing>`_ for details.
+Client-Server API are functioning correctly. See the `installation instructions
+<https://github.com/matrix-org/sytest#installing>`_ for details.


 Platform dependencies
@@ -1,39 +0,0 @@
-# Documentation for possible options in this file is at
-# https://rust-lang.github.io/mdBook/format/config.html
-[book]
-title = "Synapse"
-authors = ["The Matrix.org Foundation C.I.C."]
-language = "en"
-multilingual = false
-
-# The directory that documentation files are stored in
-src = "docs"
-
-[build]
-# Prevent markdown pages from being automatically generated when they're
-# linked to in SUMMARY.md
-create-missing = false
-
-[output.html]
-# The URL visitors will be directed to when they try to edit a page
-edit-url-template = "https://github.com/matrix-org/synapse/edit/develop/{path}"
-
-# Remove the numbers that appear before each item in the sidebar, as they can
-# get quite messy as we nest deeper
-no-section-label = true
-
-# The source code URL of the repository
-git-repository-url = "https://github.com/matrix-org/synapse"
-
-# The path that the docs are hosted on
-site-url = "/synapse/"
-
-# Additional HTML, JS, CSS that's injected into each page of the book.
-# More information available in docs/website_files/README.md
-additional-css = [
-    "docs/website_files/table-of-contents.css",
-    "docs/website_files/remove-nav-buttons.css",
-    "docs/website_files/indent-section-headers.css",
-]
-additional-js = ["docs/website_files/table-of-contents.js"]
-theme = "docs/website_files/theme"
@@ -1 +0,0 @@
-Add support for [MSC3231 - Token authenticated registration](https://github.com/matrix-org/matrix-doc/pull/3231). Users can be required to submit a token during registration to authenticate themselves. Contributed by Callum Brown.
@@ -1 +0,0 @@
-Add documentation on how to connect Django with synapse using oidc and django-oauth-toolkit. Contributed by @HugoDelval.
@@ -1 +0,0 @@
-Add support for [MSC3283](https://github.com/matrix-org/matrix-doc/pull/3283): Expose enable_set_displayname in capabilities.
@@ -1 +0,0 @@
-Port the PresenceRouter module interface to the new generic interface.
@@ -1 +0,0 @@
-Display an error on User-Interactive Authentication fallback pages when authentication fails. Contributed by Callum Brown.
@@ -1 +0,0 @@
-Reject Client-Server `/keys/query` requests which provide `device_ids` incorrectly.
@@ -1 +0,0 @@
-Improve type hints for the proxy agent and SRV resolver modules. Contributed by @dklimpel.
@@ -1 +0,0 @@
-Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).
@@ -1 +0,0 @@
-Clean up some of the federation event authentication code for clarity.
@@ -1 +0,0 @@
-Clean up some of the federation event authentication code for clarity.
@@ -1 +0,0 @@
-Clean up some of the federation event authentication code for clarity.
@@ -1 +0,0 @@
-Remove not needed database updates in modify user admin API.
@@ -1 +0,0 @@
-Convert room member storage tuples to `attrs` classes.
@@ -1 +0,0 @@
-Use auto-attribs for the attrs classes used in sync.
@@ -1 +0,0 @@
-Fix some of the titles not rendering in the OIDC documentation.
@@ -1 +0,0 @@
-Clean up some of the federation event authentication code for clarity.
@@ -1 +0,0 @@
-Convert room member storage tuples to `attrs` classes.
@@ -1 +0,0 @@
-Rooms with unsupported room versions are no longer returned via `/sync`.
@@ -1 +0,0 @@
-Run a nightly CI build against Twisted trunk.
@@ -1 +0,0 @@
-Enforce the maximum length for per-room display names and avatar URLs.
@@ -1 +0,0 @@
-Do not print out stack traces for network errors when fetching data over federation.
@@ -1 +0,0 @@
-Simplify tests for device admin rest API.
@@ -1 +0,0 @@
-Add missing type hints to REST servlets.
@@ -1 +0,0 @@
-Add missing type hints to REST servlets.
@@ -1 +0,0 @@
-Flatten the `tests.synapse.rests` package by moving the contents of `v1` and `v2_alpha` into the parent.
@@ -1 +0,0 @@
-Run a nightly CI build against Twisted trunk.
@@ -1 +0,0 @@
-Fix a bug which caused the `synapse_user_logins_total` Prometheus metric not to be correctly initialised on restart.
@@ -1 +0,0 @@
-Port the SAML user mapping providers module interface to the new generic interface.
@@ -1 +0,0 @@
-Remove deprecated Shutdown Room and Purge Room Admin API.
@@ -0,0 +1 @@
+Add a dockerfile for running Synapse in worker-mode under Complement.
@@ -0,0 +1 @@
+Speed up federation transmission by using fewer database calls. Contributed by @ShadowJonathan.
@@ -0,0 +1 @@
+Fixes the OIDC SSO flow when using a `public_baseurl` value including a non-root URL path.
@@ -0,0 +1 @@
+Apply `pyupgrade` across the codebase.
@@ -0,0 +1 @@
+Fix thumbnail generation for some sites with non-standard content types. Contributed by @rkfg.
@@ -0,0 +1 @@
+Move some replication processing out of `generic_worker`.
@@ -0,0 +1 @@
+Update experimental support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083): restricting room access via group membership.
@@ -0,0 +1 @@
+Add a note to the docker docs mentioning that we mirror upstream's supported Docker platforms.
@@ -0,0 +1 @@
+Add some sanity checks to identity server passed to 3PID bind/unbind endpoints.
@@ -0,0 +1 @@
+Update experimental support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083): restricting room access via group membership.
@@ -0,0 +1 @@
+Replace `HomeServer.get_config()` with inline references.
@@ -0,0 +1 @@
+Rename some handlers and config modules to not duplicate the top-level module.
@@ -0,0 +1 @@
+Fix a long-standing bug which caused `max_upload_size` to not be correctly enforced.
@@ -0,0 +1 @@
+Add experimental support for handling presence on a worker.
@@ -0,0 +1 @@
+Add experimental support for handling presence on a worker.
@@ -0,0 +1 @@
+Reduce CPU usage of the user directory by reusing existing calculated room membership.
@@ -0,0 +1 @@
+Small speed up for joining large remote rooms.
@@ -0,0 +1 @@
+Add experimental support for handling presence on a worker.
@@ -0,0 +1 @@
+Don't return an error when a user attempts to renew their account multiple times with the same token. Instead, state when their account is set to expire. This change concerns the optional account validity feature.
@@ -0,0 +1 @@
+Limit the size of HTTP responses read over federation.
@@ -0,0 +1 @@
+Introduce flake8-bugbear to the test suite and fix some of its lint violations.
@@ -0,0 +1 @@
+Only store the raw data in the in-memory caches, rather than objects that include references to e.g. the data stores.
@@ -0,0 +1 @@
+Add experimental support for handling presence on a worker.
@@ -0,0 +1 @@
+Limit length of accepted email addresses.
@@ -0,0 +1 @@
+Remove redundant `synapse.types.Collection` type definition.
@@ -0,0 +1 @@
+Handle recently added rate limits correctly when using `--no-rate-limit` with the demo scripts.
@@ -0,0 +1 @@
+Fix a bug which could cause Synapse to get stuck in a loop of resyncing device lists.
@@ -0,0 +1 @@
+Disable invite rate-limiting by default when running the unit tests.
@@ -0,0 +1 @@
+Pass a reactor into `SynapseSite` to make testing easier.
@@ -0,0 +1 @@
+Make `DomainSpecificString` an `attrs` class.
@@ -0,0 +1 @@
+Add type hints to `synapse.api.auth` and `synapse.api.auth_blocking` modules.
@@ -0,0 +1 @@
+Remove redundant `_PushHTTPChannel` test class.
@@ -0,0 +1 @@
+Small performance improvement around handling new local presence updates.
@@ -56,7 +56,7 @@ services:
      - POSTGRES_USER=synapse
      - POSTGRES_PASSWORD=changeme
      # ensure the database gets created correctly
-      # https://matrix-org.github.io/synapse/latest/postgres.html#set-up-database
+      # https://github.com/matrix-org/synapse/blob/master/docs/postgres.md#set-up-database
      - POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=C --lc-ctype=C
    volumes:
      # You may store the database tables in a local folder..
@@ -46,14 +46,14 @@ class CursesStdIO:
        self.callback = callback

    def fileno(self):
-        """We want to select on FD 0"""
+        """ We want to select on FD 0 """
        return 0

    def connectionLost(self, reason):
        self.close()

    def print_line(self, text):
-        """add a line to the internal list of lines"""
+        """ add a line to the internal list of lines"""

        self.lines.append(text)
        self.redraw()
@@ -92,7 +92,7 @@ class CursesStdIO:
        )

    def doRead(self):
-        """Input is ready!"""
+        """ Input is ready! """
        curses.noecho()
        c = self.stdscr.getch()  # read a character

@@ -132,7 +132,7 @@ class CursesStdIO:
        return "CursesStdIO"

    def close(self):
-        """clean up"""
+        """ clean up """

        curses.nocbreak()
        self.stdscr.keypad(0)
@@ -224,14 +224,16 @@ class HomeServer(ReplicationHandler):
        destinations = yield self.get_servers_for_context(room_name)

        try:
-            yield self.replication_layer.send_pdu(
-                Pdu.create_new(
-                    context=room_name,
-                    pdu_type="sy.room.message",
-                    content={"sender": sender, "body": body},
-                    origin=self.server_name,
-                    destinations=destinations,
-                )
+            yield self.replication_layer.send_pdus(
+                [
+                    Pdu.create_new(
+                        context=room_name,
+                        pdu_type="sy.room.message",
+                        content={"sender": sender, "body": body},
+                        origin=self.server_name,
+                        destinations=destinations,
+                    )
+                ]
            )
        except Exception as e:
            logger.exception(e)
@@ -253,7 +255,7 @@ class HomeServer(ReplicationHandler):
                origin=self.server_name,
                destinations=destinations,
            )
-            yield self.replication_layer.send_pdu(pdu)
+            yield self.replication_layer.send_pdus([pdu])
        except Exception as e:
            logger.exception(e)

@@ -265,16 +267,18 @@ class HomeServer(ReplicationHandler):
        destinations = yield self.get_servers_for_context(room_name)

        try:
-            yield self.replication_layer.send_pdu(
-                Pdu.create_new(
-                    context=room_name,
-                    is_state=True,
-                    pdu_type="sy.room.member",
-                    state_key=invitee,
-                    content={"membership": "invite"},
-                    origin=self.server_name,
-                    destinations=destinations,
-                )
+            yield self.replication_layer.send_pdus(
+                [
+                    Pdu.create_new(
+                        context=room_name,
+                        is_state=True,
+                        pdu_type="sy.room.member",
+                        state_key=invitee,
+                        content={"membership": "invite"},
+                        origin=self.server_name,
+                        destinations=destinations,
+                    )
+                ]
            )
        except Exception as e:
            logger.exception(e)
@@ -1,6 +1,6 @@
 # Using the Synapse Grafana dashboard

 0. Set up Prometheus and Grafana. Out of scope for this readme. Useful documentation about using Grafana with Prometheus: http://docs.grafana.org/features/datasources/prometheus/
-1. Have your Prometheus scrape your Synapse. https://matrix-org.github.io/synapse/latest/metrics-howto.html
+1. Have your Prometheus scrape your Synapse. https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md
 2. Import dashboard into Grafana. Download `synapse.json`. Import it to Grafana and select the correct Prometheus datasource. http://docs.grafana.org/reference/export_import/
-3. Set up required recording rules. [contrib/prometheus](../prometheus)
+3. Set up required recording rules. https://github.com/matrix-org/synapse/tree/master/contrib/prometheus
@@ -34,7 +34,7 @@ Add a new job to the main prometheus.yml file:
 ```

 An example of a Prometheus configuration with workers can be found in
-[metrics-howto.md](https://matrix-org.github.io/synapse/latest/metrics-howto.html).
+[metrics-howto.md](https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md).

 To use `synapse.rules` add

@@ -3,9 +3,8 @@ Purge history API examples

 # `purge_history.sh`

-A bash file, that uses the
-[purge history API](https://matrix-org.github.io/synapse/latest/admin_api/purge_history_api.html)
-to purge all messages in a list of rooms up to a certain event. You can select a 
+A bash file, that uses the [purge history API](/docs/admin_api/purge_history_api.rst) to 
+purge all messages in a list of rooms up to a certain event. You can select a 
 timeframe or a number of messages that you want to keep in the room.

 Just configure the variables DOMAIN, ADMIN, ROOMS_ARRAY and TIME at the top of
@@ -13,6 +12,5 @@ the script.

 # `purge_remote_media.sh`

-A bash file, that uses the
-[purge history API](https://matrix-org.github.io/synapse/latest/admin_api/purge_history_api.html)
-to purge all old cached remote media.
+A bash file, that uses the [purge history API](/docs/admin_api/purge_history_api.rst) to 
+purge all old cached remote media.
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash

 # this script will use the api:
-#    https://matrix-org.github.io/synapse/latest/admin_api/purge_history_api.html
+#    https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst
 # 
 # It will purge all messages in a list of rooms up to a cetrain event

@@ -1,3 +1,2 @@
 The documentation for using systemd to manage synapse workers is now part of
-the main synapse distribution. See
-[docs/systemd-with-workers](https://matrix-org.github.io/synapse/latest/systemd-with-workers/index.html).
+the main synapse distribution. See [docs/systemd-with-workers](../../docs/systemd-with-workers).
@@ -2,8 +2,7 @@
 This is a setup for managing synapse with a user contributed systemd unit 
 file. It provides a `matrix-synapse` systemd unit file that should be tailored 
 to accommodate your installation in accordance with the installation 
-instructions provided in
-[installation instructions](https://matrix-org.github.io/synapse/latest/setup/installation.html).
+instructions provided in [installation instructions](../../INSTALL.md).

 ## Setup
 1. Under the service section, ensure the `User` variable matches which user
@@ -1,71 +0,0 @@
-[Service]
-# The following directives give the synapse service R/W access to:
-# - /run/matrix-synapse
-# - /var/lib/matrix-synapse
-# - /var/log/matrix-synapse
-
-RuntimeDirectory=matrix-synapse
-StateDirectory=matrix-synapse
-LogsDirectory=matrix-synapse
-
-######################
-## Security Sandbox ##
-######################
-
-# Make sure that the service has its own unshared tmpfs at /tmp and that it
-# cannot see or change any real devices
-PrivateTmp=true
-PrivateDevices=true
-
-# We give no capabilities to a service by default
-CapabilityBoundingSet=
-AmbientCapabilities=
-
-# Protect the following from modification:
-# - The entire filesystem
-# - sysctl settings and loaded kernel modules
-# - No modifications allowed to Control Groups
-# - Hostname
-# - System Clock
-ProtectSystem=strict
-ProtectKernelTunables=true
-ProtectKernelModules=true
-ProtectControlGroups=true
-ProtectClock=true
-ProtectHostname=true
-
-# Prevent access to the following:
-# - /home directory
-# - Kernel logs
-ProtectHome=tmpfs
-ProtectKernelLogs=true
-
-# Make sure that the process can only see PIDs and process details of itself,
-# and the second option disables seeing details of things like system load and
-# I/O etc
-ProtectProc=invisible
-ProcSubset=pid
-
-# While not needed, we set these options explicitly
-# - This process has been given access to the host network
-# - It can also communicate with any IP Address
-PrivateNetwork=false
-RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
-IPAddressAllow=any
-
-# Restrict system calls to a sane bunch
-SystemCallArchitectures=native
-SystemCallFilter=@system-service
-SystemCallFilter=~@privileged @resources @obsolete
-
-# Misc restrictions
-# - Since the process is a python process it needs to be able to write and
-#   execute memory regions, so we set MemoryDenyWriteExecute to false
-RestrictSUIDSGID=true
-RemoveIPC=true
-NoNewPrivileges=true
-RestrictRealtime=true
-RestrictNamespaces=true
-LockPersonality=true
-PrivateUsers=true
-MemoryDenyWriteExecute=false
@@ -33,11 +33,13 @@ esac
 # Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather
 # than the 2/3 compatible `virtualenv`.

+# Pin pip to 20.3.4 to fix breakage in 21.0 on py3.5 (xenial)
+
 dh_virtualenv \
    --install-suffix "matrix-synapse" \
    --builtin-venv \
    --python "$SNAKE" \
-    --upgrade-pip \
+    --upgrade-pip-to="20.3.4" \
    --preinstall="lxml" \
    --preinstall="mock" \
    --extra-pip-arg="--no-cache-dir" \
@@ -100,18 +102,3 @@ esac
 # add a dependency on the right version of python to substvars.
 PYPKG=`basename $SNAKE`
 echo "synapse:pydepends=$PYPKG" >> debian/matrix-synapse-py3.substvars
-
-
-# add a couple of triggers.  This is needed so that dh-virtualenv can rebuild
-# the venv when the system python changes (see
-# https://dh-virtualenv.readthedocs.io/en/latest/tutorial.html#step-2-set-up-packaging-for-your-project)
-#
-# we do it here rather than the more conventional way of just adding it to
-# debian/matrix-synapse-py3.triggers, because we need to add a trigger on the
-# right version of python.
-cat >>"debian/.debhelper/generated/matrix-synapse-py3/triggers" <<EOF
-# triggers for dh-virtualenv
-interest-noawait $SNAKE
-interest dh-virtualenv-interpreter-update
-
-EOF
--- a/Show More
+++ b/Show More
				`@@ -1 +0,0 @@`
				`Add support for [MSC3231 - Token authenticated registration](https://github.com/matrix-org/matrix-doc/pull/3231). Users can be required to submit a token during registration to authenticate themselves. Contributed by Callum Brown.`
				`@@ -1 +0,0 @@`
				`Add documentation on how to connect Django with synapse using oidc and django-oauth-toolkit. Contributed by @HugoDelval.`
				`@@ -1 +0,0 @@`
				`Add support for [MSC3283](https://github.com/matrix-org/matrix-doc/pull/3283): Expose enable_set_displayname in capabilities.`
				`@@ -1 +0,0 @@`
				`Port the PresenceRouter module interface to the new generic interface.`
				`@@ -1 +0,0 @@`
				`Display an error on User-Interactive Authentication fallback pages when authentication fails. Contributed by Callum Brown.`
				`@@ -1 +0,0 @@`
				Reject Client-Server `/keys/query` requests which provide `device_ids` incorrectly.
				`@@ -1 +0,0 @@`
				`Improve type hints for the proxy agent and SRV resolver modules. Contributed by @dklimpel.`
				`@@ -1 +0,0 @@`
				`Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).`
				`@@ -1 +0,0 @@`
				`Clean up some of the federation event authentication code for clarity.`
				`@@ -1 +0,0 @@`
				`Remove not needed database updates in modify user admin API.`