Basic release script

.buildkite/postgres-config.yaml

@@ -3,7 +3,7 @@
 # CI's Docker setup at the point where this file is considered.
 server_name: "localhost:8800"
 
-signing_key_path: ".buildkite/test.signing.key"
+signing_key_path: "/src/.buildkite/test.signing.key"
 
 report_stats: false
 
@@ -16,4 +16,6 @@ database:
     database: synapse
 
 # Suppress the key server warning.
-trusted_key_servers: []
+trusted_key_servers:
+  - server_name: "matrix.org"
+suppress_key_server_warning: true

.buildkite/scripts/create_postgres_db.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from synapse.storage.engines import create_engine
+
+logger = logging.getLogger("create_postgres_db")
+
+if __name__ == "__main__":
+    # Create a PostgresEngine.
+    db_engine = create_engine({"name": "psycopg2", "args": {}})
+
+    # Connect to postgres to create the base database.
+    # We use "postgres" as a database because it's bound to exist and the "synapse" one
+    # doesn't exist yet.
+    db_conn = db_engine.module.connect(
+        user="postgres", host="postgres", password="postgres", dbname="postgres"
+    )
+    db_conn.autocommit = True
+    cur = db_conn.cursor()
+    cur.execute("CREATE DATABASE synapse;")
+    cur.close()
+    db_conn.close()

.buildkite/scripts/postgres_exec.py

@@ -1,31 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import sys
-
-import psycopg2
-
-# a very simple replacment for `psql`, to make up for the lack of the postgres client
-# libraries in the synapse docker image.
-
-# We use "postgres" as a database because it's bound to exist and the "synapse" one
-# doesn't exist yet.
-db_conn = psycopg2.connect(
-    user="postgres", host="postgres", password="postgres", dbname="postgres"
-)
-db_conn.autocommit = True
-cur = db_conn.cursor()
-for c in sys.argv[1:]:
-    cur.execute(c)

.buildkite/scripts/test_old_deps.sh

@@ -1,16 +1,13 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
-# this script is run by buildkite in a plain `bionic` container; it installs the
-# minimal requirements for tox and hands over to the py3-old tox environment.
+# this script is run by buildkite in a plain `xenial` container; it installs the
+# minimal requirements for tox and hands over to the py35-old tox environment.
 
 set -ex
 
 apt-get update
-apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox
+apt-get install -y python3.5 python3.5-dev python3-pip libxml2-dev libxslt-dev zlib1g-dev tox
 
 export LANG="C.UTF-8"
 
-# Prevent virtualenv from auto-updating pip to an incompatible version
-export VIRTUALENV_NO_DOWNLOAD=1
-
-exec tox -e py3-old,combine
+exec tox -e py35-old,combine

.buildkite/scripts/test_synapse_port_db.sh

@@ -1,10 +1,10 @@
-#!/usr/bin/env bash
+#!/bin/bash
 #
-# Test script for 'synapse_port_db'.
-#   - sets up synapse and deps
-#   - runs the port script on a prepopulated test sqlite db
-#   - also runs it against an new sqlite db
-
+# Test script for 'synapse_port_db', which creates a virtualenv, installs Synapse along
+# with additional dependencies needed for the test (such as coverage or the PostgreSQL
+# driver), update the schema of the test SQLite database and run background updates on it,
+# create an empty test database in PostgreSQL, then run the 'synapse_port_db' script to
+# test porting the SQLite database to the PostgreSQL database (with coverage).
 
 set -xe
 cd `dirname $0`/../..
@@ -22,36 +22,15 @@ echo "--- Generate the signing key"
 # Generate the server's signing key.
 python -m synapse.app.homeserver --generate-keys -c .buildkite/sqlite-config.yaml
 
-echo "--- Prepare test database"
+echo "--- Prepare the databases"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
 scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
 
 # Create the PostgreSQL database.
-./.buildkite/scripts/postgres_exec.py "CREATE DATABASE synapse"
+./.buildkite/scripts/create_postgres_db.py
 
-echo "+++ Run synapse_port_db against test database"
-coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
-
-# We should be able to run twice against the same database.
-echo "+++ Run synapse_port_db a second time"
-coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
-
-#####
-
-# Now do the same again, on an empty database.
-
-echo "--- Prepare empty SQLite database"
-
-# we do this by deleting the sqlite db, and then doing the same again.
-rm .buildkite/test_db.db
-
-scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
-
-# re-create the PostgreSQL database.
-./.buildkite/scripts/postgres_exec.py \
-  "DROP DATABASE synapse" \
-  "CREATE DATABASE synapse"
-
-echo "+++ Run synapse_port_db against empty database"
+echo "+++ Run synapse_port_db"
+
+# Run the script
 coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml

.buildkite/sqlite-config.yaml

@@ -3,7 +3,7 @@
 # schema and run background updates on it.
 server_name: "localhost:8800"
 
-signing_key_path: ".buildkite/test.signing.key"
+signing_key_path: "/src/.buildkite/test.signing.key"
 
 report_stats: false
 
@@ -13,4 +13,6 @@ database:
     database: ".buildkite/test_db.db"
 
 # Suppress the key server warning.
-trusted_key_servers: []
+trusted_key_servers:
+  - server_name: "matrix.org"
+suppress_key_server_warning: true

.circleci/config.yml

@@ -0,0 +1,75 @@
+version: 2.1
+jobs:
+  dockerhubuploadrelease:
+    docker:
+      - image: docker:git
+    steps:
+      - checkout
+      - setup_remote_docker
+      - docker_prepare
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      - docker_build:
+          tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
+          platforms: linux/amd64
+      - docker_build:
+          tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
+          platforms: linux/amd64,linux/arm/v7,linux/arm64
+
+  dockerhubuploadlatest:
+    docker:
+      - image: docker:git
+    steps:
+      - checkout
+      - setup_remote_docker
+      - docker_prepare
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      - docker_build:
+          tag: -t matrixdotorg/synapse:latest
+          platforms: linux/amd64
+      - docker_build:
+          tag: -t matrixdotorg/synapse:latest
+          platforms: linux/amd64,linux/arm/v7,linux/arm64
+
+workflows:
+  build:
+    jobs:
+      - dockerhubuploadrelease:
+          filters:
+            tags:
+              only: /v[0-9].[0-9]+.[0-9]+.*/
+            branches:
+              ignore: /.*/
+      - dockerhubuploadlatest:
+          filters:
+            branches:
+              only: master
+
+commands:
+  docker_prepare:
+    description: Downloads the buildx cli plugin and enables multiarch images
+    parameters:
+      buildx_version:
+        type: string
+        default: "v0.4.1"
+    steps:
+      - run: apk add --no-cache curl
+      - run: mkdir -vp ~/.docker/cli-plugins/ ~/dockercache
+      - run: curl --silent -L "https://github.com/docker/buildx/releases/download/<< parameters.buildx_version >>/buildx-<< parameters.buildx_version >>.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
+      - run: chmod a+x ~/.docker/cli-plugins/docker-buildx
+      # install qemu links in /proc/sys/fs/binfmt_misc on the docker instance running the circleci job
+      - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+      # create a context named `builder` for the builds
+      - run: docker context create builder
+      # create a buildx builder using the new context, and set it as the default
+      - run: docker buildx create builder --use
+
+  docker_build:
+    description: Builds and pushed images to dockerhub using buildx
+    parameters:
+      platforms:
+        type: string
+        default: linux/amd64
+      tag:
+        type: string
+    steps:
+      - run: docker buildx build -f docker/Dockerfile --push --platform << parameters.platforms >> --label gitsha1=${CIRCLE_SHA1} << parameters.tag >> --progress=plain .

.git-blame-ignore-revs

@@ -1,8 +0,0 @@
-# Black reformatting (#5482).
-32e7c9e7f20b57dd081023ac42d6931a8da9b3a3
-
-# Target Python 3.5 with black (#8664).
-aff1eb7c671b0a3813407321d2702ec46c71fa56
-
-# Update black to 20.8b1 (#9381).
-0a00b7ff14890987f09112a2ae696c61001e6cf1

.github/workflows/docker.yml

@@ -1,72 +0,0 @@
-# GitHub actions workflow which builds and publishes the docker images.
-
-name: Build docker images
-
-on:
-  push:
-    tags: ["v*"]
-    branches: [ master, main ]
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up QEMU
-        id: qemu
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: arm64
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Inspect builder
-        run: docker buildx inspect
-          
-      - name: Log in to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Calculate docker image tag
-        id: set-tag
-        run: |
-          case "${GITHUB_REF}" in
-              refs/heads/master|refs/heads/main)
-                  tag=latest
-                  ;;
-              refs/tags/*)
-                  tag=${GITHUB_REF#refs/tags/}
-                  ;;
-              *)
-                  tag=${GITHUB_SHA}
-                  ;;
-          esac
-          echo "::set-output name=tag::$tag"
-
-        # for release builds, we want to get the amd64 image out asap, so first
-        # we do an amd64-only build, before following up with a multiarch build.
-      - name: Build and push amd64
-        uses: docker/build-push-action@v2
-        if: "${{ startsWith(github.ref, 'refs/tags/v') }}"
-        with:
-          push: true
-          labels: "gitsha1=${{ github.sha }}"
-          tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
-          file: "docker/Dockerfile"
-          platforms: linux/amd64
-
-      - name: Build and push all platforms
-        uses: docker/build-push-action@v2
-        with:
-          push: true
-          labels: "gitsha1=${{ github.sha }}"
-          tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
-          file: "docker/Dockerfile"
-          platforms: linux/amd64,linux/arm64

.github/workflows/docs.yaml

@@ -1,66 +0,0 @@
-name: Deploy the documentation
-
-on:
-  push:
-    branches:
-      # For bleeding-edge documentation
-      - develop
-      # For documentation specific to a release
-      - 'release-v*'
-      # stable docs
-      - master
-
-  workflow_dispatch:
-
-jobs:
-  pages:
-    name: GitHub Pages
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Setup mdbook
-        uses: peaceiris/actions-mdbook@4b5ef36b314c2599664ca107bb8c02412548d79d # v1.1.14
-        with:
-          mdbook-version: '0.4.9'
-
-      - name: Build the documentation
-        # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
-        # However, we're using docs/README.md for other purposes and need to pick a new page
-        # as the default. Let's opt for the welcome page instead.
-        run: |
-          mdbook build
-          cp book/welcome_and_overview.html book/index.html
-
-      # Figure out the target directory.
-      #
-      # The target directory depends on the name of the branch
-      #
-      - name: Get the target directory name
-        id: vars
-        run: |
-          # first strip the 'refs/heads/' prefix with some shell foo
-          branch="${GITHUB_REF#refs/heads/}"
-
-          case $branch in
-              release-*)
-                  # strip 'release-' from the name for release branches.
-                  branch="${branch#release-}"
-                  ;;
-              master)
-                  # deploy to "latest" for the master branch.
-                  branch="latest"
-                  ;;
-          esac
-
-          # finally, set the 'branch-version' var.
-          echo "::set-output name=branch-version::$branch"
-          
-      # Deploy to the target directory.
-      - name: Deploy to gh pages
-        uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          keep_files: true
-          publish_dir: ./book
-          destination_dir: ./${{ steps.vars.outputs.branch-version }}

.github/workflows/release-artifacts.yml

@@ -1,130 +0,0 @@
-# GitHub actions workflow which builds the release artifacts.
-
-name: Build release artifacts
-
-on:
-  # we build on PRs and develop to (hopefully) get early warning
-  # of things breaking (but only build one set of debs)
-  pull_request:
-  push:
-    branches: ["develop"]
-
-    # we do the full build on tags.
-    tags: ["v*"]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-  
-permissions:
-  contents: write
-
-jobs:
-  get-distros:
-    name: "Calculate list of debian distros"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - id: set-distros
-        run: |
-          # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
-          dists='["debian:sid"]'
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-              dists=$(scripts-dev/build_debian_packages --show-dists-json)
-          fi
-          echo "::set-output name=distros::$dists"
-    # map the step outputs to job outputs
-    outputs:
-      distros: ${{ steps.set-distros.outputs.distros }}
-
-  # now build the packages with a matrix build.
-  build-debs:
-    needs: get-distros
-    name: "Build .deb packages"
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        distro: ${{ fromJson(needs.get-distros.outputs.distros) }}
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          path: src
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          install: true
-
-      - name: Set up docker layer caching
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
-
-      - name: Set up python
-        uses: actions/setup-python@v2
-
-      - name: Build the packages
-        # see https://github.com/docker/build-push-action/issues/252
-        # for the cache magic here
-        run: |
-          ./src/scripts-dev/build_debian_packages \
-            --docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \
-            --docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
-            --docker-build-arg=--progress=plain \
-            --docker-build-arg=--load \
-            "${{ matrix.distro }}"
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
-
-      - name: Upload debs as artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: debs
-          path: debs/*
-
-  build-sdist:
-    name: "Build pypi distribution files"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install wheel
-      - run: |
-          python setup.py sdist bdist_wheel
-      - uses: actions/upload-artifact@v2
-        with:
-          name: python-dist
-          path: dist/*
-
-  # if it's a tag, create a release and attach the artifacts to it
-  attach-assets:
-    name: "Attach assets to release"
-    if: ${{ !failure() && !cancelled() && startsWith(github.ref, 'refs/tags/') }}
-    needs:
-      - build-debs
-      - build-sdist
-    runs-on: ubuntu-latest
-    steps:
-      - name: Download all workflow run artifacts
-        uses: actions/download-artifact@v2
-      - name: Build a tarball for the debs
-        run: tar -cvJf debs.tar.xz debs
-      - name: Attach to release
-        uses: softprops/action-gh-release@a929a66f232c1b11af63782948aa2210f981808a  # PR#109
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          files: |
-            python-dist/*
-            debs.tar.xz
-          # if it's not already published, keep the release as a draft.
-          draft: true
-          # mark it as a prerelease if the tag contains 'rc'.
-          prerelease: ${{ contains(github.ref, 'rc') }}

.github/workflows/tests.yml

@@ -1,379 +0,0 @@
-name: Tests
-
-on:
-  push:
-    branches: ["develop", "release-*"]
-  pull_request:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-  
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        toxenv:
-          - "check-sampleconfig"
-          - "check_codestyle"
-          - "check_isort"
-          - "mypy"
-          - "packaging"
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install tox
-      - run: tox -e ${{ matrix.toxenv }}
-
-  lint-crlf:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Check line endings
-        run: scripts-dev/check_line_terminators.sh
-
-  lint-newsfile:
-    if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
-    runs-on: ubuntu-latest
-    steps:
-      # Note: This and the script can be simplified once we drop Buildkite. See:
-      #   https://github.com/actions/checkout/issues/266#issuecomment-638346893
-      #   https://github.com/actions/checkout/issues/416
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
-      - uses: actions/setup-python@v2
-      - run: pip install tox
-      - name: Patch Buildkite-specific test script
-        run: |
-          sed -i -e 's/\$BUILDKITE_PULL_REQUEST/${{ github.event.number }}/' \
-            scripts-dev/check-newsfragment
-      - run: scripts-dev/check-newsfragment
-
-  lint-sdist:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: "3.x"
-      - run: pip install wheel
-      - run: python setup.py sdist bdist_wheel
-      - uses: actions/upload-artifact@v2
-        with:
-          name: Python Distributions
-          path: dist/*
-
-  # Dummy step to gate other tests on without repeating the whole list
-  linting-done:
-    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    needs: [lint, lint-crlf, lint-newsfile, lint-sdist]
-    runs-on: ubuntu-latest
-    steps:
-      - run: "true"
-
-  trial:
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9"]
-        database: ["sqlite"]
-        include:
-          # Newest Python without optional deps
-          - python-version: "3.9"
-            toxenv: "py-noextras,combine"
-
-          # Oldest Python with PostgreSQL
-          - python-version: "3.6"
-            database: "postgres"
-            postgres-version: "9.6"
-
-          # Newest Python with PostgreSQL
-          - python-version: "3.9"
-            database: "postgres"
-            postgres-version: "13"
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
-        if: ${{ matrix.postgres-version }}
-        run: |
-          docker run -d -p 5432:5432 \
-            -e POSTGRES_PASSWORD=postgres \
-            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
-            postgres:${{ matrix.postgres-version }}
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - run: pip install tox
-      - name: Await PostgreSQL
-        if: ${{ matrix.postgres-version }}
-        timeout-minutes: 2
-        run: until pg_isready -h localhost; do sleep 1; done
-      - run: tox -e py,combine
-        env:
-          TRIAL_FLAGS: "--jobs=2"
-          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
-          SYNAPSE_POSTGRES_HOST: localhost
-          SYNAPSE_POSTGRES_USER: postgres
-          SYNAPSE_POSTGRES_PASSWORD: postgres
-      - name: Dump logs
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  trial-olddeps:
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Test with old deps
-        uses: docker://ubuntu:bionic # For old python and sqlite
-        with:
-          workdir: /github/workspace
-          entrypoint: .buildkite/scripts/test_old_deps.sh
-        env:
-          TRIAL_FLAGS: "--jobs=2"
-      - name: Dump logs
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  trial-pypy:
-    # Very slow; only run if the branch name includes 'pypy'
-    if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["pypy-3.6"]
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - run: pip install tox
-      - run: tox -e py,combine
-        env:
-          TRIAL_FLAGS: "--jobs=2"
-      - name: Dump logs
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  sytest:
-    if: ${{ !failure() && !cancelled() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    container:
-      image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
-      volumes:
-        - ${{ github.workspace }}:/src
-      env:
-        BUILDKITE_BRANCH: ${{ github.head_ref }}
-        POSTGRES: ${{ matrix.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
-        WORKERS: ${{ matrix.workers && 1 }}
-        REDIS: ${{ matrix.redis && 1 }}
-        BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - sytest-tag: bionic
-
-          - sytest-tag: bionic
-            postgres: postgres
-
-          - sytest-tag: testing
-            postgres: postgres
-
-          - sytest-tag: bionic
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: postgres
-            workers: workers
-            redis: redis
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Prepare test blacklist
-        run: cat sytest-blacklist .buildkite/worker-blacklist > synapse-blacklist-with-workers
-      - name: Run SyTest
-        run: /bootstrap.sh synapse
-        working-directory: /src
-      - name: Summarise results.tap
-        if: ${{ always() }}
-        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
-      - name: Upload SyTest logs
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
-        with:
-          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
-          path: |
-            /logs/results.tap
-            /logs/**/*.log*
-
-  portdb:
-    if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        include:
-          - python-version: "3.6"
-            postgres-version: "9.6"
-
-          - python-version: "3.9"
-            postgres-version: "13"
-
-    services:
-      postgres:
-        image: postgres:${{ matrix.postgres-version }}
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_PASSWORD: "postgres"
-          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Patch Buildkite-specific test scripts
-        run: |
-          sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/postgres_exec.py
-          sed -i -e 's/host: postgres/host: localhost/' .buildkite/postgres-config.yaml
-          sed -i -e 's|/src/||' .buildkite/{sqlite,postgres}-config.yaml
-          sed -i -e 's/\$TOP/\$GITHUB_WORKSPACE/' .coveragerc
-      - run: .buildkite/scripts/test_synapse_port_db.sh
-
-  complement:
-    if: ${{ !failure() && !cancelled() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    container:
-      # https://github.com/matrix-org/complement/blob/master/dockerfiles/ComplementCIBuildkite.Dockerfile
-      image: matrixdotorg/complement:latest
-      env:
-        CI: true
-      ports:
-        - 8448:8448
-      volumes:
-        - /var/run/docker.sock:/var/run/docker.sock
-
-    steps:
-      - name: Run actions/checkout@v2 for synapse
-        uses: actions/checkout@v2
-        with:
-          path: synapse
-
-      # Attempt to check out the same branch of Complement as the PR. If it
-      # doesn't exist, fallback to master.
-      - name: Checkout complement
-        shell: bash
-        run: |
-          mkdir -p complement
-          # Attempt to use the version of complement which best matches the current
-          # build. Depending on whether this is a PR or release, etc. we need to
-          # use different fallbacks.
-          #
-          # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
-          #    for pull requests, otherwise GITHUB_REF).
-          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
-          #    (GITHUB_BASE_REF for pull requests).
-          # 3. Use the default complement branch ("master").
-          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
-            # Skip empty branch names and merge commits.
-            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
-              continue
-            fi
-
-            (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
-          done
-
-      # Build initial Synapse image
-      - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
-        working-directory: synapse
-
-      # Build a ready-to-run Synapse image based on the initial image above.
-      # This new image includes a config file, keys for signing and TLS, and
-      # other settings to make it suitable for testing under Complement.
-      - run: docker build -t complement-synapse -f Synapse.Dockerfile .
-        working-directory: complement/dockerfiles
-
-      # Run Complement
-      - run: go test -v -tags synapse_blacklist,msc2403,msc2946,msc3083 ./tests/...
-        env:
-          COMPLEMENT_BASE_IMAGE: complement-synapse:latest
-        working-directory: complement
-
-  # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
-  tests-done:
-    if: ${{ always() }}
-    needs:
-      - lint
-      - lint-crlf
-      - lint-newsfile
-      - lint-sdist
-      - trial
-      - trial-olddeps
-      - sytest
-      - portdb
-      - complement
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set build result
-        env:
-          NEEDS_CONTEXT: ${{ toJSON(needs) }}
-        # the `jq` incantation dumps out a series of "<job> <result>" lines
-        run: |
-          set -o pipefail
-          jq -r 'to_entries[] | [.key,.value.result] | join(" ")' \
-                          <<< $NEEDS_CONTEXT |
-              while read job result; do
-                  if [ "$result" != "success" ]; then
-                      echo "::set-failed ::Job $job returned $result"
-                  fi
-              done

.gitignore

@@ -6,19 +6,16 @@
 *.egg
 *.egg-info
 *.lock
-*.py[cod]
+*.pyc
 *.snap
 *.tac
 _trial_temp/
 _trial_temp*/
 /out
-.DS_Store
-__pycache__/
 
 # stuff that is likely to exist when you run a server locally
 /*.db
 /*.log
-/*.log.*
 /*.log.config
 /*.pid
 /.python-version
@@ -46,6 +43,3 @@ __pycache__/
 /docs/build/
 /htmlcov
 /pip-wheel-metadata/
-
-# docs
-book/

CONTRIBUTING.md

@@ -1,31 +1,4 @@
-Welcome to Synapse
-
-This document aims to get you started with contributing to this repo! 
-
-- [1. Who can contribute to Synapse?](#1-who-can-contribute-to-synapse)
-- [2. What do I need?](#2-what-do-i-need)
-- [3. Get the source.](#3-get-the-source)
-- [4. Install the dependencies](#4-install-the-dependencies)
-  * [Under Unix (macOS, Linux, BSD, ...)](#under-unix-macos-linux-bsd-)
-  * [Under Windows](#under-windows)
-- [5. Get in touch.](#5-get-in-touch)
-- [6. Pick an issue.](#6-pick-an-issue)
-- [7. Turn coffee and documentation into code and documentation!](#7-turn-coffee-and-documentation-into-code-and-documentation)
-- [8. Test, test, test!](#8-test-test-test)
-  * [Run the linters.](#run-the-linters)
-  * [Run the unit tests.](#run-the-unit-tests)
-  * [Run the integration tests.](#run-the-integration-tests)
-- [9. Submit your patch.](#9-submit-your-patch)
-  * [Changelog](#changelog)
-    + [How do I know what to call the changelog file before I create the PR?](#how-do-i-know-what-to-call-the-changelog-file-before-i-create-the-pr)
-    + [Debian changelog](#debian-changelog)
-  * [Sign off](#sign-off)
-- [10. Turn feedback into better code.](#10-turn-feedback-into-better-code)
-- [11. Find a new issue.](#11-find-a-new-issue)
-- [Notes for maintainers on merging PRs etc](#notes-for-maintainers-on-merging-prs-etc)
-- [Conclusion](#conclusion)
-
-# 1. Who can contribute to Synapse?
+# Contributing code to Synapse
 
 Everyone is welcome to contribute code to [matrix.org
 projects](https://github.com/matrix-org), provided that they are willing to
@@ -36,186 +9,70 @@ license the code under the same terms as the project's overall 'outbound'
 license - in our case, this is almost always Apache Software License v2 (see
 [LICENSE](LICENSE)).
 
-# 2. What do I need?
-
-The code of Synapse is written in Python 3. To do pretty much anything, you'll need [a recent version of Python 3](https://wiki.python.org/moin/BeginnersGuide/Download).
-
-The source code of Synapse is hosted on GitHub. You will also need [a recent version of git](https://github.com/git-guides/install-git).
-
-For some tests, you will need [a recent version of Docker](https://docs.docker.com/get-docker/).
-
-
-# 3. Get the source.
+## How to contribute
 
 The preferred and easiest way to contribute changes is to fork the relevant
-project on GitHub, and then [create a pull request](
+project on github, and then [create a pull request](
 https://help.github.com/articles/using-pull-requests/) to ask us to pull your
 changes into our repo.
 
-Please base your changes on the `develop` branch.
+Some other points to follow:
 
-```sh
-git clone git@github.com:YOUR_GITHUB_USER_NAME/synapse.git
-git checkout develop
-```
+ * Please base your changes on the `develop` branch.
 
-If you need help getting started with git, this is beyond the scope of the document, but you
-can find many good git tutorials on the web.
+ * Please follow the [code style requirements](#code-style).
 
-# 4. Install the dependencies
+ * Please include a [changelog entry](#changelog) with each PR.
 
-## Under Unix (macOS, Linux, BSD, ...)
+ * Please [sign off](#sign-off) your contribution.
 
-Once you have installed Python 3 and added the source, please open a terminal and
-setup a *virtualenv*, as follows:
+ * Please keep an eye on the pull request for feedback from the [continuous
+   integration system](#continuous-integration-and-testing) and try to fix any
+   errors that come up.
 
-```sh
-cd path/where/you/have/cloned/the/repository
-python3 -m venv ./env
-source ./env/bin/activate
-pip install -e ".[all,lint,mypy,test]"
-pip install tox
-```
+ * If you need to [update your PR](#updating-your-pull-request), just add new
+   commits to your branch rather than rebasing.
 
-This will install the developer dependencies for the project.
-
-## Under Windows
-
-TBD
-
-
-# 5. Get in touch.
-
-Join our developer community on Matrix: #synapse-dev:matrix.org !
-
-
-# 6. Pick an issue.
-
-Fix your favorite problem or perhaps find a [Good First Issue](https://github.com/matrix-org/synapse/issues?q=is%3Aopen+is%3Aissue+label%3A%22Good+First+Issue%22)
-to work on.
-
-
-# 7. Turn coffee and documentation into code and documentation!
+## Code style
 
 Synapse's code style is documented [here](docs/code_style.md). Please follow
 it, including the conventions for the [sample configuration
 file](docs/code_style.md#configuration-file-format).
 
-There is a growing amount of documentation located in the [docs](docs)
-directory. This documentation is intended primarily for sysadmins running their
-own Synapse instance, as well as developers interacting externally with
-Synapse. [docs/dev](docs/dev) exists primarily to house documentation for
-Synapse developers. [docs/admin_api](docs/admin_api) houses documentation
-regarding Synapse's Admin API, which is used mostly by sysadmins and external
-service developers.
+Many of the conventions are enforced by scripts which are run as part of the
+[continuous integration system](#continuous-integration-and-testing). To help
+check if you have followed the code style, you can run `scripts-dev/lint.sh`
+locally. You'll need python 3.6 or later, and to install a number of tools:
 
-If you add new files added to either of these folders, please use [GitHub-Flavoured
-Markdown](https://guides.github.com/features/mastering-markdown/).
+```
+# Install the dependencies
+pip install -e ".[lint,mypy]"
 
-Some documentation also exists in [Synapse's GitHub
-Wiki](https://github.com/matrix-org/synapse/wiki), although this is primarily
-contributed to by community authors.
-
-
-# 8. Test, test, test!
-<a name="test-test-test"></a>
-
-While you're developing and before submitting a patch, you'll
-want to test your code.
-
-## Run the linters.
-
-The linters look at your code and do two things:
-
-- ensure that your code follows the coding style adopted by the project;
-- catch a number of errors in your code.
-
-They're pretty fast, don't hesitate!
-
-```sh
-source ./env/bin/activate
+# Run the linter script
 ./scripts-dev/lint.sh
 ```
 
-Note that this script *will modify your files* to fix styling errors.
-Make sure that you have saved all your files.
+**Note that the script does not just test/check, but also reformats code, so you
+may wish to ensure any new code is committed first**.
 
-If you wish to restrict the linters to only the files changed since the last commit
-(much faster!), you can instead run:
+By default, this script checks all files and can take some time; if you alter
+only certain files, you might wish to specify paths as arguments to reduce the
+run-time:
 
-```sh
-source ./env/bin/activate
-./scripts-dev/lint.sh -d
 ```
-
-Or if you know exactly which files you wish to lint, you can instead run:
-
-```sh
-source ./env/bin/activate
 ./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder
 ```
 
-## Run the unit tests.
+You can also provide the `-d` option, which will lint the files that have been
+changed since the last git commit. This will often be significantly faster than
+linting the whole codebase.
 
-The unit tests run parts of Synapse, including your changes, to see if anything
-was broken. They are slower than the linters but will typically catch more errors.
-
-```sh
-source ./env/bin/activate
-trial tests
-```
-
-If you wish to only run *some* unit tests, you may specify
-another module instead of `tests` - or a test class or a method:
-
-```sh
-source ./env/bin/activate
-trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite
-```
-
-If your tests fail, you may wish to look at the logs (the default log level is `ERROR`):
-
-```sh
-less _trial_temp/test.log
-```
-
-To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`:
-
-```sh
-SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests
-```
-
-
-## Run the integration tests.
-
-The integration tests are a more comprehensive suite of tests. They
-run a full version of Synapse, including your changes, to check if
-anything was broken. They are slower than the unit tests but will
-typically catch more errors.
-
-The following command will let you run the integration test with the most common
-configuration:
-
-```sh
-$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:py37
-```
-
-This configuration should generally cover  your needs. For more details about other configurations, see [documentation in the SyTest repo](https://github.com/matrix-org/sytest/blob/develop/docker/README.md).
-
-
-# 9. Submit your patch.
-
-Once you're happy with your patch, it's time to prepare a Pull Request.
-
-To prepare a Pull Request, please:
-
-1. verify that [all the tests pass](#test-test-test), including the coding style;
-2. [sign off](#sign-off) your contribution;
-3. `git push` your commit to your fork of Synapse;
-4. on GitHub, [create the Pull Request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request);
-5. add a [changelog entry](#changelog) and push it to your Pull Request;
-6. for most contributors, that's all - however, if you are a member of the organization `matrix-org`, on GitHub, please request a review from `matrix.org / Synapse Core`.
+Before pushing new changes, ensure they don't produce linting errors. Commit any
+files that were corrected.
 
+Please ensure your changes match the cosmetic style of the existing project,
+and **never** mix cosmetic and functional changes in the same commit, as it
+makes it horribly hard to review otherwise.
 
 ## Changelog
 
@@ -365,36 +222,47 @@ Git allows you to add this signoff automatically when using the `-s`
 flag to `git commit`, which uses the name and email set in your
 `user.name` and `user.email` git configs.
 
+## Continuous integration and testing
 
-# 10. Turn feedback into better code.
+[Buildkite](https://buildkite.com/matrix-dot-org/synapse) will automatically
+run a series of checks and tests against any PR which is opened against the
+project; if your change breaks the build, this will be shown in GitHub, with
+links to the build results. If your build fails, please try to fix the errors
+and update your branch.
 
-Once the Pull Request is opened, you will see a few things:
+To run unit tests in a local development environment, you can use:
 
-1. our automated CI (Continuous Integration) pipeline will run (again) the linters, the unit tests, the integration tests and more;
-2. one or more of the developers will take a look at your Pull Request and offer feedback.
+- ``tox -e py35`` (requires tox to be installed by ``pip install tox``)
+  for SQLite-backed Synapse on Python 3.5.
+- ``tox -e py36`` for SQLite-backed Synapse on Python 3.6.
+- ``tox -e py36-postgres`` for PostgreSQL-backed Synapse on Python 3.6
+  (requires a running local PostgreSQL with access to create databases).
+- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 3.5
+  (requires Docker). Entirely self-contained, recommended if you don't want to
+  set up PostgreSQL yourself.
 
-From this point, you should:
+Docker images are available for running the integration tests (SyTest) locally,
+see the [documentation in the SyTest repo](
+https://github.com/matrix-org/sytest/blob/develop/docker/README.md) for more
+information.
 
-1. Look at the results of the CI pipeline.
-   - If there is any error, fix the error.
-2. If a developer has requested changes, make these changes and let us know if it is ready for a developer to review again.
-3. Create a new commit with the changes.
-   - Please do NOT overwrite the history. New commits make the reviewer's life easier.
-   - Push this commits to your Pull Request.
-4. Back to 1.
+## Updating your pull request
 
-Once both the CI and the developers are happy, the patch will be merged into Synapse and released shortly!
+If you decide to make changes to your pull request - perhaps to address issues
+raised in a review, or to fix problems highlighted by [continuous
+integration](#continuous-integration-and-testing) - just add new commits to your
+branch, and push to GitHub. The pull request will automatically be updated.
 
-# 11. Find a new issue.
+Please **avoid** rebasing your branch, especially once the PR has been
+reviewed: doing so makes it very difficult for a reviewer to see what has
+changed since a previous review.
 
-By now, you know the drill!
-
-# Notes for maintainers on merging PRs etc
+## Notes for maintainers on merging PRs etc
 
 There are some notes for those with commit access to the project on how we
-manage git [here](docs/development/git.md).
+manage git [here](docs/dev/git.md).
 
-# Conclusion
+## Conclusion
 
 That's it! Matrix is a very open and collaborative project as you might expect
 given our obsession with open communication. If we're going to successfully

INSTALL.md

@@ -1,7 +1,577 @@
-# Installation Instructions
+- [Choosing your server name](#choosing-your-server-name)
+- [Picking a database engine](#picking-a-database-engine)
+- [Installing Synapse](#installing-synapse)
+  - [Installing from source](#installing-from-source)
+    - [Platform-Specific Instructions](#platform-specific-instructions)
+  - [Prebuilt packages](#prebuilt-packages)
+- [Setting up Synapse](#setting-up-synapse)
+  - [TLS certificates](#tls-certificates)
+  - [Client Well-Known URI](#client-well-known-uri)
+  - [Email](#email)
+  - [Registering a user](#registering-a-user)
+  - [Setting up a TURN server](#setting-up-a-turn-server)
+  - [URL previews](#url-previews)
+- [Troubleshooting Installation](#troubleshooting-installation)
 
-This document has moved to the
-[Synapse documentation website](https://matrix-org.github.io/synapse/latest/setup/installation.html).
-Please update your links.
+# Choosing your server name
 
-The markdown source is available in [docs/setup/installation.md](docs/setup/installation.md).
+It is important to choose the name for your server before you install Synapse,
+because it cannot be changed later.
+
+The server name determines the "domain" part of user-ids for users on your
+server: these will all be of the format `@user:my.domain.name`. It also
+determines how other matrix servers will reach yours for federation.
+
+For a test configuration, set this to the hostname of your server. For a more
+production-ready setup, you will probably want to specify your domain
+(`example.com`) rather than a matrix-specific hostname here (in the same way
+that your email address is probably `user@example.com` rather than
+`user@email.example.com`) - but doing so may require more advanced setup: see
+[Setting up Federation](docs/federate.md).
+
+# Picking a database engine
+
+Synapse offers two database engines:
+ * [PostgreSQL](https://www.postgresql.org)
+ * [SQLite](https://sqlite.org/)
+
+Almost all installations should opt to use PostgreSQL. Advantages include:
+
+* significant performance improvements due to the superior threading and
+  caching model, smarter query optimiser
+* allowing the DB to be run on separate hardware
+
+For information on how to install and use PostgreSQL, please see
+[docs/postgres.md](docs/postgres.md)
+
+By default Synapse uses SQLite and in doing so trades performance for convenience.
+SQLite is only recommended in Synapse for testing purposes or for servers with
+light workloads.
+
+# Installing Synapse
+
+## Installing from source
+
+(Prebuilt packages are available for some platforms - see [Prebuilt packages](#prebuilt-packages).)
+
+System requirements:
+
+- POSIX-compliant system (tested on Linux & OS X)
+- Python 3.5.2 or later, up to Python 3.9.
+- At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
+
+Synapse is written in Python but some of the libraries it uses are written in
+C. So before we can install Synapse itself we need a working C compiler and the
+header files for Python C extensions. See [Platform-Specific
+Instructions](#platform-specific-instructions) for information on installing
+these on various platforms.
+
+To install the Synapse homeserver run:
+
+```
+mkdir -p ~/synapse
+virtualenv -p python3 ~/synapse/env
+source ~/synapse/env/bin/activate
+pip install --upgrade pip
+pip install --upgrade setuptools
+pip install matrix-synapse
+```
+
+This will download Synapse from [PyPI](https://pypi.org/project/matrix-synapse)
+and install it, along with the python libraries it uses, into a virtual environment
+under `~/synapse/env`.  Feel free to pick a different directory if you
+prefer.
+
+This Synapse installation can then be later upgraded by using pip again with the
+update flag:
+
+```
+source ~/synapse/env/bin/activate
+pip install -U matrix-synapse
+```
+
+Before you can start Synapse, you will need to generate a configuration
+file. To do this, run (in your virtualenv, as before):
+
+```
+cd ~/synapse
+python -m synapse.app.homeserver \
+    --server-name my.domain.name \
+    --config-path homeserver.yaml \
+    --generate-config \
+    --report-stats=[yes|no]
+```
+
+... substituting an appropriate value for `--server-name`.
+
+This command will generate you a config file that you can then customise, but it will
+also generate a set of keys for you. These keys will allow your homeserver to
+identify itself to other homeserver, so don't lose or delete them. It would be
+wise to back them up somewhere safe. (If, for whatever reason, you do need to
+change your homeserver's keys, you may find that other homeserver have the
+old key cached. If you update the signing key, you should change the name of the
+key in the `<server name>.signing.key` file (the second word) to something
+different. See the
+[spec](https://matrix.org/docs/spec/server_server/latest.html#retrieving-server-keys)
+for more information on key management).
+
+To actually run your new homeserver, pick a working directory for Synapse to
+run (e.g. `~/synapse`), and:
+
+```
+cd ~/synapse
+source env/bin/activate
+synctl start
+```
+
+### Platform-Specific Instructions
+
+#### Debian/Ubuntu/Raspbian
+
+Installing prerequisites on Ubuntu or Debian:
+
+```
+sudo apt-get install build-essential python3-dev libffi-dev \
+                     python3-pip python3-setuptools sqlite3 \
+                     libssl-dev virtualenv libjpeg-dev libxslt1-dev
+```
+
+#### ArchLinux
+
+Installing prerequisites on ArchLinux:
+
+```
+sudo pacman -S base-devel python python-pip \
+               python-setuptools python-virtualenv sqlite3
+```
+
+#### CentOS/Fedora
+
+Installing prerequisites on CentOS 8 or Fedora>26:
+
+```
+sudo dnf install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
+                 libwebp-devel tk-devel redhat-rpm-config \
+                 python3-virtualenv libffi-devel openssl-devel
+sudo dnf groupinstall "Development Tools"
+```
+
+Installing prerequisites on CentOS 7 or Fedora<=25:
+
+```
+sudo yum install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
+                 lcms2-devel libwebp-devel tcl-devel tk-devel redhat-rpm-config \
+                 python3-virtualenv libffi-devel openssl-devel
+sudo yum groupinstall "Development Tools"
+```
+
+Note that Synapse does not support versions of SQLite before 3.11, and CentOS 7
+uses SQLite 3.7. You may be able to work around this by installing a more
+recent SQLite version, but it is recommended that you instead use a Postgres
+database: see [docs/postgres.md](docs/postgres.md).
+
+#### macOS
+
+Installing prerequisites on macOS:
+
+```
+xcode-select --install
+sudo easy_install pip
+sudo pip install virtualenv
+brew install pkg-config libffi
+```
+
+On macOS Catalina (10.15) you may need to explicitly install OpenSSL
+via brew and inform `pip` about it so that `psycopg2` builds:
+
+```
+brew install openssl@1.1
+export LDFLAGS=-L/usr/local/Cellar/openssl\@1.1/1.1.1d/lib/
+```
+
+#### OpenSUSE
+
+Installing prerequisites on openSUSE:
+
+```
+sudo zypper in -t pattern devel_basis
+sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \
+               python-devel libffi-devel libopenssl-devel libjpeg62-devel
+```
+
+#### OpenBSD
+
+A port of Synapse is available under `net/synapse`. The filesystem
+underlying the homeserver directory (defaults to `/var/synapse`) has to be
+mounted with `wxallowed` (cf. `mount(8)`), so creating a separate filesystem
+and mounting it to `/var/synapse` should be taken into consideration.
+
+To be able to build Synapse's dependency on python the `WRKOBJDIR`
+(cf. `bsd.port.mk(5)`) for building python, too, needs to be on a filesystem
+mounted with `wxallowed` (cf. `mount(8)`).
+
+Creating a `WRKOBJDIR` for building python under `/usr/local` (which on a
+default OpenBSD installation is mounted with `wxallowed`):
+
+```
+doas mkdir /usr/local/pobj_wxallowed
+```
+
+Assuming `PORTS_PRIVSEP=Yes` (cf. `bsd.port.mk(5)`) and `SUDO=doas` are
+configured in `/etc/mk.conf`:
+
+```
+doas chown _pbuild:_pbuild /usr/local/pobj_wxallowed
+```
+
+Setting the `WRKOBJDIR` for building python:
+
+```
+echo WRKOBJDIR_lang/python/3.7=/usr/local/pobj_wxallowed  \\nWRKOBJDIR_lang/python/2.7=/usr/local/pobj_wxallowed >> /etc/mk.conf
+```
+
+Building Synapse:
+
+```
+cd /usr/ports/net/synapse
+make install
+```
+
+#### Windows
+
+If you wish to run or develop Synapse on Windows, the Windows Subsystem For
+Linux provides a Linux environment on Windows 10 which is capable of using the
+Debian, Fedora, or source installation methods. More information about WSL can
+be found at https://docs.microsoft.com/en-us/windows/wsl/install-win10 for
+Windows 10 and https://docs.microsoft.com/en-us/windows/wsl/install-on-server
+for Windows Server.
+
+## Prebuilt packages
+
+As an alternative to installing from source, prebuilt packages are available
+for a number of platforms.
+
+### Docker images and Ansible playbooks
+
+There is an offical synapse image available at
+https://hub.docker.com/r/matrixdotorg/synapse which can be used with
+the docker-compose file available at [contrib/docker](contrib/docker). Further
+information on this including configuration options is available in the README
+on hub.docker.com.
+
+Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
+Dockerfile to automate a synapse server in a single Docker image, at
+https://hub.docker.com/r/avhost/docker-matrix/tags/
+
+Slavi Pantaleev has created an Ansible playbook,
+which installs the offical Docker image of Matrix Synapse
+along with many other Matrix-related services (Postgres database, Element, coturn,
+ma1sd, SSL support, etc.).
+For more details, see
+https://github.com/spantaleev/matrix-docker-ansible-deploy
+
+
+### Debian/Ubuntu
+
+#### Matrix.org packages
+
+Matrix.org provides Debian/Ubuntu packages of the latest stable version of
+Synapse via https://packages.matrix.org/debian/. They are available for Debian
+9 (Stretch), Ubuntu 16.04 (Xenial), and later. To use them:
+
+```
+sudo apt install -y lsb-release wget apt-transport-https
+sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
+echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" |
+    sudo tee /etc/apt/sources.list.d/matrix-org.list
+sudo apt update
+sudo apt install matrix-synapse-py3
+```
+
+**Note**: if you followed a previous version of these instructions which
+recommended using `apt-key add` to add an old key from
+`https://matrix.org/packages/debian/`, you should note that this key has been
+revoked. You should remove the old key with `sudo apt-key remove
+C35EB17E1EAE708E6603A9B3AD0592FE47F0DF61`, and follow the above instructions to
+update your configuration.
+
+The fingerprint of the repository signing key (as shown by `gpg
+/usr/share/keyrings/matrix-org-archive-keyring.gpg`) is
+`AAF9AE843A7584B5A3E4CD2BCF45A512DE2DA058`.
+
+#### Downstream Debian packages
+
+We do not recommend using the packages from the default Debian `buster`
+repository at this time, as they are old and suffer from known security
+vulnerabilities. You can install the latest version of Synapse from
+[our repository](#matrixorg-packages) or from `buster-backports`. Please
+see the [Debian documentation](https://backports.debian.org/Instructions/)
+for information on how to use backports.
+
+If you are using Debian `sid` or testing, Synapse is available in the default
+repositories and it should be possible to install it simply with:
+
+```
+sudo apt install matrix-synapse
+```
+
+#### Downstream Ubuntu packages
+
+We do not recommend using the packages in the default Ubuntu repository
+at this time, as they are old and suffer from known security vulnerabilities.
+The latest version of Synapse can be installed from [our repository](#matrixorg-packages).
+
+### Fedora
+
+Synapse is in the Fedora repositories as `matrix-synapse`:
+
+```
+sudo dnf install matrix-synapse
+```
+
+Oleg Girko provides Fedora RPMs at
+https://obs.infoserver.lv/project/monitor/matrix-synapse
+
+### OpenSUSE
+
+Synapse is in the OpenSUSE repositories as `matrix-synapse`:
+
+```
+sudo zypper install matrix-synapse
+```
+
+### SUSE Linux Enterprise Server
+
+Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 repository at
+https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/
+
+### ArchLinux
+
+The quickest way to get up and running with ArchLinux is probably with the community package
+https://www.archlinux.org/packages/community/any/matrix-synapse/, which should pull in most of
+the necessary dependencies.
+
+pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):
+
+```
+sudo pip install --upgrade pip
+```
+
+If you encounter an error with lib bcrypt causing an Wrong ELF Class:
+ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly
+compile it under the right architecture. (This should not be needed if
+installing under virtualenv):
+
+```
+sudo pip uninstall py-bcrypt
+sudo pip install py-bcrypt
+```
+
+### Void Linux
+
+Synapse can be found in the void repositories as 'synapse':
+
+```
+xbps-install -Su
+xbps-install -S synapse
+```
+
+### FreeBSD
+
+Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from:
+
+ - Ports: `cd /usr/ports/net-im/py-matrix-synapse && make install clean`
+ - Packages: `pkg install py37-matrix-synapse`
+
+### OpenBSD
+
+As of OpenBSD 6.7 Synapse is available as a pre-compiled binary. The filesystem
+underlying the homeserver directory (defaults to `/var/synapse`) has to be
+mounted with `wxallowed` (cf. `mount(8)`), so creating a separate filesystem
+and mounting it to `/var/synapse` should be taken into consideration.
+
+Installing Synapse:
+
+```
+doas pkg_add synapse
+```
+
+### NixOS
+
+Robin Lambertz has packaged Synapse for NixOS at:
+https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-synapse.nix
+
+# Setting up Synapse
+
+Once you have installed synapse as above, you will need to configure it.
+
+## TLS certificates
+
+The default configuration exposes a single HTTP port on the local
+interface: `http://localhost:8008`. It is suitable for local testing,
+but for any practical use, you will need Synapse's APIs to be served
+over HTTPS.
+
+The recommended way to do so is to set up a reverse proxy on port
+`8448`. You can find documentation on doing so in
+[docs/reverse_proxy.md](docs/reverse_proxy.md).
+
+Alternatively, you can configure Synapse to expose an HTTPS port. To do
+so, you will need to edit `homeserver.yaml`, as follows:
+
+* First, under the `listeners` section, uncomment the configuration for the
+  TLS-enabled listener. (Remove the hash sign (`#`) at the start of
+  each line). The relevant lines are like this:
+
+  ```
+    - port: 8448
+      type: http
+      tls: true
+      resources:
+        - names: [client, federation]
+  ```
+
+* You will also need to uncomment the `tls_certificate_path` and
+  `tls_private_key_path` lines under the `TLS` section. You will need to manage
+  provisioning of these certificates yourself — Synapse had built-in ACME
+  support, but the ACMEv1 protocol Synapse implements is deprecated, not
+  allowed by LetsEncrypt for new sites, and will break for existing sites in
+  late 2020. See [ACME.md](docs/ACME.md).
+
+  If you are using your own certificate, be sure to use a `.pem` file that
+  includes the full certificate chain including any intermediate certificates
+  (for instance, if using certbot, use `fullchain.pem` as your certificate, not
+  `cert.pem`).
+
+For a more detailed guide to configuring your server for federation, see
+[federate.md](docs/federate.md).
+
+## Client Well-Known URI
+
+Setting up the client Well-Known URI is optional but if you set it up, it will
+allow users to enter their full username (e.g. `@user:<server_name>`) into clients
+which support well-known lookup to automatically configure the homeserver and
+identity server URLs. This is useful so that users don't have to memorize or think
+about the actual homeserver URL you are using.
+
+The URL `https://<server_name>/.well-known/matrix/client` should return JSON in
+the following format.
+
+```
+{
+  "m.homeserver": {
+    "base_url": "https://<matrix.example.com>"
+  }
+}
+```
+
+It can optionally contain identity server information as well.
+
+```
+{
+  "m.homeserver": {
+    "base_url": "https://<matrix.example.com>"
+  },
+  "m.identity_server": {
+    "base_url": "https://<identity.example.com>"
+  }
+}
+```
+
+To work in browser based clients, the file must be served with the appropriate
+Cross-Origin Resource Sharing (CORS) headers. A recommended value would be
+`Access-Control-Allow-Origin: *` which would allow all browser based clients to
+view it.
+
+In nginx this would be something like:
+```
+location /.well-known/matrix/client {
+    return 200 '{"m.homeserver": {"base_url": "https://<matrix.example.com>"}}';
+    add_header Content-Type application/json;
+    add_header Access-Control-Allow-Origin *;
+}
+```
+
+You should also ensure the `public_baseurl` option in `homeserver.yaml` is set
+correctly. `public_baseurl` should be set to the URL that clients will use to
+connect to your server. This is the same URL you put for the `m.homeserver`
+`base_url` above.
+
+```
+public_baseurl: "https://<matrix.example.com>"
+```
+
+## Email
+
+It is desirable for Synapse to have the capability to send email. This allows
+Synapse to send password reset emails, send verifications when an email address
+is added to a user's account, and send email notifications to users when they
+receive new messages.
+
+To configure an SMTP server for Synapse, modify the configuration section
+headed `email`, and be sure to have at least the `smtp_host`, `smtp_port`
+and `notif_from` fields filled out.  You may also need to set `smtp_user`,
+`smtp_pass`, and `require_transport_security`.
+
+If email is not configured, password reset, registration and notifications via
+email will be disabled.
+
+## Registering a user
+
+The easiest way to create a new user is to do so from a client like [Element](https://element.io/).
+
+Alternatively you can do so from the command line if you have installed via pip.
+
+This can be done as follows:
+
+```
+$ source ~/synapse/env/bin/activate
+$ synctl start # if not already running
+$ register_new_matrix_user -c homeserver.yaml http://localhost:8008
+New user localpart: erikj
+Password:
+Confirm password:
+Make admin [no]:
+Success!
+```
+
+This process uses a setting `registration_shared_secret` in
+`homeserver.yaml`, which is shared between Synapse itself and the
+`register_new_matrix_user` script. It doesn't matter what it is (a random
+value is generated by `--generate-config`), but it should be kept secret, as
+anyone with knowledge of it can register users, including admin accounts,
+on your server even if `enable_registration` is `false`.
+
+## Setting up a TURN server
+
+For reliable VoIP calls to be routed via this homeserver, you MUST configure
+a TURN server. See [docs/turn-howto.md](docs/turn-howto.md) for details.
+
+## URL previews
+
+Synapse includes support for previewing URLs, which is disabled by default.  To
+turn it on you must enable the `url_preview_enabled: True` config parameter
+and explicitly specify the IP ranges that Synapse is not allowed to spider for
+previewing in the `url_preview_ip_range_blacklist` configuration parameter.
+This is critical from a security perspective to stop arbitrary Matrix users
+spidering 'internal' URLs on your network. At the very least we recommend that
+your loopback and RFC1918 IP addresses are blacklisted.
+
+This also requires the optional `lxml` and `netaddr` python dependencies to be
+installed. This in turn requires the `libxml2` library to be available - on
+Debian/Ubuntu this means `apt-get install libxml2-dev`, or equivalent for
+your OS.
+
+# Troubleshooting Installation
+
+`pip` seems to leak *lots* of memory during installation. For instance, a Linux
+host with 512MB of RAM may run out of memory whilst installing Twisted. If this
+happens, you will have to individually install the dependencies which are
+failing, e.g.:
+
+```
+pip install twisted
+```
+
+If you have any other problems, feel free to ask in
+[#synapse:matrix.org](https://matrix.to/#/#synapse:matrix.org).

MANIFEST.in

@@ -20,10 +20,9 @@ recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
 recursive-include tests *.py
-recursive-include tests *.pem
-recursive-include tests *.p8
-recursive-include tests *.crt
-recursive-include tests *.key
+include tests/http/ca.crt
+include tests/http/ca.key
+include tests/http/server.key
 
 recursive-include synapse/res *
 recursive-include synapse/static *.css
@@ -40,7 +39,6 @@ exclude mypy.ini
 exclude sytest-blacklist
 exclude test_postgresql.sh
 
-include book.toml
 include pyproject.toml
 recursive-include changelog.d *
 

README.rst

@@ -25,7 +25,7 @@ The overall architecture is::
 
 ``#matrix:matrix.org`` is the official support room for Matrix, and can be
 accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html or
-via IRC bridge at irc://irc.libera.chat/matrix.
+via IRC bridge at irc://irc.freenode.net/matrix.
 
 Synapse is currently in rapid development, but as of version 0.5 we believe it
 is sufficiently stable to be run as an internet-facing service for real usage!
@@ -94,8 +94,7 @@ Synapse Installation
 
 .. _federation:
 
-* For details on how to install synapse, see
-  `Installation Instructions <https://matrix-org.github.io/synapse/latest/setup/installation.html>`_.
+* For details on how to install synapse, see `<INSTALL.md>`_.
 * For specific details on how to configure Synapse for federation see `docs/federate.md <docs/federate.md>`_
 
 
@@ -107,8 +106,7 @@ from a web client.
 
 Unless you are running a test instance of Synapse on your local machine, in
 general, you will need to enable TLS support before you can successfully
-connect from a client: see
-`TLS certificates <https://matrix-org.github.io/synapse/latest/setup/installation.html#tls-certificates>`_.
+connect from a client: see `<INSTALL.md#tls-certificates>`_.
 
 An easy way to get started is to login or register via Element at
 https://app.element.io/#/login or https://app.element.io/#/register respectively.
@@ -144,55 +142,38 @@ the form of::
 As when logging in, you will need to specify a "Custom server".  Specify your
 desired ``localpart`` in the 'User name' box.
 
-Security note
+ACME setup
+==========
+
+For details on having Synapse manage your federation TLS certificates
+automatically, please see `<docs/ACME.md>`_.
+
+
+Security Note
 =============
 
-Matrix serves raw, user-supplied data in some APIs -- specifically the `content
-repository endpoints`_.
+Matrix serves raw user generated data in some APIs - specifically the `content
+repository endpoints <https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid>`_.
 
-.. _content repository endpoints: https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid
+Whilst we have tried to mitigate against possible XSS attacks (e.g.
+https://github.com/matrix-org/synapse/pull/1021) we recommend running
+matrix homeservers on a dedicated domain name, to limit any malicious user generated
+content served to web browsers a matrix API from being able to attack webapps hosted
+on the same domain.  This is particularly true of sharing a matrix webclient and
+server on the same domain.
 
-Whilst we make a reasonable effort to mitigate against XSS attacks (for
-instance, by using `CSP`_), a Matrix homeserver should not be hosted on a
-domain hosting other web applications. This especially applies to sharing
-the domain with Matrix web clients and other sensitive applications like
-webmail. See
-https://developer.github.com/changes/2014-04-25-user-content-security for more
-information.
-
-.. _CSP: https://github.com/matrix-org/synapse/pull/1021
-
-Ideally, the homeserver should not simply be on a different subdomain, but on
-a completely different `registered domain`_ (also known as top-level site or
-eTLD+1). This is because `some attacks`_ are still possible as long as the two
-applications share the same registered domain.
-
-.. _registered domain: https://tools.ietf.org/html/draft-ietf-httpbis-rfc6265bis-03#section-2.3
-
-.. _some attacks: https://en.wikipedia.org/wiki/Session_fixation#Attacks_using_cross-subdomain_cookie
-
-To illustrate this with an example, if your Element Web or other sensitive web
-application is hosted on ``A.example1.com``, you should ideally host Synapse on
-``example2.com``. Some amount of protection is offered by hosting on
-``B.example1.com`` instead, so this is also acceptable in some scenarios.
-However, you should *not* host your Synapse on ``A.example1.com``.
-
-Note that all of the above refers exclusively to the domain used in Synapse's
-``public_baseurl`` setting. In particular, it has no bearing on the domain
-mentioned in MXIDs hosted on that server.
-
-Following this advice ensures that even if an XSS is found in Synapse, the
-impact to other applications will be minimal.
+See https://github.com/vector-im/riot-web/issues/1977 and
+https://developer.github.com/changes/2014-04-25-user-content-security for more details.
 
 
 Upgrading an existing Synapse
 =============================
 
-The instructions for upgrading synapse are in `the upgrade notes`_.
+The instructions for upgrading synapse are in `UPGRADE.rst`_.
 Please check these instructions as upgrading may require extra steps for some
 versions of synapse.
 
-.. _the upgrade notes: https://matrix-org.github.io/synapse/develop/upgrade.html
+.. _UPGRADE.rst: UPGRADE.rst
 
 .. _reverse-proxy:
 
@@ -202,9 +183,8 @@ Using a reverse proxy with Synapse
 It is recommended to put a reverse proxy such as
 `nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
 `Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_,
-`Caddy <https://caddyserver.com/docs/quick-starts/reverse-proxy>`_,
-`HAProxy <https://www.haproxy.org/>`_ or
-`relayd <https://man.openbsd.org/relayd.8>`_ in front of Synapse. One advantage of
+`Caddy <https://caddyserver.com/docs/quick-starts/reverse-proxy>`_ or
+`HAProxy <https://www.haproxy.org/>`_ in front of Synapse. One advantage of
 doing so is that it means that you can expose the default https port (443) to
 Matrix clients without needing to run Synapse with root privileges.
 
@@ -263,11 +243,9 @@ Then update the ``users`` table in the database::
 Synapse Development
 ===================
 
-Join our developer community on Matrix: `#synapse-dev:matrix.org <https://matrix.to/#/#synapse-dev:matrix.org>`_
-
 Before setting up a development environment for synapse, make sure you have the
 system dependencies (such as the python header files) installed - see
-`Installing from source <https://matrix-org.github.io/synapse/latest/setup/installation.html#installing-from-source>`_.
+`Installing from source <INSTALL.md#installing-from-source>`_.
 
 To check out a synapse for development, clone the git repo into a working
 directory of your choice::
@@ -283,48 +261,18 @@ to install using pip and a virtualenv::
     pip install -e ".[all,test]"
 
 This will run a process of downloading and installing all the needed
-dependencies into a virtual env. If any dependencies fail to install,
-try installing the failing modules individually::
+dependencies into a virtual env.
 
-    pip install -e "module-name"
+Once this is done, you may wish to run Synapse's unit tests, to
+check that everything is installed as it should be::
 
-We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`
+    python -m twisted.trial tests
 
-    ./demo/start.sh
+This should end with a 'PASSED' result::
 
-(to stop, you can use `./demo/stop.sh`)
-
-If you just want to start a single instance of the app and run it directly::
-
-    # Create the homeserver.yaml config once
-    python -m synapse.app.homeserver \
-      --server-name my.domain.name \
-      --config-path homeserver.yaml \
-      --generate-config \
-      --report-stats=[yes|no]
-
-    # Start the app
-    python -m synapse.app.homeserver --config-path homeserver.yaml
-
-
-Running the unit tests
-======================
-
-After getting up and running, you may wish to run Synapse's unit tests to
-check that everything is installed correctly::
-
-    trial tests
-
-This should end with a 'PASSED' result (note that exact numbers will
-differ)::
-
-    Ran 1337 tests in 716.064s
-
-    PASSED (skips=15, successes=1322)
-
-For more tips on running the unit tests, like running a specific test or
-to see the logging output, see the `CONTRIBUTING doc <CONTRIBUTING.md#run-the-unit-tests>`_.
+    Ran 1266 tests in 643.930s
 
+    PASSED (skips=15, successes=1251)
 
 Running the Integration Tests
 =============================
@@ -335,17 +283,8 @@ access the API as a Matrix client would. It is able to run Synapse directly from
 the source tree, so installation of the server is not required.
 
 Testing with SyTest is recommended for verifying that changes related to the
-Client-Server API are functioning correctly. See the `SyTest installation
-instructions <https://github.com/matrix-org/sytest#installing>`_ for details.
-
-
-Platform dependencies
-=====================
-
-Synapse uses a number of platform dependencies such as Python and PostgreSQL,
-and aims to follow supported upstream versions. See the
-`<docs/deprecation_policy.md>`_ document for more details.
-
+Client-Server API are functioning correctly. See the `installation instructions
+<https://github.com/matrix-org/sytest#installing>`_ for details.
 
 Troubleshooting
 ===============
@@ -417,17 +356,12 @@ massive excess of outgoing federation requests (see `discussion
 indicate that your server is also issuing far more outgoing federation
 requests than can be accounted for by your users' activity, this is a
 likely cause. The misbehavior can be worked around by setting
-the following in the Synapse config file:
-
-.. code-block:: yaml
-
-   presence:
-       enabled: false
+``use_presence: false`` in the Synapse config file.
 
 People can't accept room invitations from me
 --------------------------------------------
 
-The typical failure mode here is that you send an invitation to someone
+The typical failure mode here is that you send an invitation to someone 
 to join a room or direct chat, but when they go to accept it, they get an
 error (typically along the lines of "Invalid signature"). They might see
 something like the following in their logs::

book.toml

@@ -1,39 +0,0 @@
-# Documentation for possible options in this file is at
-# https://rust-lang.github.io/mdBook/format/config.html
-[book]
-title = "Synapse"
-authors = ["The Matrix.org Foundation C.I.C."]
-language = "en"
-multilingual = false
-
-# The directory that documentation files are stored in
-src = "docs"
-
-[build]
-# Prevent markdown pages from being automatically generated when they're
-# linked to in SUMMARY.md
-create-missing = false
-
-[output.html]
-# The URL visitors will be directed to when they try to edit a page
-edit-url-template = "https://github.com/matrix-org/synapse/edit/develop/{path}"
-
-# Remove the numbers that appear before each item in the sidebar, as they can
-# get quite messy as we nest deeper
-no-section-label = true
-
-# The source code URL of the repository
-git-repository-url = "https://github.com/matrix-org/synapse"
-
-# The path that the docs are hosted on
-site-url = "/synapse/"
-
-# Additional HTML, JS, CSS that's injected into each page of the book.
-# More information available in docs/website_files/README.md
-additional-css = [
-    "docs/website_files/table-of-contents.css",
-    "docs/website_files/remove-nav-buttons.css",
-    "docs/website_files/indent-section-headers.css",
-]
-additional-js = ["docs/website_files/table-of-contents.js"]
-theme = "docs/website_files/theme"

changelog.d/10254.feature

@@ -1 +0,0 @@
-Update support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) to consider changes in the MSC around which servers can issue join events.

changelog.d/10283.feature

@@ -1 +0,0 @@
-Initial support for MSC3244, Room version capabilities over the /capabilities API.

changelog.d/10407.feature

@@ -1 +0,0 @@
-Add a buffered logging handler which periodically flushes itself.

changelog.d/10408.misc

@@ -1 +0,0 @@
-Add type hints to `synapse.federation.transport.client` module.

changelog.d/10426.feature

@@ -1 +0,0 @@
-Email notifications now state whether an invitation is to a room or a space.

changelog.d/10429.misc

@@ -1 +0,0 @@
-Drop backwards-compatibility code that was required to support Ubuntu Xenial.

changelog.d/10431.misc

@@ -1 +0,0 @@
-Use a docker image cache for the prerequisites for the debian package build.

changelog.d/10432.misc

@@ -1 +0,0 @@
-Connect historical chunks together with chunk events instead of a content field (MSC2716).

changelog.d/10437.misc

@@ -1 +0,0 @@
-Improve servlet type hints.

changelog.d/10438.misc

@@ -1 +0,0 @@
-Improve servlet type hints.

changelog.d/10442.misc

@@ -1 +0,0 @@
-Replace usage of `or_ignore` in `simple_insert` with `simple_upsert` usage, to stop spamming postgres logs with spurious ERROR messages.

changelog.d/10444.misc

@@ -1 +0,0 @@
-Update the `tests-done` Github Actions status.

changelog.d/10445.doc

@@ -1 +0,0 @@
-Fix hierarchy of providers on the OpenID page.

changelog.d/10446.misc

@@ -1 +0,0 @@
-Update type annotations to work with forthcoming Twisted 21.7.0 release.

changelog.d/10448.feature

@@ -1 +0,0 @@
-Add `creation_ts` to list users admin API.

changelog.d/10451.misc

@@ -1 +0,0 @@
-Cancel redundant GHA workflows when a new commit is pushed.

changelog.d/10453.doc

@@ -1 +0,0 @@
-Consolidate development documentation to `docs/development/`.

changelog.d/10461.misc

@@ -1 +0,0 @@
-Fix an error which prevented the Github Actions workflow to build the docker images from running.

changelog.d/10463.misc

@@ -1 +0,0 @@
-Disable `msc2716` Complement tests until Complement updates are merged.

changelog.d/10468.misc

@@ -1 +0,0 @@
-Mitigate media repo XSS attacks on IE11 via the non-standard X-Content-Security-Policy header.

changelog.d/10482.misc

@@ -1 +0,0 @@
-Additional type hints in the state handler.

changelog.d/10488.misc

@@ -1 +0,0 @@
-Update syntax used to run complement tests.

changelog.d/8455.bugfix

@@ -0,0 +1 @@
+Fix fetching of E2E cross signing keys over federation when only one of the master key and device signing key is cached already.

changelog.d/8519.feature

@@ -0,0 +1 @@
+Add an admin api to delete a single file or files were not used for a defined time from server. Contributed by @dklimpel.

changelog.d/8539.feature

@@ -0,0 +1 @@
+Split admin API for reported events (`GET /_synapse/admin/v1/event_reports`) into detail and list endpoints. This is a breaking change to #8217 which was introduced in Synapse v1.21.0. Those who already use this API should check their scripts. Contributed by @dklimpel.

changelog.d/8559.misc

@@ -0,0 +1 @@
+Optimise `/createRoom` with multiple invited users.

changelog.d/8580.bugfix

@@ -0,0 +1 @@
+Fix a bug where Synapse would blindly forward bad responses from federation to clients when retrieving profile information.

changelog.d/8582.doc

@@ -0,0 +1 @@
+Instructions for Azure AD in the OpenID Connect documentation. Contributed by peterk.

changelog.d/8595.misc

@@ -0,0 +1 @@
+Implement and use an @lru_cache decorator.

changelog.d/8607.feature

@@ -0,0 +1 @@
+Support generating structured logs via the standard logging configuration.

changelog.d/8610.feature

@@ -0,0 +1 @@
+Add an admin APIs to allow server admins to list users' pushers. Contributed by @dklimpel.

changelog.d/8614.misc

@@ -0,0 +1 @@
+Don't instansiate Requester directly.

changelog.d/8615.misc

@@ -0,0 +1 @@
+Type hints for `RegistrationStore`.

changelog.d/8616.misc

@@ -0,0 +1 @@
+Change schema to support access tokens belonging to one user but granting access to another.

changelog.d/8620.bugfix

@@ -0,0 +1 @@
+Fix a bug where the account validity endpoint would silently fail if the user ID did not have an expiration time. It now returns a 400 error.

changelog.d/8621.misc

@@ -0,0 +1 @@
+Remove unused OPTIONS handlers.

changelog.d/8627.bugfix

@@ -0,0 +1 @@
+Fix email notifications for invites without local state.

changelog.d/8628.bugfix

@@ -0,0 +1 @@
+Fix handling of invalid group IDs to return a 400 rather than log an exception and return a 500.

changelog.d/8632.bugfix

@@ -0,0 +1 @@
+Fix handling of User-Agent headers that are invalid UTF-8, which caused user agents of users to not get correctly recorded.

changelog.d/8633.misc

@@ -0,0 +1 @@
+Run `mypy` as part of the lint.sh script.

changelog.d/8634.misc

@@ -0,0 +1 @@
+Correct Synapse's PyPI package name in the OpenID Connect installation instructions.

changelog.d/8635.doc

@@ -0,0 +1 @@
+Improve the sample configuration for single sign-on providers.

changelog.d/8639.misc

@@ -0,0 +1 @@
+Fix typos and spelling errors in the code.

changelog.d/8640.misc

@@ -0,0 +1 @@
+Reduce number of OpenTracing spans started.

changelog.d/8643.bugfix

@@ -0,0 +1 @@
+Fix a bug in the `joined_rooms` admin API if the user has never joined any rooms. The bug was introduced, along with the API, in v1.21.0.

changelog.d/8644.misc

@@ -0,0 +1 @@
+Add field `total` to device list in admin API.

changelog.d/8647.feature

@@ -0,0 +1 @@
+Add an admin API `GET /_synapse/admin/v1/users/<user_id>/media` to get information about uploaded media. Contributed by @dklimpel.

changelog.d/8655.misc

@@ -0,0 +1 @@
+Add more type hints to the application services code.

changelog.d/8657.doc

@@ -0,0 +1 @@
+Fix the filepath of Dex's example config and the link to Dex's Getting Started guide in the OpenID Connect docs.

changelog.d/8664.misc

@@ -0,0 +1 @@
+Tell Black to format code for Python 3.5.

changelog.d/8665.doc

@@ -0,0 +1 @@
+Note support for Python 3.9.

changelog.d/8666.doc

@@ -0,0 +1 @@
+Minor updates to docs on running tests.

changelog.d/8667.doc

@@ -0,0 +1 @@
+Interlink prometheus/grafana documentation.

changelog.d/8668.misc

@@ -0,0 +1 @@
+Reduce number of OpenTracing spans started.

changelog.d/8669.misc

@@ -0,0 +1 @@
+Don't pull event from DB when handling replication traffic.

changelog.d/8670.misc

@@ -0,0 +1 @@
+Reduce number of OpenTracing spans started.

changelog.d/8671.misc

@@ -0,0 +1 @@
+Abstract some invite-related code in preparation for landing knocking.

changelog.d/8679.misc

@@ -0,0 +1 @@
+Clarify representation of events in logfiles.

changelog.d/8680.misc

@@ -0,0 +1 @@
+Don't require `hiredis` package to be installed to run unit tests.

changelog.d/8682.bugfix

@@ -0,0 +1 @@
+Fix exception during handling multiple concurrent requests for remote media when using multiple media repositories.

changelog.d/8684.misc

@@ -0,0 +1 @@
+Fix typing info on cache call signature to accept `on_invalidate`.

changelog.d/8685.feature

@@ -0,0 +1 @@
+Support generating structured logs via the standard logging configuration.

changelog.d/8688.misc

@@ -0,0 +1 @@
+Abstract some invite-related code in preparation for landing knocking.

changelog.d/8689.feature

@@ -0,0 +1 @@
+Add an admin APIs to allow server admins to list users' pushers. Contributed by @dklimpel.

changelog.d/8690.misc

@@ -0,0 +1 @@
+Fail tests if they do not await coroutines.

changelog.d/8693.misc

@@ -0,0 +1 @@
+Add more type hints to the application services code.

changelog.d/9918.feature

@@ -1 +0,0 @@
-Add support for [MSC2033](https://github.com/matrix-org/matrix-doc/pull/2033): `device_id` on `/account/whoami`.

contrib/cmdclient/console.py

@@ -24,7 +24,6 @@ import sys
 import time
 import urllib
 from http import TwistedHttpClient
-from typing import Optional
 
 import nacl.encoding
 import nacl.signing
@@ -93,7 +92,7 @@ class SynapseCmd(cmd.Cmd):
         return self.config["user"].split(":")[1]
 
     def do_config(self, line):
-        """Show the config for this client: "config"
+        """ Show the config for this client: "config"
         Edit a key value mapping: "config key value" e.g. "config token 1234"
         Config variables:
             user: The username to auth with.
@@ -361,7 +360,7 @@ class SynapseCmd(cmd.Cmd):
             print(e)
 
     def do_topic(self, line):
-        """ "topic [set|get] <roomid> [<newtopic>]"
+        """"topic [set|get] <roomid> [<newtopic>]"
         Set the topic for a room: topic set <roomid> <newtopic>
         Get the topic for a room: topic get <roomid>
         """
@@ -691,7 +690,7 @@ class SynapseCmd(cmd.Cmd):
         self._do_presence_state(2, line)
 
     def _parse(self, line, keys, force_keys=False):
-        """Parses the given line.
+        """ Parses the given line.
 
         Args:
             line : The line to parse
@@ -719,10 +718,10 @@ class SynapseCmd(cmd.Cmd):
         method,
         path,
         data=None,
-        query_params: Optional[dict] = None,
+        query_params={"access_token": None},
         alt_text=None,
     ):
-        """Runs an HTTP request and pretty prints the output.
+        """ Runs an HTTP request and pretty prints the output.
 
         Args:
             method: HTTP method
@@ -730,8 +729,6 @@ class SynapseCmd(cmd.Cmd):
             data: Raw JSON data if any
             query_params: dict of query parameters to add to the url
         """
-        query_params = query_params or {"access_token": None}
-
         url = self._url() + path
         if "access_token" in query_params:
             query_params["access_token"] = self._tok()

contrib/cmdclient/http.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +16,6 @@
 import json
 import urllib
 from pprint import pformat
-from typing import Optional
 
 from twisted.internet import defer, reactor
 from twisted.web.client import Agent, readBody
@@ -23,10 +23,11 @@ from twisted.web.http_headers import Headers
 
 
 class HttpClient:
-    """Interface for talking json over http"""
+    """ Interface for talking json over http
+    """
 
     def put_json(self, url, data):
-        """Sends the specifed json data using PUT
+        """ Sends the specifed json data using PUT
 
         Args:
             url (str): The URL to PUT data to.
@@ -40,7 +41,7 @@ class HttpClient:
         pass
 
     def get_json(self, url, args=None):
-        """Gets some json from the given host homeserver and path
+        """ Gets some json from the given host homeserver and path
 
         Args:
             url (str): The URL to GET data from.
@@ -57,7 +58,7 @@ class HttpClient:
 
 
 class TwistedHttpClient(HttpClient):
-    """Wrapper around the twisted HTTP client api.
+    """ Wrapper around the twisted HTTP client api.
 
     Attributes:
         agent (twisted.web.client.Agent): The twisted Agent used to send the
@@ -85,9 +86,9 @@ class TwistedHttpClient(HttpClient):
         body = yield readBody(response)
         defer.returnValue(json.loads(body))
 
-    def _create_put_request(self, url, json_data, headers_dict: Optional[dict] = None):
-        """Wrapper of _create_request to issue a PUT request"""
-        headers_dict = headers_dict or {}
+    def _create_put_request(self, url, json_data, headers_dict={}):
+        """ Wrapper of _create_request to issue a PUT request
+        """
 
         if "Content-Type" not in headers_dict:
             raise defer.error(RuntimeError("Must include Content-Type header for PUTs"))
@@ -96,22 +97,15 @@ class TwistedHttpClient(HttpClient):
             "PUT", url, producer=_JsonProducer(json_data), headers_dict=headers_dict
         )
 
-    def _create_get_request(self, url, headers_dict: Optional[dict] = None):
-        """Wrapper of _create_request to issue a GET request"""
-        return self._create_request("GET", url, headers_dict=headers_dict or {})
+    def _create_get_request(self, url, headers_dict={}):
+        """ Wrapper of _create_request to issue a GET request
+        """
+        return self._create_request("GET", url, headers_dict=headers_dict)
 
     @defer.inlineCallbacks
     def do_request(
-        self,
-        method,
-        url,
-        data=None,
-        qparams=None,
-        jsonreq=True,
-        headers: Optional[dict] = None,
+        self, method, url, data=None, qparams=None, jsonreq=True, headers={}
     ):
-        headers = headers or {}
-
         if qparams:
             url = "%s?%s" % (url, urllib.urlencode(qparams, True))
 
@@ -132,12 +126,9 @@ class TwistedHttpClient(HttpClient):
         defer.returnValue(json.loads(body))
 
     @defer.inlineCallbacks
-    def _create_request(
-        self, method, url, producer=None, headers_dict: Optional[dict] = None
-    ):
-        """Creates and sends a request to the given url"""
-        headers_dict = headers_dict or {}
-
+    def _create_request(self, method, url, producer=None, headers_dict={}):
+        """ Creates and sends a request to the given url
+        """
         headers_dict["User-Agent"] = ["Synapse Cmd Client"]
 
         retries_left = 5
@@ -194,7 +185,8 @@ class _RawProducer:
 
 
 class _JsonProducer:
-    """Used by the twisted http client to create the HTTP body from json"""
+    """ Used by the twisted http client to create the HTTP body from json
+    """
 
     def __init__(self, jsn):
         self.data = jsn

contrib/docker/docker-compose.yml

@@ -56,7 +56,7 @@ services:
       - POSTGRES_USER=synapse
       - POSTGRES_PASSWORD=changeme
       # ensure the database gets created correctly
-      # https://matrix-org.github.io/synapse/latest/postgres.html#set-up-database
+      # https://github.com/matrix-org/synapse/blob/master/docs/postgres.md#set-up-database
       - POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=C --lc-ctype=C
     volumes:
       # You may store the database tables in a local folder..

contrib/experiments/cursesio.py

@@ -46,14 +46,14 @@ class CursesStdIO:
         self.callback = callback
 
     def fileno(self):
-        """We want to select on FD 0"""
+        """ We want to select on FD 0 """
         return 0
 
     def connectionLost(self, reason):
         self.close()
 
     def print_line(self, text):
-        """add a line to the internal list of lines"""
+        """ add a line to the internal list of lines"""
 
         self.lines.append(text)
         self.redraw()
@@ -63,7 +63,8 @@ class CursesStdIO:
         self.redraw()
 
     def redraw(self):
-        """method for redisplaying lines based on internal list of lines"""
+        """ method for redisplaying lines
+            based on internal list of lines """
 
         self.stdscr.clear()
         self.paintStatus(self.statusText)
@@ -92,7 +93,7 @@ class CursesStdIO:
         )
 
     def doRead(self):
-        """Input is ready!"""
+        """ Input is ready! """
         curses.noecho()
         c = self.stdscr.getch()  # read a character
 
@@ -132,7 +133,7 @@ class CursesStdIO:
         return "CursesStdIO"
 
     def close(self):
-        """clean up"""
+        """ clean up """
 
         curses.nocbreak()
         self.stdscr.keypad(0)

contrib/experiments/test_messaging.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -55,7 +56,7 @@ def excpetion_errback(failure):
 
 
 class InputOutput:
-    """This is responsible for basic I/O so that a user can interact with
+    """ This is responsible for basic I/O so that a user can interact with
     the example app.
     """
 
@@ -67,7 +68,8 @@ class InputOutput:
         self.server = server
 
     def on_line(self, line):
-        """This is where we process commands."""
+        """ This is where we process commands.
+        """
 
         try:
             m = re.match(r"^join (\S+)$", line)
@@ -131,7 +133,7 @@ class IOLoggerHandler(logging.Handler):
 
 
 class Room:
-    """Used to store (in memory) the current membership state of a room, and
+    """ Used to store (in memory) the current membership state of a room, and
     which home servers we should send PDUs associated with the room to.
     """
 
@@ -146,7 +148,8 @@ class Room:
         self.have_got_metadata = False
 
     def add_participant(self, participant):
-        """Someone has joined the room"""
+        """ Someone has joined the room
+        """
         self.participants.add(participant)
         self.invited.discard(participant)
 
@@ -157,13 +160,14 @@ class Room:
             self.oldest_server = server
 
     def add_invited(self, invitee):
-        """Someone has been invited to the room"""
+        """ Someone has been invited to the room
+        """
         self.invited.add(invitee)
         self.servers.add(origin_from_ucid(invitee))
 
 
 class HomeServer(ReplicationHandler):
-    """A very basic home server implentation that allows people to join a
+    """ A very basic home server implentation that allows people to join a
     room and then invite other people.
     """
 
@@ -177,7 +181,8 @@ class HomeServer(ReplicationHandler):
         self.output = output
 
     def on_receive_pdu(self, pdu):
-        """We just received a PDU"""
+        """ We just received a PDU
+        """
         pdu_type = pdu.pdu_type
 
         if pdu_type == "sy.room.message":
@@ -194,20 +199,23 @@ class HomeServer(ReplicationHandler):
             )
 
     def _on_message(self, pdu):
-        """We received a message"""
+        """ We received a message
+        """
         self.output.print_line(
             "#%s %s %s" % (pdu.context, pdu.content["sender"], pdu.content["body"])
         )
 
     def _on_join(self, context, joinee):
-        """Someone has joined a room, either a remote user or a local user"""
+        """ Someone has joined a room, either a remote user or a local user
+        """
         room = self._get_or_create_room(context)
         room.add_participant(joinee)
 
         self.output.print_line("#%s %s %s" % (context, joinee, "*** JOINED"))
 
     def _on_invite(self, origin, context, invitee):
-        """Someone has been invited"""
+        """ Someone has been invited
+        """
         room = self._get_or_create_room(context)
         room.add_invited(invitee)
 
@@ -220,7 +228,8 @@ class HomeServer(ReplicationHandler):
 
     @defer.inlineCallbacks
     def send_message(self, room_name, sender, body):
-        """Send a message to a room!"""
+        """ Send a message to a room!
+        """
         destinations = yield self.get_servers_for_context(room_name)
 
         try:
@@ -238,7 +247,8 @@ class HomeServer(ReplicationHandler):
 
     @defer.inlineCallbacks
     def join_room(self, room_name, sender, joinee):
-        """Join a room!"""
+        """ Join a room!
+        """
         self._on_join(room_name, joinee)
 
         destinations = yield self.get_servers_for_context(room_name)
@@ -259,7 +269,8 @@ class HomeServer(ReplicationHandler):
 
     @defer.inlineCallbacks
     def invite_to_room(self, room_name, sender, invitee):
-        """Invite someone to a room!"""
+        """ Invite someone to a room!
+        """
         self._on_invite(self.server_name, room_name, invitee)
 
         destinations = yield self.get_servers_for_context(room_name)

contrib/grafana/README.md

@@ -1,6 +1,6 @@
 # Using the Synapse Grafana dashboard
 
 0. Set up Prometheus and Grafana. Out of scope for this readme. Useful documentation about using Grafana with Prometheus: http://docs.grafana.org/features/datasources/prometheus/
-1. Have your Prometheus scrape your Synapse. https://matrix-org.github.io/synapse/latest/metrics-howto.html
+1. Have your Prometheus scrape your Synapse. https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md
 2. Import dashboard into Grafana. Download `synapse.json`. Import it to Grafana and select the correct Prometheus datasource. http://docs.grafana.org/reference/export_import/
-3. Set up required recording rules. [contrib/prometheus](../prometheus)
+3. Set up required recording rules. https://github.com/matrix-org/synapse/tree/master/contrib/prometheus

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -193,12 +193,15 @@ class TrivialXmppClient:
         time.sleep(7)
         print("SSRC spammer started")
         while self.running:
-            ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % {
-                "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
-                "nick": self.userId,
-                "assrc": self.ssrcs["audio"],
-                "vssrc": self.ssrcs["video"],
-            }
+            ssrcMsg = (
+                "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>"
+                % {
+                    "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
+                    "nick": self.userId,
+                    "assrc": self.ssrcs["audio"],
+                    "vssrc": self.ssrcs["video"],
+                }
+            )
             res = self.sendIq(ssrcMsg)
             print("reply from ssrc announce: ", res)
             time.sleep(10)

contrib/prometheus/README.md

@@ -20,7 +20,6 @@ Add a new job to the main prometheus.conf file:
 ```
 
 ### for Prometheus v2
-
 Add a new job to the main prometheus.yml file:
 
 ```yaml
@@ -30,17 +29,14 @@ Add a new job to the main prometheus.yml file:
     scheme: "https"
 
     static_configs:
-    - targets: ["my.server.here:port"]
+    - targets: ['SERVER.LOCATION:PORT']
 ```
 
-An example of a Prometheus configuration with workers can be found in
-[metrics-howto.md](https://matrix-org.github.io/synapse/latest/metrics-howto.html).
-
 To use `synapse.rules` add
 
 ```yaml
-  rule_files:
-    - "/PATH/TO/synapse-v2.rules"
+    rule_files:
+      - "/PATH/TO/synapse-v2.rules"
 ```
 
 Metrics are disabled by default when running synapse; they must be enabled

contrib/prometheus/consoles/synapse.html

@@ -9,7 +9,7 @@
 new PromConsole.Graph({
   node: document.querySelector("#process_resource_utime"),
   expr: "rate(process_cpu_seconds_total[2m]) * 100",
-  name: "[[job]]-[[index]]",
+  name: "[[job]]",  
   min: 0,
   max: 100,
   renderer: "line",
@@ -22,12 +22,12 @@ new PromConsole.Graph({
 </script>
 
 <h3>Memory</h3>
-<div id="process_resident_memory_bytes"></div>
+<div id="process_resource_maxrss"></div>
 <script>
 new PromConsole.Graph({
-  node: document.querySelector("#process_resident_memory_bytes"),
-  expr: "process_resident_memory_bytes",
-  name: "[[job]]-[[index]]",
+  node: document.querySelector("#process_resource_maxrss"),
+  expr: "process_psutil_rss:max",
+  name: "Maxrss",
   min: 0,
   renderer: "line",
   height: 150,
@@ -43,8 +43,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#process_fds"),
-  expr: "process_open_fds",
-  name: "[[job]]-[[index]]",
+  expr: "process_open_fds{job='synapse'}",
+  name: "FDs",
   min: 0,
   renderer: "line",
   height: 150,
@@ -62,8 +62,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#reactor_total_time"),
-  expr: "rate(python_twisted_reactor_tick_time_sum[2m])",
-  name: "[[job]]-[[index]]",
+  expr: "rate(python_twisted_reactor_tick_time:total[2m]) / 1000",
+  name: "time",
   max: 1,
   min: 0,
   renderer: "area",
@@ -80,8 +80,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#reactor_average_time"),
-  expr: "rate(python_twisted_reactor_tick_time_sum[2m]) / rate(python_twisted_reactor_tick_time_count[2m])",
-  name: "[[job]]-[[index]]",
+  expr: "rate(python_twisted_reactor_tick_time:total[2m]) / rate(python_twisted_reactor_tick_time:count[2m]) / 1000",
+  name: "time",
   min: 0,
   renderer: "line",
   height: 150,
@@ -97,14 +97,14 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#reactor_pending_calls"),
-  expr: "rate(python_twisted_reactor_pending_calls_sum[30s]) / rate(python_twisted_reactor_pending_calls_count[30s])",
-  name: "[[job]]-[[index]]",
+  expr: "rate(python_twisted_reactor_pending_calls:total[30s])/rate(python_twisted_reactor_pending_calls:count[30s])",
+  name: "calls",
   min: 0,
   renderer: "line",
   height: 150,
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
-  yTitle: "Pending Calls"
+  yTitle: "Pending Cals"
 })
 </script>
 
@@ -115,7 +115,7 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_storage_query_time"),
-  expr: "sum(rate(synapse_storage_query_time_count[2m])) by (verb)",
+  expr: "rate(synapse_storage_query_time:count[2m])",
   name: "[[verb]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
@@ -129,8 +129,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_storage_transaction_time"),
-  expr: "topk(10, rate(synapse_storage_transaction_time_count[2m]))",
-  name: "[[job]]-[[index]] [[desc]]",
+  expr: "rate(synapse_storage_transaction_time:count[2m])",
+  name: "[[desc]]",
   min: 0,
   yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
@@ -140,12 +140,12 @@ new PromConsole.Graph({
 </script>
 
 <h3>Transaction execution time</h3>
-<div id="synapse_storage_transactions_time_sec"></div>
+<div id="synapse_storage_transactions_time_msec"></div>
 <script>
 new PromConsole.Graph({
-  node: document.querySelector("#synapse_storage_transactions_time_sec"),
-  expr: "rate(synapse_storage_transaction_time_sum[2m])",
-  name: "[[job]]-[[index]] [[desc]]",
+  node: document.querySelector("#synapse_storage_transactions_time_msec"),
+  expr: "rate(synapse_storage_transaction_time:total[2m]) / 1000",
+  name: "[[desc]]",
   min: 0,
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
@@ -154,33 +154,34 @@ new PromConsole.Graph({
 })
 </script>
 
-<h3>Average time waiting for database connection</h3>
-<div id="synapse_storage_avg_waiting_time"></div>
+<h3>Database scheduling latency</h3>
+<div id="synapse_storage_schedule_time"></div>
 <script>
 new PromConsole.Graph({
-  node: document.querySelector("#synapse_storage_avg_waiting_time"),
-  expr: "rate(synapse_storage_schedule_time_sum[2m]) / rate(synapse_storage_schedule_time_count[2m])",
-  name: "[[job]]-[[index]]",
+  node: document.querySelector("#synapse_storage_schedule_time"),
+  expr: "rate(synapse_storage_schedule_time:total[2m]) / 1000",
+  name: "Total latency",
   min: 0,
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
-  yUnits: "s",
-  yTitle: "Time"
+  yUnits: "s/s",
+  yTitle: "Usage"
 })
 </script>
 
-<h3>Cache request rate</h3>
-<div id="synapse_cache_request_rate"></div>
+<h3>Cache hit ratio</h3>
+<div id="synapse_cache_ratio"></div>
 <script>
 new PromConsole.Graph({
-  node: document.querySelector("#synapse_cache_request_rate"),
-  expr: "rate(synapse_util_caches_cache:total[2m])",
-  name: "[[job]]-[[index]] [[name]]",
+  node: document.querySelector("#synapse_cache_ratio"),
+  expr: "rate(synapse_util_caches_cache:total[2m]) * 100",
+  name: "[[name]]",
   min: 0,
+  max: 100,
   yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-  yUnits: "rps",
-  yTitle: "Cache request rate"
+  yUnits: "%",
+  yTitle: "Percentage"
 })
 </script>
 
@@ -190,7 +191,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_cache_size"),
   expr: "synapse_util_caches_cache:size",
-  name: "[[job]]-[[index]] [[name]]",
+  name: "[[name]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yUnits: "",
@@ -205,8 +206,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_request_count_servlet"),
-  expr: "rate(synapse_http_server_in_flight_requests_count[2m])",
-  name: "[[job]]-[[index]] [[method]] [[servlet]]",
+  expr: "rate(synapse_http_server_request_count:servlet[2m])",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "req/s",
@@ -218,8 +219,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_request_count_servlet_minus_events"),
-  expr: "rate(synapse_http_server_in_flight_requests_count{servlet!=\"EventStreamRestServlet\", servlet!=\"SyncRestServlet\"}[2m])",
-  name: "[[job]]-[[index]] [[method]] [[servlet]]",
+  expr: "rate(synapse_http_server_request_count:servlet{servlet!=\"EventStreamRestServlet\", servlet!=\"SyncRestServlet\"}[2m])",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "req/s",
@@ -232,8 +233,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_response_time_avg"),
-  expr: "rate(synapse_http_server_response_time_seconds_sum[2m]) / rate(synapse_http_server_response_count[2m])",
-  name: "[[job]]-[[index]] [[servlet]]",
+  expr: "rate(synapse_http_server_response_time_seconds[2m]) / rate(synapse_http_server_response_count[2m]) / 1000",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "s/req",
@@ -276,7 +277,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_response_ru_utime"),
   expr: "rate(synapse_http_server_response_ru_utime_seconds[2m])",
-  name: "[[job]]-[[index]] [[servlet]]",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "s/s",
@@ -291,7 +292,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_response_db_txn_duration"),
   expr: "rate(synapse_http_server_response_db_txn_duration_seconds[2m])",
-  name: "[[job]]-[[index]] [[servlet]]",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "s/s",
@@ -305,8 +306,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_send_time_avg"),
-  expr: "rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet'}[2m]) / rate(synapse_http_server_response_count{servlet='RoomSendEventRestServlet'}[2m])",
-  name: "[[job]]-[[index]] [[servlet]]",
+  expr: "rate(synapse_http_server_response_time_second{servlet='RoomSendEventRestServlet'}[2m]) / rate(synapse_http_server_response_count{servlet='RoomSendEventRestServlet'}[2m]) / 1000",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "s/req",
@@ -322,7 +323,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_federation_client_sent"),
   expr: "rate(synapse_federation_client_sent[2m])",
-  name: "[[job]]-[[index]] [[type]]",
+  name: "[[type]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "req/s",
@@ -336,7 +337,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_federation_server_received"),
   expr: "rate(synapse_federation_server_received[2m])",
-  name: "[[job]]-[[index]] [[type]]",
+  name: "[[type]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "req/s",
@@ -366,7 +367,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_notifier_listeners"),
   expr: "synapse_notifier_listeners",
-  name: "[[job]]-[[index]]",
+  name: "listeners",
   min: 0,
   yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
@@ -381,7 +382,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_notifier_notified_events"),
   expr: "rate(synapse_notifier_notified_events[2m])",
-  name: "[[job]]-[[index]]",
+  name: "events",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "events/s",

contrib/prometheus/synapse-v2.rules

@@ -58,21 +58,3 @@ groups:
     labels:
       type: "PDU"
     expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
-
-  - record: synapse_storage_events_persisted_by_source_type
-    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_type="remote"})
-    labels:
-      type: remote
-  - record: synapse_storage_events_persisted_by_source_type
-    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_entity="*client*",origin_type="local"})
-    labels:
-      type: local
-  - record: synapse_storage_events_persisted_by_source_type
-    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_entity!="*client*",origin_type="local"})
-    labels:
-      type: bridges
-  - record: synapse_storage_events_persisted_by_event_type
-    expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep)
-  - record: synapse_storage_events_persisted_by_origin
-    expr: sum without(type) (synapse_storage_events_persisted_events_sep)
-

contrib/purge_api/README.md

@@ -3,9 +3,8 @@ Purge history API examples
 
 # `purge_history.sh`
 
-A bash file, that uses the
-[purge history API](https://matrix-org.github.io/synapse/latest/admin_api/purge_history_api.html)
-to purge all messages in a list of rooms up to a certain event. You can select a 
+A bash file, that uses the [purge history API](/docs/admin_api/purge_history_api.rst) to 
+purge all messages in a list of rooms up to a certain event. You can select a 
 timeframe or a number of messages that you want to keep in the room.
 
 Just configure the variables DOMAIN, ADMIN, ROOMS_ARRAY and TIME at the top of
@@ -13,6 +12,5 @@ the script.
 
 # `purge_remote_media.sh`
 
-A bash file, that uses the
-[purge history API](https://matrix-org.github.io/synapse/latest/admin_api/purge_history_api.html)
-to purge all old cached remote media.
+A bash file, that uses the [purge history API](/docs/admin_api/purge_history_api.rst) to 
+purge all old cached remote media.