disable background updates

.ci/scripts/test_export_data_command.sh

@@ -2,24 +2,29 @@
 
 # Test for the export-data admin command against sqlite and postgres
 
-# Expects Synapse to have been already installed with `poetry install --extras postgres`.
-# Expects `poetry` to be available on the `PATH`.
-
 set -xe
-cd "$(dirname "$0")/../.."
+cd `dirname $0`/../..
+
+echo "--- Install dependencies"
+
+# Install dependencies for this test.
+pip install psycopg2
+
+# Install Synapse itself. This won't update any libraries.
+pip install -e .
 
 echo "--- Generate the signing key"
 
 # Generate the server's signing key.
-poetry run synapse_homeserver --generate-keys -c .ci/sqlite-config.yaml
+python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
 
 echo "--- Prepare test database"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
-poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # Run the export-data command on the sqlite test database
-poetry run python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
+python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
 --output-directory /tmp/export_data
 
 # Test that the output directory exists and contains the rooms directory
@@ -32,14 +37,14 @@ else
 fi
 
 # Create the PostgreSQL database.
-poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
+.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
 # Port the SQLite databse to postgres so we can check command works against postgres
 echo "+++ Port SQLite3 databse to postgres"
-poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 # Run the export-data command on postgres database
-poetry run python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
+python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
 --output-directory /tmp/export_data2
 
 # Test that the output directory exists and contains the rooms directory

.ci/scripts/test_old_deps.sh

@@ -1,81 +1,16 @@
 #!/usr/bin/env bash
-# this script is run by GitHub Actions in a plain `focal` container; it
-# - installs the minimal system requirements, and poetry;
-# - patches the project definition file to refer to old versions only;
-# - creates a venv with these old versions using poetry; and finally
-# - invokes `trial` to run the tests with old deps.
 
-# Prevent tzdata from asking for user input
-export DEBIAN_FRONTEND=noninteractive
+# this script is run by GitHub Actions in a plain `bionic` container; it installs the
+# minimal requirements for tox and hands over to the py3-old tox environment.
 
 set -ex
 
 apt-get update
-apt-get install -y \
-        python3 python3-dev python3-pip python3-venv pipx \
-        libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
+apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox
 
 export LANG="C.UTF-8"
 
 # Prevent virtualenv from auto-updating pip to an incompatible version
 export VIRTUALENV_NO_DOWNLOAD=1
 
-# TODO: in the future, we could use an implementation of
-#   https://github.com/python-poetry/poetry/issues/3527
-#   https://github.com/pypa/pip/issues/8085
-# to select the lowest possible versions, rather than resorting to this sed script.
-
-# Patch the project definitions in-place:
-# - Replace all lower and tilde bounds with exact bounds
-# - Make the pyopenssl 17.0, which is the oldest version that works with
-#   a `cryptography` compiled against OpenSSL 1.1.
-# - Delete all lines referring to psycopg2 --- so no testing of postgres support.
-# - Omit systemd: we're not logging to journal here.
-
-# TODO: also replace caret bounds, see https://python-poetry.org/docs/dependency-specification/#version-constraints
-# We don't use these yet, but IIRC they are the default bound used when you `poetry add`.
-# The sed expression 's/\^/==/g' ought to do the trick. But it would also change
-# `python = "^3.7"` to `python = "==3.7", which would mean we fail because olddeps
-# runs on 3.8 (#12343).
-
-sed -i \
-   -e "s/[~>]=/==/g" \
-   -e "/psycopg2/d" \
-   -e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \
-   -e '/systemd/d' \
-   pyproject.toml
-
-# Use poetry to do the installation. This ensures that the versions are all mutually
-# compatible (as far the package metadata declares, anyway); pip's package resolver
-# is more lax.
-#
-# Rather than `poetry install --no-dev`, we drop all dev dependencies from the
-# toml file. This means we don't have to ensure compatibility between old deps and
-# dev tools.
-
-pip install --user toml
-
-REMOVE_DEV_DEPENDENCIES="
-import toml
-with open('pyproject.toml', 'r') as f:
-    data = toml.loads(f.read())
-
-del data['tool']['poetry']['dev-dependencies']
-
-with open('pyproject.toml', 'w') as f:
-    toml.dump(data, f)
-"
-python3 -c "$REMOVE_DEV_DEPENDENCIES"
-
-pipx install poetry==1.1.12
-~/.local/bin/poetry lock
-
-echo "::group::Patched pyproject.toml"
-cat pyproject.toml
-echo "::endgroup::"
-echo "::group::Lockfile after patch"
-cat poetry.lock
-echo "::endgroup::"
-
-~/.local/bin/poetry install -E "all test"
-~/.local/bin/poetry run trial --jobs=2 tests
+exec tox -e py3-old,combine

.ci/scripts/test_synapse_port_db.sh

@@ -1,37 +1,41 @@
 #!/usr/bin/env bash
 #
 # Test script for 'synapse_port_db'.
-#   - configures synapse and a postgres server.
+#   - sets up synapse and deps
 #   - runs the port script on a prepopulated test sqlite db
 #   - also runs it against an new sqlite db
-#
-# Expects Synapse to have been already installed with `poetry install --extras postgres`.
-# Expects `poetry` to be available on the `PATH`.
+
 
 set -xe
-cd "$(dirname "$0")/../.."
+cd `dirname $0`/../..
+
+echo "--- Install dependencies"
+
+# Install dependencies for this test.
+pip install psycopg2 coverage coverage-enable-subprocess
+
+# Install Synapse itself. This won't update any libraries.
+pip install -e .
 
 echo "--- Generate the signing key"
 
 # Generate the server's signing key.
-poetry run synapse_homeserver --generate-keys -c .ci/sqlite-config.yaml
+python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
 
 echo "--- Prepare test database"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
-poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # Create the PostgreSQL database.
-poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
+.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against test database"
-# TODO: this invocation of synapse_port_db (and others below) used to be prepended with `coverage run`,
-# but coverage seems unable to find the entrypoints installed by `pip install -e .`.
-poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 # We should be able to run twice against the same database.
 echo "+++ Run synapse_port_db a second time"
-poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 #####
 
@@ -42,12 +46,12 @@ echo "--- Prepare empty SQLite database"
 # we do this by deleting the sqlite db, and then doing the same again.
 rm .ci/test_db.db
 
-poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # re-create the PostgreSQL database.
-poetry run .ci/scripts/postgres_exec.py \
+.ci/scripts/postgres_exec.py \
   "DROP DATABASE synapse" \
   "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against empty database"
-poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

.dockerignore

@@ -3,13 +3,11 @@
 
 # things to include
 !docker
+!scripts
 !synapse
-!README.rst
-!pyproject.toml
-!poetry.lock
-
-# TODO: remove these once we have moved over to using poetry-core in pyproject.toml
 !MANIFEST.in
+!README.rst
 !setup.py
+!synctl
 
 **/__pycache__

.flake8

@@ -1,11 +0,0 @@
-# TODO: incorporate this into pyproject.toml if flake8 supports it in the future.
-# See https://github.com/PyCQA/flake8/issues/234
-[flake8]
-# see https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes
-# for error codes. The ones we ignore are:
-#  W503: line break before binary operator
-#  W504: line break after binary operator
-#  E203: whitespace before ':' (which is contrary to pep8?)
-#  E731: do not assign a lambda expression, use a def
-#  E501: Line too long (black enforces this for us)
-ignore=W503,W504,E203,E731,E501

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,14 +1,12 @@
 ### Pull Request Checklist
 
-<!-- Please read https://matrix-org.github.io/synapse/latest/development/contributing_guide.html before submitting your pull request -->
+<!-- Please read CONTRIBUTING.md before submitting your pull request -->
 
 * [ ] Pull request is based on the develop branch
-* [ ] Pull request includes a [changelog file](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should:
+* [ ] Pull request includes a [changelog file](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.md#changelog). The entry should:
   - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.".
   - Use markdown where necessary, mostly for `code blocks`.
   - End with either a period (.) or an exclamation mark (!).
   - Start with a capital letter.
-  - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry.
-* [ ] Pull request includes a [sign off](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#sign-off)
-* [ ] [Code style](https://matrix-org.github.io/synapse/latest/code_style.html) is correct
-  (run the [linters](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#run-the-linters))
+* [ ] Pull request includes a [sign off](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.md#sign-off)
+* [ ] Code style is correct (run the [linters](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.md#code-style))

.github/workflows/docker.yml

@@ -5,7 +5,7 @@ name: Build docker images
 on:
   push:
     tags: ["v*"]
-    branches: [ master, main, develop ]
+    branches: [ master, main ]
   workflow_dispatch:
 
 permissions:
@@ -34,15 +34,10 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      # TODO: consider using https://github.com/docker/metadata-action instead of this
-      # custom magic
       - name: Calculate docker image tag
         id: set-tag
         run: |
           case "${GITHUB_REF}" in
-              refs/heads/develop)
-                  tag=develop
-                  ;;
               refs/heads/master|refs/heads/main)
                   tag=latest
                   ;;
@@ -55,6 +50,18 @@ jobs:
           esac
           echo "::set-output name=tag::$tag"
 
+        # for release builds, we want to get the amd64 image out asap, so first
+        # we do an amd64-only build, before following up with a multiarch build.
+      - name: Build and push amd64
+        uses: docker/build-push-action@v2
+        if: "${{ startsWith(github.ref, 'refs/tags/v') }}"
+        with:
+          push: true
+          labels: "gitsha1=${{ github.sha }}"
+          tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
+          file: "docker/Dockerfile"
+          platforms: linux/amd64
+
       - name: Build and push all platforms
         uses: docker/build-push-action@v2
         with:

.github/workflows/docs.yaml

@@ -22,7 +22,7 @@ jobs:
       - name: Setup mdbook
         uses: peaceiris/actions-mdbook@4b5ef36b314c2599664ca107bb8c02412548d79d # v1.1.14
         with:
-          mdbook-version: '0.4.17'
+          mdbook-version: '0.4.9'
 
       - name: Build the documentation
         # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.

.github/workflows/release-artifacts.yml

@@ -7,7 +7,7 @@ on:
   # of things breaking (but only build one set of debs)
   pull_request:
   push:
-    branches: ["develop", "release-*"]
+    branches: ["develop"]
 
     # we do the full build on tags.
     tags: ["v*"]
@@ -31,7 +31,7 @@ jobs:
           # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
           dists='["debian:sid"]'
           if [[ $GITHUB_REF == refs/tags/* ]]; then
-              dists=$(scripts-dev/build_debian_packages.py --show-dists-json)
+              dists=$(scripts-dev/build_debian_packages --show-dists-json)
           fi
           echo "::set-output name=distros::$dists"
     # map the step outputs to job outputs
@@ -74,7 +74,7 @@ jobs:
         # see https://github.com/docker/build-push-action/issues/252
         # for the cache magic here
         run: |
-          ./src/scripts-dev/build_debian_packages.py \
+          ./src/scripts-dev/build_debian_packages \
             --docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \
             --docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
             --docker-build-arg=--progress=plain \
@@ -91,7 +91,17 @@ jobs:
 
   build-sdist:
     name: "Build pypi distribution files"
-    uses: "matrix-org/backend-meta/.github/workflows/packaging.yml@v1"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: pip install wheel
+      - run: |
+          python setup.py sdist bdist_wheel
+      - uses: actions/upload-artifact@v2
+        with:
+          name: python-dist
+          path: dist/*
 
   # if it's a tag, create a release and attach the artifacts to it
   attach-assets:
@@ -112,8 +122,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           files: |
-            Sdist/*
-            Wheel/*
+            python-dist/*
             debs.tar.xz
           # if it's not already published, keep the release as a draft.
           draft: true

.github/workflows/tests.yml

@@ -10,23 +10,16 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  check-sampleconfig:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install -e .
-      - run: scripts-dev/generate_sample_config.sh --check
-      - run: scripts-dev/config-lint.sh
-
   lint:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         toxenv:
+          - "check-sampleconfig"
           - "check_codestyle"
           - "check_isort"
           - "mypy"
+          - "packaging"
 
     steps:
       - uses: actions/checkout@v2
@@ -50,15 +43,29 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
       - uses: actions/setup-python@v2
-      - run: "pip install 'towncrier>=18.6.0rc1'"
-      - run: scripts-dev/check-newsfragment.sh
+      - run: pip install tox
+      - run: scripts-dev/check-newsfragment
         env:
           PULL_REQUEST_NUMBER: ${{ github.event.number }}
 
+  lint-sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+      - run: pip install wheel
+      - run: python setup.py sdist bdist_wheel
+      - uses: actions/upload-artifact@v2
+        with:
+          name: Python Distributions
+          path: dist/*
+
   # Dummy step to gate other tests on without repeating the whole list
   linting-done:
     if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    needs: [lint, lint-crlf, lint-newsfile, check-sampleconfig]
+    needs: [lint, lint-crlf, lint-newsfile, lint-sdist]
     runs-on: ubuntu-latest
     steps:
       - run: "true"
@@ -69,7 +76,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
         database: ["sqlite"]
         toxenv: ["py"]
         include:
@@ -78,9 +85,9 @@ jobs:
             toxenv: "py-noextras"
 
           # Oldest Python with PostgreSQL
-          - python-version: "3.7"
+          - python-version: "3.6"
             database: "postgres"
-            postgres-version: "10"
+            postgres-version: "9.6"
             toxenv: "py"
 
           # Newest Python with newest PostgreSQL
@@ -128,19 +135,18 @@ jobs:
           || true
 
   trial-olddeps:
-    # Note: sqlite only; no postgres
     if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
     needs: linting-done
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - name: Test with old deps
-        uses: docker://ubuntu:focal # For old python and sqlite
-        # Note: focal seems to be using 3.8, but the oldest is 3.7?
-        # See https://github.com/matrix-org/synapse/issues/12343
+        uses: docker://ubuntu:bionic # For old python and sqlite
         with:
           workdir: /github/workspace
           entrypoint: .ci/scripts/test_old_deps.sh
+        env:
+          TRIAL_FLAGS: "--jobs=2"
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
         if: ${{ always() }}
@@ -161,7 +167,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["pypy-3.7"]
+        python-version: ["pypy-3.6"]
 
     steps:
       - uses: actions/checkout@v2
@@ -207,15 +213,15 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - sytest-tag: focal
+          - sytest-tag: bionic
 
-          - sytest-tag: focal
+          - sytest-tag: bionic
             postgres: postgres
 
           - sytest-tag: testing
             postgres: postgres
 
-          - sytest-tag: focal
+          - sytest-tag: bionic
             postgres: multi-postgres
             workers: workers
 
@@ -271,10 +277,9 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-      - uses: matrix-org/setup-python-poetry@v1
+      - uses: actions/setup-python@v2
         with:
-          python-version: ${{ matrix.python-version }}
-          extras: "postgres"
+          python-version: "3.9"
       - run: .ci/scripts/test_export_data_command.sh
 
   portdb:
@@ -286,8 +291,8 @@ jobs:
     strategy:
       matrix:
         include:
-          - python-version: "3.7"
-            postgres-version: "10"
+          - python-version: "3.6"
+            postgres-version: "9.6"
 
           - python-version: "3.10"
             postgres-version: "14"
@@ -309,39 +314,33 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-      - uses: matrix-org/setup-python-poetry@v1
+      - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-          extras: "postgres"
       - run: .ci/scripts/test_synapse_port_db.sh
 
   complement:
     if: ${{ !failure() && !cancelled() }}
     needs: linting-done
     runs-on: ubuntu-latest
+    container:
+      # https://github.com/matrix-org/complement/blob/master/dockerfiles/ComplementCIBuildkite.Dockerfile
+      image: matrixdotorg/complement:latest
+      env:
+        CI: true
+      ports:
+        - 8448:8448
+      volumes:
+        - /var/run/docker.sock:/var/run/docker.sock
 
     steps:
-      # The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
-      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
-      - name: "Set Go Version"
-        run: |
-          # Add Go 1.17 to the PATH: see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-2
-          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
-          # Add the Go path to the PATH: We need this so we can call gotestfmt
-          echo "~/go/bin" >> $GITHUB_PATH
-
-      - name: "Install Complement Dependencies"
-        run: |
-          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
-
       - name: Run actions/checkout@v2 for synapse
         uses: actions/checkout@v2
         with:
           path: synapse
 
       # Attempt to check out the same branch of Complement as the PR. If it
-      # doesn't exist, fallback to HEAD.
+      # doesn't exist, fallback to master.
       - name: Checkout complement
         shell: bash
         run: |
@@ -354,8 +353,8 @@ jobs:
           #    for pull requests, otherwise GITHUB_REF).
           # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
           #    (GITHUB_BASE_REF for pull requests).
-          # 3. Use the default complement branch ("HEAD").
-          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "HEAD"; do
+          # 3. Use the default complement branch ("master").
+          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
             # Skip empty branch names and merge commits.
             if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
               continue
@@ -364,32 +363,55 @@ jobs:
             (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
           done
 
-      - run: |
-          set -o pipefail
-          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
-        shell: bash
-        name: Run Complement Tests
+      # Build initial Synapse image
+      - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
+        working-directory: synapse
+
+      # Build a ready-to-run Synapse image based on the initial image above.
+      # This new image includes a config file, keys for signing and TLS, and
+      # other settings to make it suitable for testing under Complement.
+      - run: docker build -t complement-synapse -f Synapse.Dockerfile .
+        working-directory: complement/dockerfiles
+
+      # Run Complement
+      - run: go test -v -tags synapse_blacklist,msc2403,msc2946,msc3083 ./tests/...
+        env:
+          COMPLEMENT_BASE_IMAGE: complement-synapse:latest
+        working-directory: complement
 
   # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
   tests-done:
     if: ${{ always() }}
     needs:
-      - check-sampleconfig
       - lint
       - lint-crlf
       - lint-newsfile
+      - lint-sdist
       - trial
       - trial-olddeps
       - sytest
-      - export-data
       - portdb
       - complement
     runs-on: ubuntu-latest
     steps:
-      - uses: matrix-org/done-action@v2
-        with:
-          needs: ${{ toJSON(needs) }}
+      - name: Set build result
+        env:
+          NEEDS_CONTEXT: ${{ toJSON(needs) }}
+        # the `jq` incantation dumps out a series of "<job> <result>" lines.
+        # we set it to an intermediate variable to avoid a pipe, which makes it
+        # hard to set $rc.
+        run: |
+          rc=0
+          results=$(jq -r 'to_entries[] | [.key,.value.result] | join(" ")' <<< $NEEDS_CONTEXT)
+          while read job result ; do
+              # The newsfile lint may be skipped on non PR builds
+              if [ $result == "skipped" ] && [ $job == "lint-newsfile" ]; then
+                continue
+              fi
 
-          # The newsfile lint may be skipped on non PR builds
-          skippable:
-            lint-newsfile
+              if [ "$result" != "success" ]; then
+                  echo "::set-failed ::Job $job returned $result"
+                  rc=1
+              fi
+          done <<< $results
+          exit $rc

.github/workflows/twisted_trunk.yml

@@ -25,7 +25,7 @@ jobs:
       - run: sudo apt-get -qq install xmlsec1
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.6
       - run: .ci/patch_for_twisted_trunk.sh
       - run: pip install tox
       - run: tox -e py

.gitignore

@@ -15,10 +15,6 @@ _trial_temp*/
 .DS_Store
 __pycache__/
 
-# We do want the poetry lockfile. TODO: is there a good reason for ignoring
-# '*.lock' above? If not, let's nuke it.
-!poetry.lock
-
 # stuff that is likely to exist when you run a server locally
 /*.db
 /*.log
@@ -34,9 +30,6 @@ __pycache__/
 /media_store/
 /uploads
 
-# For direnv users
-/.envrc
-
 # IDEs
 /.idea/
 /.ropeproject/
@@ -57,7 +50,3 @@ __pycache__/
 
 # docs
 book/
-
-# complement
-/complement-*
-/master.tar.gz

MANIFEST.in

@@ -1,3 +1,4 @@
+include synctl
 include LICENSE
 include VERSION
 include *.rst
@@ -16,6 +17,7 @@ recursive-include synapse/storage *.txt
 recursive-include synapse/storage *.md
 
 recursive-include docs *
+recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
 recursive-include tests *.py
@@ -43,7 +45,6 @@ include book.toml
 include pyproject.toml
 recursive-include changelog.d *
 
-include .flake8
 prune .circleci
 prune .github
 prune .ci
@@ -51,4 +52,5 @@ prune contrib
 prune debian
 prune demo/etc
 prune docker
+prune snap
 prune stubs

README.rst

@@ -246,7 +246,7 @@ Password reset
 ==============
 
 Users can reset their password through their client. Alternatively, a server admin
-can reset a users password using the `admin API <docs/admin_api/user_admin_api.md#reset-password>`_
+can reset a users password using the `admin API <docs/admin_api/user_admin_api.rst#reset-password>`_
 or by directly editing the database as shown below.
 
 First calculate the hash of the new password::
@@ -312,9 +312,6 @@ We recommend using the demo which starts 3 federated instances running on ports
 
 (to stop, you can use `./demo/stop.sh`)
 
-See the [demo documentation](https://matrix-org.github.io/synapse/develop/development/demo.html)
-for more information.
-
 If you just want to start a single instance of the app and run it directly::
 
     # Create the homeserver.yaml config once

book.toml

@@ -34,14 +34,6 @@ additional-css = [
     "docs/website_files/table-of-contents.css",
     "docs/website_files/remove-nav-buttons.css",
     "docs/website_files/indent-section-headers.css",
-    "docs/website_files/version-picker.css",
 ]
-additional-js = [
-    "docs/website_files/table-of-contents.js",
-    "docs/website_files/version-picker.js",
-    "docs/website_files/version.js",
-]
-theme = "docs/website_files/theme"
-
-[preprocessor.schema_versions]
-command = "./scripts-dev/schema_versions.py"
+additional-js = ["docs/website_files/table-of-contents.js"]
+theme = "docs/website_files/theme"

contrib/docker/docker-compose.yml

@@ -14,7 +14,6 @@ services:
     # failure
     restart: unless-stopped
     # See the readme for a full documentation of the environment settings
-    # NOTE: You must edit homeserver.yaml to use postgres, it defaults to sqlite
     environment:
       - SYNAPSE_CONFIG_PATH=/data/homeserver.yaml
     volumes:

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -193,15 +193,12 @@ class TrivialXmppClient:
         time.sleep(7)
         print("SSRC spammer started")
         while self.running:
-            ssrcMsg = (
-                "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>"
-                % {
-                    "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
-                    "nick": self.userId,
-                    "assrc": self.ssrcs["audio"],
-                    "vssrc": self.ssrcs["video"],
-                }
-            )
+            ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % {
+                "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
+                "nick": self.userId,
+                "assrc": self.ssrcs["audio"],
+                "vssrc": self.ssrcs["video"],
+            }
             res = self.sendIq(ssrcMsg)
             print("reply from ssrc announce: ", res)
             time.sleep(10)

contrib/prometheus/consoles/synapse.html

@@ -92,6 +92,22 @@ new PromConsole.Graph({
 })
 </script>
 
+<h3>Pending calls per tick</h3>
+<div id="reactor_pending_calls"></div>
+<script>
+new PromConsole.Graph({
+  node: document.querySelector("#reactor_pending_calls"),
+  expr: "rate(python_twisted_reactor_pending_calls_sum[30s]) / rate(python_twisted_reactor_pending_calls_count[30s])",
+  name: "[[job]]-[[index]]",
+  min: 0,
+  renderer: "line",
+  height: 150,
+  yAxisFormatter: PromConsole.NumberFormatter.humanize,
+  yHoverFormatter: PromConsole.NumberFormatter.humanize,
+  yTitle: "Pending Calls"
+})
+</script>
+
 <h1>Storage</h1>
 
 <h3>Queries</h3>

contrib/purge_api/purge_history.sh

@@ -84,9 +84,7 @@ AUTH="Authorization: Bearer $TOKEN"
 ###################################################################################################
 # finally start pruning the room:
 ###################################################################################################
-# this will really delete local events, so the messages in the room really
-# disappear unless they are restored by remote federation. This is because
-# we pass {"delete_local_events":true} to the curl invocation below.
+POSTDATA='{"delete_local_events":"true"}' # this will really delete local events, so the messages in the room really disappear unless they are restored by remote federation
 
 for ROOM in "${ROOMS_ARRAY[@]}"; do
     echo "########################################### $(date) ################# "
@@ -106,7 +104,7 @@ for ROOM in "${ROOMS_ARRAY[@]}"; do
       SLEEP=2
       set -x
       # call purge
-      OUT=$(curl --header "$AUTH" -s -d '{"delete_local_events":true}' POST "$API_URL/admin/purge_history/$ROOM/$EVENT_ID")
+      OUT=$(curl --header "$AUTH" -s -d $POSTDATA POST "$API_URL/admin/purge_history/$ROOM/$EVENT_ID")
       PURGE_ID=$(echo "$OUT" |grep purge_id|cut -d'"' -f4 )
       if [ "$PURGE_ID" == "" ]; then
         # probably the history purge is already in progress for $ROOM

debian/build_virtualenv

@@ -15,7 +15,7 @@ export DH_VIRTUALENV_INSTALL_ROOT=/opt/venvs
 # python won't look in the right directory. At least this way, the error will
 # be a *bit* more obvious.
 #
-SNAKE=$(readlink -e /usr/bin/python3)
+SNAKE=`readlink -e /usr/bin/python3`
 
 # try to set the CFLAGS so any compiled C extensions are compiled with the most
 # generic as possible x64 instructions, so that compiling it on a new Intel chip
@@ -24,7 +24,7 @@ SNAKE=$(readlink -e /usr/bin/python3)
 # TODO: add similar things for non-amd64, or figure out a more generic way to
 # do this.
 
-case $(dpkg-architecture -q DEB_HOST_ARCH) in
+case `dpkg-architecture -q DEB_HOST_ARCH` in
     amd64)
         export CFLAGS=-march=x86-64
         ;;
@@ -40,7 +40,6 @@ dh_virtualenv \
     --upgrade-pip \
     --preinstall="lxml" \
     --preinstall="mock" \
-    --preinstall="wheel" \
     --extra-pip-arg="--no-cache-dir" \
     --extra-pip-arg="--compile" \
     --extras="all,systemd,test"
@@ -57,8 +56,8 @@ case "$DEB_BUILD_OPTIONS" in
     *)
         # Copy tests to a temporary directory so that we can put them on the
         # PYTHONPATH without putting the uninstalled synapse on the pythonpath.
-        tmpdir=$(mktemp -d)
-        trap 'rm -r $tmpdir' EXIT
+        tmpdir=`mktemp -d`
+        trap "rm -r $tmpdir" EXIT
 
         cp -r tests "$tmpdir"
 
@@ -99,7 +98,7 @@ esac
         --output-file="${PACKAGE_BUILD_DIR}/etc/matrix-synapse/log.yaml"
 
 # add a dependency on the right version of python to substvars.
-PYPKG=$(basename "$SNAKE")
+PYPKG=`basename $SNAKE`
 echo "synapse:pydepends=$PYPKG" >> debian/matrix-synapse-py3.substvars
 
 

debian/changelog

@@ -1,209 +1,3 @@
-matrix-synapse-py3 (1.57.1) stable; urgency=medium
-
-  * New synapse release 1.57.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 20 Apr 2022 15:27:21 +0100
-
-matrix-synapse-py3 (1.57.0) stable; urgency=medium
-
-  * New synapse release 1.57.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 19 Apr 2022 10:58:42 +0100
-
-matrix-synapse-py3 (1.57.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.57.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 12 Apr 2022 13:36:25 +0100
-
-matrix-synapse-py3 (1.56.0) stable; urgency=medium
-
-  * New synapse release 1.56.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 05 Apr 2022 12:38:39 +0100
-
-matrix-synapse-py3 (1.56.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.56.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 29 Mar 2022 10:40:50 +0100
-
-matrix-synapse-py3 (1.55.2) stable; urgency=medium
-
-  * New synapse release 1.55.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 24 Mar 2022 19:07:11 +0000
-
-matrix-synapse-py3 (1.55.1) stable; urgency=medium
-
-  * New synapse release 1.55.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 24 Mar 2022 17:44:23 +0000
-
-matrix-synapse-py3 (1.55.0) stable; urgency=medium
-
-  * New synapse release 1.55.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 22 Mar 2022 13:59:26 +0000
-
-matrix-synapse-py3 (1.55.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.55.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 15 Mar 2022 10:59:31 +0000
-
-matrix-synapse-py3 (1.54.0) stable; urgency=medium
-
-  * New synapse release 1.54.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 08 Mar 2022 10:54:52 +0000
-
-matrix-synapse-py3 (1.54.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.54.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 02 Mar 2022 10:43:22 +0000
-
-matrix-synapse-py3 (1.53.0) stable; urgency=medium
-
-  * New synapse release 1.53.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 22 Feb 2022 11:32:06 +0000
-
-matrix-synapse-py3 (1.53.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.53.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 15 Feb 2022 10:40:50 +0000
-
-matrix-synapse-py3 (1.52.0) stable; urgency=medium
-
-  * New synapse release 1.52.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 08 Feb 2022 11:34:54 +0000
-
-matrix-synapse-py3 (1.52.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.52.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 01 Feb 2022 11:04:09 +0000
-
-matrix-synapse-py3 (1.51.0) stable; urgency=medium
-
-  * New synapse release 1.51.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 25 Jan 2022 11:28:51 +0000
-
-matrix-synapse-py3 (1.51.0~rc2) stable; urgency=medium
-
-  * New synapse release 1.51.0~rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 24 Jan 2022 12:25:00 +0000
-
-matrix-synapse-py3 (1.51.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.51.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 21 Jan 2022 10:46:02 +0000
-
-matrix-synapse-py3 (1.50.2) stable; urgency=medium
-
-  * New synapse release 1.50.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 24 Jan 2022 13:37:11 +0000
-
-matrix-synapse-py3 (1.50.1) stable; urgency=medium
-
-  * New synapse release 1.50.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 18 Jan 2022 16:06:26 +0000
-
-matrix-synapse-py3 (1.50.0) stable; urgency=medium
-
-  * New synapse release 1.50.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 18 Jan 2022 10:40:38 +0000
-
-matrix-synapse-py3 (1.50.0~rc2) stable; urgency=medium
-
-  * New synapse release 1.50.0~rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 14 Jan 2022 11:18:06 +0000
-
-matrix-synapse-py3 (1.50.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.50.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 05 Jan 2022 12:36:17 +0000
-
-matrix-synapse-py3 (1.49.2) stable; urgency=medium
-
-  * New synapse release 1.49.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 21 Dec 2021 17:31:03 +0000
-
-matrix-synapse-py3 (1.49.1) stable; urgency=medium
-
-  * New synapse release 1.49.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 21 Dec 2021 11:07:30 +0000
-
-matrix-synapse-py3 (1.49.0) stable; urgency=medium
-
-  * New synapse release 1.49.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 14 Dec 2021 12:39:46 +0000
-
-matrix-synapse-py3 (1.49.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.49.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 07 Dec 2021 13:52:21 +0000
-
-matrix-synapse-py3 (1.48.0) stable; urgency=medium
-
-  * New synapse release 1.48.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 30 Nov 2021 11:24:15 +0000
-
-matrix-synapse-py3 (1.48.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.48.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 25 Nov 2021 15:56:03 +0000
-
-matrix-synapse-py3 (1.47.1) stable; urgency=medium
-
-  * New synapse release 1.47.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 19 Nov 2021 13:44:32 +0000
-
-matrix-synapse-py3 (1.47.0) stable; urgency=medium
-
-  * New synapse release 1.47.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 17 Nov 2021 13:09:43 +0000
-
-matrix-synapse-py3 (1.47.0~rc3) stable; urgency=medium
-
-  * New synapse release 1.47.0~rc3.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 16 Nov 2021 14:32:47 +0000
-
-matrix-synapse-py3 (1.47.0~rc2) stable; urgency=medium
-
-  [ Dan Callahan ]
-  * Update scripts to pass Shellcheck lints.
-  * Remove unused Vagrant scripts from debian/ directory.
-  * Allow building Debian packages for any architecture, not just amd64.
-  * Preinstall the "wheel" package when building virtualenvs.
-  * Do not error if /etc/default/matrix-synapse is missing.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.47.0~rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 10 Nov 2021 09:41:01 +0000
-
 matrix-synapse-py3 (1.46.0) stable; urgency=medium
 
   [ Richard van der Hoff ]

debian/control

@@ -19,7 +19,7 @@ Standards-Version: 3.9.8
 Homepage: https://github.com/matrix-org/synapse
 
 Package: matrix-synapse-py3
-Architecture: any
+Architecture: amd64
 Provides: matrix-synapse
 Conflicts:
  matrix-synapse (<< 0.34.0.1-0matrix2),

debian/matrix-synapse-py3.config

@@ -2,7 +2,6 @@
 
 set -e
 
-# shellcheck disable=SC1091
 . /usr/share/debconf/confmodule
 
 # try to update the debconf db according to whatever is in the config files

debian/matrix-synapse-py3.postinst

@@ -1,6 +1,5 @@
 #!/bin/sh -e
 
-# shellcheck disable=SC1091
 . /usr/share/debconf/confmodule
 
 CONFIGFILE_SERVERNAME="/etc/matrix-synapse/conf.d/server_name.yaml"

debian/matrix-synapse.service

@@ -5,7 +5,7 @@ Description=Synapse Matrix homeserver
 Type=notify
 User=matrix-synapse
 WorkingDirectory=/var/lib/matrix-synapse
-EnvironmentFile=-/etc/default/matrix-synapse
+EnvironmentFile=/etc/default/matrix-synapse
 ExecStartPre=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys
 ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/
 ExecReload=/bin/kill -HUP $MAINPID

debian/test/.gitignore

@@ -0,0 +1,2 @@
+.vagrant
+*.log

debian/test/provision.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# provisioning script for vagrant boxes for testing the matrix-synapse debs.
+#
+# Will install the most recent matrix-synapse-py3 deb for this platform from
+# the /debs directory.
+
+set -e
+
+apt-get update
+apt-get install -y lsb-release
+
+deb=`ls /debs/matrix-synapse-py3_*+$(lsb_release -cs)*.deb | sort | tail -n1`
+
+debconf-set-selections <<EOF
+matrix-synapse matrix-synapse/report-stats boolean false
+matrix-synapse matrix-synapse/server-name string localhost:18448
+EOF
+
+dpkg -i "$deb"
+
+sed -i -e '/port: 8...$/{s/8448/18448/; s/8008/18008/}' -e '$aregistration_shared_secret: secret' /etc/matrix-synapse/homeserver.yaml
+systemctl restart matrix-synapse

debian/test/stretch/Vagrantfile

@@ -0,0 +1,13 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+ver = `cd ../../..; dpkg-parsechangelog -S Version`.strip()
+
+Vagrant.configure("2") do |config|
+  config.vm.box = "debian/stretch64"
+
+  config.vm.synced_folder ".", "/vagrant", disabled: true
+  config.vm.synced_folder "../../../../debs", "/debs", type: "nfs"
+
+  config.vm.provision "shell", path: "../provision.sh"
+end

debian/test/xenial/Vagrantfile

@@ -0,0 +1,10 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure("2") do |config|
+  config.vm.box = "ubuntu/xenial64"
+
+  config.vm.synced_folder ".", "/vagrant", disabled: true
+  config.vm.synced_folder "../../../../debs", "/debs"
+  config.vm.provision "shell", path: "../provision.sh"
+end

demo/.gitignore

@@ -1,4 +1,7 @@
-# Ignore all the temporary files from the demo servers.
-8080/
-8081/
-8082/
+*.db
+*.log
+*.log.*
+*.pid
+
+/media_store.*
+/etc

demo/README

@@ -0,0 +1,26 @@
+DO NOT USE THESE DEMO SERVERS IN PRODUCTION
+
+Requires you to have done:
+    python setup.py develop
+
+
+The demo start.sh will start three synapse servers on ports 8080, 8081 and 8082, with host names localhost:$port. This can be easily changed to `hostname`:$port in start.sh if required.
+
+To enable the servers to communicate untrusted ssl certs are used. In order to do this the servers do not check the certs
+and are configured in a highly insecure way. Do not use these configuration files in production.
+
+stop.sh will stop the synapse servers and the webclient.
+
+clean.sh will delete the databases and log files.
+
+To start a completely new set of servers, run:
+
+    ./demo/stop.sh; ./demo/clean.sh && ./demo/start.sh
+
+
+Logs and sqlitedb will be stored in demo/808{0,1,2}.{log,db}
+
+
+
+Also note that when joining a public room on a differnt HS via "#foo:bar.net", then you are (in the current impl) joining a room with room_id "foo". This means that it won't work if your HS already has a room with that name.
+

demo/clean.sh

@@ -4,19 +4,16 @@ set -e
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
-# Ensure that the servers are stopped.
-$DIR/stop.sh
-
 PID_FILE="$DIR/servers.pid"
 
-if [ -f "$PID_FILE" ]; then
+if [ -f $PID_FILE ]; then
     echo "servers.pid exists!"
     exit 1
 fi
 
 for port in 8080 8081 8082; do
-    rm -rf "${DIR:?}/$port"
-    rm -rf "$DIR/media_store.$port"
+    rm -rf $DIR/$port
+    rm -rf $DIR/media_store.$port
 done
 
-rm -rf "${DIR:?}/etc"
+rm -rf $DIR/etc

demo/start.sh

@@ -4,101 +4,98 @@ DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 CWD=$(pwd)
 
-cd "$DIR/.." || exit
+cd "$DIR/.."
 
-PYTHONPATH=$(readlink -f "$(pwd)")
-export PYTHONPATH
+mkdir -p demo/etc
+
+export PYTHONPATH=$(readlink -f $(pwd))
 
 
-echo "$PYTHONPATH"
+echo $PYTHONPATH
 
 for port in 8080 8081 8082; do
     echo "Starting server on port $port... "
 
     https_port=$((port + 400))
     mkdir -p demo/$port
-    pushd demo/$port || exit
+    pushd demo/$port
 
-    # Generate the configuration for the homeserver at localhost:848x.
+    #rm $DIR/etc/$port.config
     python3 -m synapse.app.homeserver \
         --generate-config \
-        --server-name "localhost:$port" \
-        --config-path "$port.config" \
+        -H "localhost:$https_port" \
+        --config-path "$DIR/etc/$port.config" \
         --report-stats no
 
-    if ! grep -F "Customisation made by demo/start.sh" -q "$port.config"; then
-        # Generate TLS keys.
-        openssl req -x509 -newkey rsa:4096 \
-          -keyout "localhost:$port.tls.key" \
-          -out "localhost:$port.tls.crt" \
-          -days 365 -nodes -subj "/O=matrix"
+    if ! grep -F "Customisation made by demo/start.sh" -q  $DIR/etc/$port.config; then
+        printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config
 
-        # Add customisations to the configuration.
-        {
-            printf '\n\n# Customisation made by demo/start.sh\n\n'
-            echo "public_baseurl: http://localhost:$port/"
-            echo 'enable_registration: true'
-            echo 'enable_registration_without_verification: true'
-            echo ''
+        echo "public_baseurl: http://localhost:$port/" >> $DIR/etc/$port.config
 
-			# Warning, this heredoc depends on the interaction of tabs and spaces.
-			# Please don't accidentaly bork me with your fancy settings.
-			listeners=$(cat <<-PORTLISTENERS
-			# Configure server to listen on both $https_port and $port
-			# This overides some of the default settings above
-			listeners:
-			  - port: $https_port
-			    type: http
-			    tls: true
-			    resources:
-			      - names: [client, federation]
+        echo 'enable_registration: true' >> $DIR/etc/$port.config
 
-			  - port: $port
-			    tls: false
-			    bind_addresses: ['::1', '127.0.0.1']
-			    type: http
-			    x_forwarded: true
-			    resources:
-			      - names: [client, federation]
-			        compress: false
-			PORTLISTENERS
-			)
+        # Warning, this heredoc depends on the interaction of tabs and spaces. Please don't
+        # accidentaly bork me with your fancy settings.
+		listeners=$(cat <<-PORTLISTENERS
+		# Configure server to listen on both $https_port and $port
+		# This overides some of the default settings above
+		listeners:
+		  - port: $https_port
+		    type: http
+		    tls: true
+		    resources:
+		      - names: [client, federation]
 
-            echo "${listeners}"
+		  - port: $port
+		    tls: false
+		    bind_addresses: ['::1', '127.0.0.1']
+		    type: http
+		    x_forwarded: true
+		    resources:
+		      - names: [client, federation]
+		        compress: false
+		PORTLISTENERS
+		)
+        echo "${listeners}" >> $DIR/etc/$port.config
 
-            # Disable TLS for the servers
-            printf '\n\n# Disable TLS for the servers.'
-            echo '# DO NOT USE IN PRODUCTION'
-            echo 'use_insecure_ssl_client_just_for_testing_do_not_use: true'
-            echo 'federation_verify_certificates: false'
+        # Disable tls for the servers
+        printf '\n\n# Disable tls on the servers.' >> $DIR/etc/$port.config
+        echo '# DO NOT USE IN PRODUCTION' >> $DIR/etc/$port.config
+        echo 'use_insecure_ssl_client_just_for_testing_do_not_use: true' >> $DIR/etc/$port.config
+        echo 'federation_verify_certificates: false' >> $DIR/etc/$port.config
 
-            # Set paths for the TLS certificates.
-            echo "tls_certificate_path: \"$DIR/$port/localhost:$port.tls.crt\""
-            echo "tls_private_key_path: \"$DIR/$port/localhost:$port.tls.key\""
+        # Set tls paths
+        echo "tls_certificate_path: \"$DIR/etc/localhost:$https_port.tls.crt\"" >> $DIR/etc/$port.config
+        echo "tls_private_key_path: \"$DIR/etc/localhost:$https_port.tls.key\"" >> $DIR/etc/$port.config
 
-            # Ignore keys from the trusted keys server
-            echo '# Ignore keys from the trusted keys server'
-            echo 'trusted_key_servers:'
-            echo '  - server_name: "matrix.org"'
-            echo '    accept_keys_insecurely: true'
-            echo ''
+        # Generate tls keys
+        openssl req -x509 -newkey rsa:4096 -keyout $DIR/etc/localhost\:$https_port.tls.key -out $DIR/etc/localhost\:$https_port.tls.crt -days 365 -nodes -subj "/O=matrix"
 
-			# Allow the servers to communicate over localhost.
-			allow_list=$(cat <<-ALLOW_LIST
-			# Allow the servers to communicate over localhost.
-			ip_range_whitelist:
-			  - '127.0.0.1/8'
-			  - '::1/128'
-			ALLOW_LIST
-			)
+        # Ignore keys from the trusted keys server
+        echo '# Ignore keys from the trusted keys server' >> $DIR/etc/$port.config
+        echo 'trusted_key_servers:' >> $DIR/etc/$port.config
+        echo '  - server_name: "matrix.org"' >> $DIR/etc/$port.config
+        echo '    accept_keys_insecurely: true' >> $DIR/etc/$port.config
 
-            echo "${allow_list}"
-        } >> "$port.config"
+        # Reduce the blacklist
+        blacklist=$(cat <<-BLACK
+		# Set the blacklist so that it doesn't include 127.0.0.1, ::1
+		federation_ip_range_blacklist:
+		  - '10.0.0.0/8'
+		  - '172.16.0.0/12'
+		  - '192.168.0.0/16'
+		  - '100.64.0.0/10'
+		  - '169.254.0.0/16'
+		  - 'fe80::/64'
+		  - 'fc00::/7'
+		BLACK
+		)
+        echo "${blacklist}" >> $DIR/etc/$port.config
     fi
 
     # Check script parameters
     if [ $# -eq 1 ]; then
-        if [ "$1" = "--no-rate-limit" ]; then
+        if [ $1 = "--no-rate-limit" ]; then
 
             # Disable any rate limiting
             ratelimiting=$(cat <<-RC
@@ -140,21 +137,22 @@ for port in 8080 8081 8082; do
 			    burst_count: 1000
 			RC
 			)
-            echo "${ratelimiting}" >> "$port.config"
+            echo "${ratelimiting}" >> $DIR/etc/$port.config
         fi
     fi
 
-    # Always disable reporting of stats if the option is not there.
-    if ! grep -F "report_stats" -q  "$port.config" ; then
-        echo "report_stats: false" >> "$port.config"
+    if ! grep -F "full_twisted_stacktraces" -q  $DIR/etc/$port.config; then
+        echo "full_twisted_stacktraces: true" >> $DIR/etc/$port.config
+    fi
+    if ! grep -F "report_stats" -q  $DIR/etc/$port.config ; then
+        echo "report_stats: false" >> $DIR/etc/$port.config
     fi
 
-    # Run the homeserver in the background.
     python3 -m synapse.app.homeserver \
-        --config-path "$port.config" \
+        --config-path "$DIR/etc/$port.config" \
         -D \
 
-    popd || exit
+    popd
 done
 
-cd "$CWD" || exit
+cd "$CWD"

demo/stop.sh

@@ -8,7 +8,7 @@ for pid_file in $FILES; do
     pid=$(cat "$pid_file")
     if [[ $pid ]]; then
         echo "Killing $pid_file with $pid"
-        kill "$pid"
+        kill $pid
     fi
 done
 

docker/Dockerfile

@@ -1,78 +1,25 @@
 # Dockerfile to build the matrixdotorg/synapse docker images.
 #
-# Note that it uses features which are only available in BuildKit - see
-# https://docs.docker.com/go/buildkit/ for more information.
-#
 # To build the image, run `docker build` command from the root of the
 # synapse repository:
 #
-#    DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile .
+#    docker build -f docker/Dockerfile .
 #
 # There is an optional PYTHON_VERSION build argument which sets the
 # version of python to build against: for example:
 #
-#    DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.10 .
+#    docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.6 .
 #
 
-# Irritatingly, there is no blessed guide on how to distribute an application with its
-# poetry-managed environment in a docker image. We have opted for
-# `poetry export | pip install -r /dev/stdin`, but there are known bugs in
-# in `poetry export` whose fixes (scheduled for poetry 1.2) have yet to be released.
-# In case we get bitten by those bugs in the future, the recommendations here might
-# be useful:
-#     https://github.com/python-poetry/poetry/discussions/1879#discussioncomment-216865
-#     https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker?answertab=scoredesc
-
-
-
-ARG PYTHON_VERSION=3.9
+ARG PYTHON_VERSION=3.8
 
 ###
-### Stage 0: generate requirements.txt
-###
-FROM docker.io/python:${PYTHON_VERSION}-slim as requirements
-
-# RUN --mount is specific to buildkit and is documented at
-# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
-# Here we use it to set up a cache for apt (and below for pip), to improve
-# rebuild speeds on slow connections.
-RUN \
-   --mount=type=cache,target=/var/cache/apt,sharing=locked \
-   --mount=type=cache,target=/var/lib/apt,sharing=locked \
- apt-get update && apt-get install -y git \
-    && rm -rf /var/lib/apt/lists/*
-
-# We install poetry in its own build stage to avoid its dependencies conflicting with
-# synapse's dependencies.
-# We use a specific commit from poetry's master branch instead of our usual 1.1.12,
-# to incorporate fixes to some bugs in `poetry export`. This commit corresponds to
-#    https://github.com/python-poetry/poetry/pull/5156 and
-#    https://github.com/python-poetry/poetry/issues/5141 ;
-# without it, we generate a requirements.txt with incorrect environment markers,
-# which causes necessary packages to be omitted when we `pip install`.
-#
-# NB: In poetry 1.2 `poetry export` will be moved into a plugin; we'll need to also
-# pip install poetry-plugin-export (https://github.com/python-poetry/poetry-plugin-export).
-RUN --mount=type=cache,target=/root/.cache/pip \
-  pip install --user git+https://github.com/python-poetry/poetry.git@fb13b3a676f476177f7937ffa480ee5cff9a90a5
-
-WORKDIR /synapse
-
-# Copy just what we need to run `poetry export`...
-COPY pyproject.toml poetry.lock README.rst /synapse/
-
-RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt
-
-###
-### Stage 1: builder
+### Stage 0: builder
 ###
 FROM docker.io/python:${PYTHON_VERSION}-slim as builder
 
 # install the OS build deps
-RUN \
-   --mount=type=cache,target=/var/cache/apt,sharing=locked \
-   --mount=type=cache,target=/var/lib/apt,sharing=locked \
- apt-get update && apt-get install -y \
+RUN apt-get update && apt-get install -y \
     build-essential \
     libffi-dev \
     libjpeg-dev \
@@ -86,27 +33,30 @@ RUN \
     zlib1g-dev \
     && rm -rf /var/lib/apt/lists/*
 
+# Copy just what we need to pip install
+COPY scripts /synapse/scripts/
+COPY MANIFEST.in README.rst setup.py synctl /synapse/
+COPY synapse/__init__.py /synapse/synapse/__init__.py
+COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py
+
 # To speed up rebuilds, install all of the dependencies before we copy over
-# the whole synapse project, so that this layer in the Docker cache can be
+# the whole synapse project so that we this layer in the Docker cache can be
 # used while you develop on the source
 #
-# This is aiming at installing the `[tool.poetry.depdendencies]` from pyproject.toml.
-COPY --from=requirements /synapse/requirements.txt /synapse/
-RUN --mount=type=cache,target=/root/.cache/pip \
-  pip install --prefix="/install" --no-deps --no-warn-script-location -r /synapse/requirements.txt
+# This is aiming at installing the `install_requires` and `extras_require` from `setup.py`
+RUN pip install --prefix="/install" --no-warn-script-location \
+    /synapse[all]
 
-# Copy over the rest of the synapse source code.
+# Copy over the rest of the project
 COPY synapse /synapse/synapse/
-# ... and what we need to `pip install`.
-# TODO: once pyproject.toml declares poetry-core as its build system, we'll need to copy
-# pyproject.toml here, ditching setup.py and MANIFEST.in.
-COPY setup.py MANIFEST.in README.rst /synapse/
 
-# Install the synapse package itself.
+# Install the synapse package itself and all of its children packages.
+#
+# This is aiming at installing only the `packages=find_packages(...)` from `setup.py
 RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
 
 ###
-### Stage 2: runtime
+### Stage 1: runtime
 ###
 
 FROM docker.io/python:${PYTHON_VERSION}-slim
@@ -116,10 +66,7 @@ LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/syna
 LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git'
 LABEL org.opencontainers.image.licenses='Apache-2.0'
 
-RUN \
-   --mount=type=cache,target=/var/cache/apt,sharing=locked \
-   --mount=type=cache,target=/var/lib/apt,sharing=locked \
-  apt-get update && apt-get install -y \
+RUN apt-get update && apt-get install -y \
     curl \
     gosu \
     libjpeg62-turbo \
@@ -135,6 +82,8 @@ COPY --from=builder /install /usr/local
 COPY ./docker/start.py /start.py
 COPY ./docker/conf /conf
 
+VOLUME ["/data"]
+
 EXPOSE 8008/tcp 8009/tcp 8448/tcp
 
 ENTRYPOINT ["/start.py"]

docker/Dockerfile-dhvirtualenv

@@ -16,7 +16,7 @@ ARG distro=""
 ### Stage 0: build a dh-virtualenv
 ###
 
-# This is only really needed on focal, since other distributions we
+# This is only really needed on bionic and focal, since other distributions we
 # care about have a recent version of dh-virtualenv by default. Unfortunately,
 # it looks like focal is going to be with us for a while.
 #
@@ -36,8 +36,9 @@ RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
         wget
 
 # fetch and unpack the package
+# TODO: Upgrade to 1.2.2 once bionic is dropped (1.2.2 requires debhelper 12; bionic has only 11)
 RUN mkdir /dh-virtualenv
-RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/spotify/dh-virtualenv/archive/refs/tags/1.2.2.tar.gz
+RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/spotify/dh-virtualenv/archive/ac6e1b1.tar.gz
 RUN tar -xv --strip-components=1 -C /dh-virtualenv -f /dh-virtualenv.tar.gz
 
 # install its build deps. We do another apt-cache-update here, because we might
@@ -85,12 +86,12 @@ RUN apt-get update -qq -o Acquire::Languages=none \
         libpq-dev \
         xmlsec1
 
-COPY --from=builder /dh-virtualenv_1.2.2-1_all.deb /
+COPY --from=builder /dh-virtualenv_1.2~dev-1_all.deb /
 
 # install dhvirtualenv. Update the apt cache again first, in case we got a
 # cached cache from docker the first time.
 RUN apt-get update -qq -o Acquire::Languages=none \
-    && apt-get install -yq /dh-virtualenv_1.2.2-1_all.deb
+    && apt-get install -yq /dh-virtualenv_1.2~dev-1_all.deb
 
 WORKDIR /synapse/source
 ENTRYPOINT ["bash","/synapse/source/docker/build_debian.sh"]

docker/Dockerfile-pgtests

@@ -0,0 +1,30 @@
+# Use the Sytest image that comes with a lot of the build dependencies
+# pre-installed
+FROM matrixdotorg/sytest:bionic
+
+# The Sytest image doesn't come with python, so install that
+RUN apt-get update && apt-get -qq install -y python3 python3-dev python3-pip
+
+# We need tox to run the tests in run_pg_tests.sh
+RUN python3 -m pip install tox
+
+# Initialise the db
+RUN su -c '/usr/lib/postgresql/10/bin/initdb -D /var/lib/postgresql/data -E "UTF-8" --lc-collate="C.UTF-8" --lc-ctype="C.UTF-8" --username=postgres' postgres
+
+# Add a user with our UID and GID so that files get created on the host owned
+# by us, not root.
+ARG UID
+ARG GID
+RUN groupadd --gid $GID user
+RUN useradd --uid $UID --gid $GID --groups sudo --no-create-home user
+
+# Ensure we can start postgres by sudo-ing as the postgres user.
+RUN apt-get update && apt-get -qq install -y sudo
+RUN echo "user ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+
+ADD run_pg_tests.sh /run_pg_tests.sh
+# Use the "exec form" of ENTRYPOINT (https://docs.docker.com/engine/reference/builder/#entrypoint)
+# so that we can `docker run` this container and pass arguments to pg_tests.sh
+ENTRYPOINT ["/run_pg_tests.sh"]
+
+USER user

docker/Dockerfile-workers

@@ -14,10 +14,10 @@ COPY ./docker/conf-workers/* /conf/
 # Expose nginx listener port
 EXPOSE 8080/tcp
 
+# Volume for user-editable config files, logs etc.
+VOLUME ["/data"]
+
 # A script to read environment variables and create the necessary
 # files to run the desired worker configuration. Will start supervisord.
 COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
 ENTRYPOINT ["/configure_workers_and_start.py"]
-
-HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
-    CMD /bin/sh /healthcheck.sh

docker/README-testing.md

@@ -10,10 +10,10 @@ Note that running Synapse's unit tests from within the docker image is not suppo
 
 ## Testing with SQLite and single-process Synapse
 
-> Note that `scripts-dev/complement.sh` is a script that will automatically build
+> Note that `scripts-dev/complement.sh` is a script that will automatically build 
 > and run an SQLite-based, single-process of Synapse against Complement.
 
-The instructions below will set up Complement testing for a single-process,
+The instructions below will set up Complement testing for a single-process, 
 SQLite-based Synapse deployment.
 
 Start by building the base Synapse docker image. If you wish to run tests with the latest
@@ -26,22 +26,23 @@ docker build -t matrixdotorg/synapse -f docker/Dockerfile .
 
 This will build an image with the tag `matrixdotorg/synapse`.
 
-Next, build the Synapse image for Complement.
+Next, build the Synapse image for Complement. You will need a local checkout 
+of Complement. Change to the root of your Complement checkout and run:
 
 ```sh
-docker build -t complement-synapse -f "docker/complement/Dockerfile" docker/complement
+docker build -t complement-synapse -f "dockerfiles/Synapse.Dockerfile" dockerfiles
 ```
 
-This will build an image with the tag `complement-synapse`, which can be handed to
-Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
-[Complement's documentation](https://github.com/matrix-org/complement/#running) for
+This will build an image with the tag `complement-synapse`, which can be handed to 
+Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to 
+[Complement's documentation](https://github.com/matrix-org/complement/#running) for 
 how to run the tests, as well as the various available command line flags.
 
 ## Testing with PostgreSQL and single or multi-process Synapse
 
-The above docker image only supports running Synapse with SQLite and in a
-single-process topology. The following instructions are used to build a Synapse image for
-Complement that supports either single or multi-process topology with a PostgreSQL
+The above docker image only supports running Synapse with SQLite and in a 
+single-process topology. The following instructions are used to build a Synapse image for 
+Complement that supports either single or multi-process topology with a PostgreSQL 
 database backend.
 
 As with the single-process image, build the base Synapse docker image. If you wish to run
@@ -54,7 +55,7 @@ docker build -t matrixdotorg/synapse -f docker/Dockerfile .
 
 This will build an image with the tag `matrixdotorg/synapse`.
 
-Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`.
+Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`. 
 Again, from the root of the repository:
 
 ```sh
@@ -63,20 +64,21 @@ docker build -t matrixdotorg/synapse-workers -f docker/Dockerfile-workers .
 
 This will build an image with the tag` matrixdotorg/synapse-workers`.
 
-It's worth noting at this point that this image is fully functional, and
-can be used for testing against locally. See instructions for using the container
+It's worth noting at this point that this image is fully functional, and 
+can be used for testing against locally. See instructions for using the container 
 under
 [Running the Dockerfile-worker image standalone](#running-the-dockerfile-worker-image-standalone)
 below.
 
 Finally, build the Synapse image for Complement, which is based on
-`matrixdotorg/synapse-workers`.
+`matrixdotorg/synapse-workers`. You will need a local checkout of Complement. Change to
+the root of your Complement checkout and run:
 
 ```sh
-docker build -t matrixdotorg/complement-synapse-workers -f docker/complement/SynapseWorkers.Dockerfile docker/complement
+docker build -t matrixdotorg/complement-synapse-workers -f dockerfiles/SynapseWorkers.Dockerfile dockerfiles
 ```
 
-This will build an image with the tag `complement-synapse-workers`, which can be handed to
+This will build an image with the tag `complement-synapse`, which can be handed to
 Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
 [Complement's documentation](https://github.com/matrix-org/complement/#running) for
 how to run the tests, as well as the various available command line flags.
@@ -89,10 +91,10 @@ bundling all necessary components together for a workerised homeserver instance.
 
 This includes any desired Synapse worker processes, a nginx to route traffic accordingly,
 a redis for worker communication and a supervisord instance to start up and monitor all
-processes. You will need to provide your own postgres container to connect to, and TLS
+processes. You will need to provide your own postgres container to connect to, and TLS 
 is not handled by the container.
 
-Once you've built the image using the above instructions, you can run it. Be sure
+Once you've built the image using the above instructions, you can run it. Be sure 
 you've set up a volume according to the [usual Synapse docker instructions](README.md).
 Then run something along the lines of:
 
@@ -110,7 +112,7 @@ docker run -d --name synapse \
     matrixdotorg/synapse-workers
 ```
 
-...substituting `POSTGRES*` variables for those that match a postgres host you have
+...substituting `POSTGRES*` variables for those that match a postgres host you have 
 available (usually a running postgres docker container).
 
 The `SYNAPSE_WORKER_TYPES` environment variable is a comma-separated list of workers to
@@ -128,11 +130,11 @@ Otherwise, `SYNAPSE_WORKER_TYPES` can either be left empty or unset to spawn no
 (leaving only the main process). The container is configured to use redis-based worker
 mode.
 
-Logs for workers and the main process are logged to stdout and can be viewed with
-standard `docker logs` tooling. Worker logs contain their worker name
+Logs for workers and the main process are logged to stdout and can be viewed with 
+standard `docker logs` tooling. Worker logs contain their worker name 
 after the timestamp.
 
 Setting `SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1` will cause worker logs to be written to
 `<data_dir>/logs/<worker_name>.log`. Logs are kept for 1 week and rotate every day at 00:
-00, according to the container's clock. Logging for the main process must still be
+00, according to the container's clock. Logging for the main process must still be 
 configured by modifying the homeserver's log config in your Synapse data volume.

docker/README.md

@@ -65,12 +65,7 @@ The following environment variables are supported in `generate` mode:
 * `SYNAPSE_DATA_DIR`: where the generated config will put persistent data
   such as the database and media store. Defaults to `/data`.
 * `UID`, `GID`: the user id and group id to use for creating the data
-  directories. If unset, and no user is set via `docker run --user`, defaults
-  to `991`, `991`.
-
-## Postgres
-
-By default the config will use SQLite. See the [docs on using Postgres](https://github.com/matrix-org/synapse/blob/develop/docs/postgres.md) for more info on how to use Postgres. Until this section is improved [this issue](https://github.com/matrix-org/synapse/issues/8304) may provide useful information.
+  directories. Defaults to `991`, `991`.
 
 ## Running synapse
 
@@ -102,9 +97,7 @@ The following environment variables are supported in `run` mode:
   `<SYNAPSE_CONFIG_DIR>/homeserver.yaml`.
 * `SYNAPSE_WORKER`: module to execute, used when running synapse with workers.
    Defaults to `synapse.app.homeserver`, which is suitable for non-worker mode.
-* `UID`, `GID`: the user and group id to run Synapse as. If unset, and no user
-  is set via `docker run --user`, defaults to `991`, `991`. Note that this user
-  must have permission to read the config files, and write to the data directories.
+* `UID`, `GID`: the user and group id to run Synapse as. Defaults to `991`, `991`.
 * `TZ`: the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) the container will run with. Defaults to `UTC`.
 
 For more complex setups (e.g. for workers) you can also pass your args directly to synapse using `run` mode. For example like this:
@@ -193,7 +186,7 @@ point to another Dockerfile.
 ## Disabling the healthcheck
 
 If you are using a non-standard port or tls inside docker you can disable the healthcheck
-whilst running the above `docker run` commands.
+whilst running the above `docker run` commands. 
 
 ```
    --no-healthcheck
@@ -219,7 +212,7 @@ If you wish to point the healthcheck at a different port with docker command, ad
 ## Setting the healthcheck in docker-compose file
 
 You can add the following to set a custom healthcheck in a docker compose file.
-You will need docker-compose version >2.1 for this to work.
+You will need docker-compose version >2.1 for this to work. 
 
 ```
 healthcheck:
@@ -233,5 +226,5 @@ healthcheck:
 ## Using jemalloc
 
 Jemalloc is embedded in the image and will be used instead of the default allocator.
-You can read about jemalloc by reading the Synapse
+You can read about jemalloc by reading the Synapse 
 [README](https://github.com/matrix-org/synapse/blob/HEAD/README.rst#help-synapse-is-slow-and-eats-all-my-ram-cpu).

docker/build_debian.sh

@@ -5,7 +5,7 @@
 set -ex
 
 # Get the codename from distro env
-DIST=$(cut -d ':' -f2 <<< "${distro:?}")
+DIST=`cut -d ':' -f2 <<< $distro`
 
 # we get a read-only copy of the source: make a writeable copy
 cp -aT /synapse/source /synapse/build
@@ -17,7 +17,7 @@ cd /synapse/build
 # Section to determine which "component" it should go into (see
 # https://manpages.debian.org/stretch/reprepro/reprepro.1.en.html#GUESSING)
 
-DEB_VERSION=$(dpkg-parsechangelog -SVersion)
+DEB_VERSION=`dpkg-parsechangelog -SVersion`
 case $DEB_VERSION in
     *~rc*|*~a*|*~b*|*~c*)
         sed -ie '/^Section:/c\Section: prerelease' debian/control

docker/complement/Dockerfile

@@ -1,22 +0,0 @@
-# A dockerfile which builds an image suitable for testing Synapse under
-# complement.
-
-ARG SYNAPSE_VERSION=latest
-
-FROM matrixdotorg/synapse:${SYNAPSE_VERSION}
-
-ENV SERVER_NAME=localhost
-
-COPY conf/* /conf/
-
-# generate a signing key
-RUN generate_signing_key -o /conf/server.signing.key
-
-WORKDIR /data
-
-EXPOSE 8008 8448
-
-ENTRYPOINT ["/conf/start.sh"]
-
-HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
-    CMD curl -fSs http://localhost:8008/health || exit 1

docker/complement/README.md

@@ -1 +0,0 @@
-Stuff for building the docker image used for testing under complement.

docker/complement/SynapseWorkers.Dockerfile

@@ -1,73 +0,0 @@
-# This dockerfile builds on top of 'docker/Dockerfile-worker' in matrix-org/synapse
-# by including a built-in postgres instance, as well as setting up the homeserver so
-# that it is ready for testing via Complement.
-#
-# Instructions for building this image from those it depends on is detailed in this guide:
-# https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse
-FROM matrixdotorg/synapse-workers
-
-# Download a caddy server to stand in front of nginx and terminate TLS using Complement's
-# custom CA.
-# We include this near the top of the file in order to cache the result.
-RUN curl -OL "https://github.com/caddyserver/caddy/releases/download/v2.3.0/caddy_2.3.0_linux_amd64.tar.gz" && \
-  tar xzf caddy_2.3.0_linux_amd64.tar.gz && rm caddy_2.3.0_linux_amd64.tar.gz && mv caddy /root
-
-# Install postgresql
-RUN apt-get update
-RUN apt-get install -y postgresql
-
-# Configure a user and create a database for Synapse
-RUN pg_ctlcluster 13 main start &&  su postgres -c "echo \
- \"ALTER USER postgres PASSWORD 'somesecret'; \
- CREATE DATABASE synapse \
-  ENCODING 'UTF8' \
-  LC_COLLATE='C' \
-  LC_CTYPE='C' \
-  template=template0;\" | psql" && pg_ctlcluster 13 main stop
-
-# Modify the shared homeserver config with postgres support, certificate setup
-# and the disabling of rate-limiting
-COPY conf-workers/workers-shared.yaml /conf/workers/shared.yaml
-
-WORKDIR /data
-
-# Copy the caddy config
-COPY conf-workers/caddy.complement.json /root/caddy.json
-
-# Expose caddy's listener ports
-EXPOSE 8008 8448
-
-ENTRYPOINT \
-  # Replace the server name in the caddy config
-  sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json && \
-  # Start postgres
-  pg_ctlcluster 13 main start 2>&1 && \
-  # Start caddy
-  /root/caddy start --config /root/caddy.json 2>&1 && \
-  # Set the server name of the homeserver
-  SYNAPSE_SERVER_NAME=${SERVER_NAME} \
-  # No need to report stats here
-  SYNAPSE_REPORT_STATS=no \
-  # Set postgres authentication details which will be placed in the homeserver config file
-  POSTGRES_PASSWORD=somesecret POSTGRES_USER=postgres POSTGRES_HOST=localhost \
-  # Specify the workers to test with
-  SYNAPSE_WORKER_TYPES="\
-    event_persister, \
-    event_persister, \
-    background_worker, \
-    frontend_proxy, \
-    event_creator, \
-    user_dir, \
-    media_repository, \
-    federation_inbound, \
-    federation_reader, \
-    federation_sender, \
-    synchrotron, \
-    appservice, \
-    pusher" \
-  # Run the script that writes the necessary config files and starts supervisord, which in turn
-  # starts everything else
-  /configure_workers_and_start.py
-
-HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
-    CMD /bin/sh /healthcheck.sh

docker/complement/conf-workers/caddy.complement.json

@@ -1,72 +0,0 @@
-{
-    "apps": {
-      "http": {
-        "servers": {
-          "srv0": {
-            "listen": [
-              ":8448"
-            ],
-            "routes": [
-              {
-                "match": [
-                  {
-                    "host": [
-                      "{{ server_name }}"
-                    ]
-                  }
-                ],
-                "handle": [
-                  {
-                    "handler": "subroute",
-                    "routes": [
-                      {
-                        "handle": [
-                          {
-                            "handler": "reverse_proxy",
-                            "upstreams": [
-                              {
-                                "dial": "localhost:8008"
-                              }
-                            ]
-                          }
-                        ]
-                      }
-                    ]
-                  }
-                ],
-                "terminal": true
-              }
-            ]
-          }
-        }
-      },
-      "tls": {
-        "automation": {
-          "policies": [
-            {
-              "subjects": [
-                "{{ server_name }}"
-              ],
-              "issuers": [
-                {
-                  "module": "internal"
-                }
-              ],
-              "on_demand": true
-            }
-          ]
-        }
-      },
-      "pki": {
-        "certificate_authorities": {
-          "local": {
-            "name": "Complement CA",
-            "root": {
-              "certificate": "/complement/ca/ca.crt",
-              "private_key": "/complement/ca/ca.key"
-            }
-          }
-        }
-      }
-    }
-  }

docker/complement/conf-workers/workers-shared.yaml

@@ -1,72 +0,0 @@
-## Server ##
-report_stats: False
-trusted_key_servers: []
-enable_registration: true
-enable_registration_without_verification: true
-bcrypt_rounds: 4
-
-## Federation ##
-
-# trust certs signed by Complement's CA
-federation_custom_ca_list:
-- /complement/ca/ca.crt
-
-# unblacklist RFC1918 addresses
-federation_ip_range_blacklist: []
-
-# Disable server rate-limiting
-rc_federation:
-  window_size: 1000
-  sleep_limit: 10
-  sleep_delay: 500
-  reject_limit: 99999
-  concurrent: 3
-
-rc_message:
-  per_second: 9999
-  burst_count: 9999
-
-rc_registration:
-  per_second: 9999
-  burst_count: 9999
-
-rc_login:
-  address:
-    per_second: 9999
-    burst_count: 9999
-  account:
-    per_second: 9999
-    burst_count: 9999
-  failed_attempts:
-    per_second: 9999
-    burst_count: 9999
-
-rc_admin_redaction:
-  per_second: 9999
-  burst_count: 9999
-
-rc_joins:
-  local:
-    per_second: 9999
-    burst_count: 9999
-  remote:
-    per_second: 9999
-    burst_count: 9999
-
-federation_rr_transactions_per_room_per_second: 9999
-
-## Experimental Features ##
-
-experimental_features:
-  # Enable history backfilling support
-  msc2716_enabled: true
-  # Enable spaces support
-  spaces_enabled: true
-  # Enable jump to date endpoint
-  msc3030_enabled: true
-
-server_notices:
-  system_mxid_localpart: _server
-  system_mxid_display_name: "Server Alert"
-  system_mxid_avatar_url: ""
-  room_name: "Server Alert"

docker/complement/conf/homeserver.yaml

@@ -1,115 +0,0 @@
-## Server ##
-
-server_name: SERVER_NAME
-log_config: /conf/log_config.yaml
-report_stats: False
-signing_key_path: /conf/server.signing.key
-trusted_key_servers: []
-enable_registration: true
-enable_registration_without_verification: true
-
-## Listeners ##
-
-tls_certificate_path: /conf/server.tls.crt
-tls_private_key_path: /conf/server.tls.key
-bcrypt_rounds: 4
-registration_shared_secret: complement
-
-listeners:
-  - port: 8448
-    bind_addresses: ['::']
-    type: http
-    tls: true
-    resources:
-      - names: [federation]
-
-  - port: 8008
-    bind_addresses: ['::']
-    type: http
-
-    resources:
-      - names: [client]
-
-## Database ##
-
-database:
-  name: "sqlite3"
-  args:
-    # We avoid /data, as it is a volume and is not transferred when the container is committed,
-    # which is a fundamental necessity in complement.
-    database: "/conf/homeserver.db"
-
-## Federation ##
-
-# trust certs signed by the complement CA
-federation_custom_ca_list:
-- /complement/ca/ca.crt
-
-# unblacklist RFC1918 addresses
-ip_range_blacklist: []
-
-# Disable server rate-limiting
-rc_federation:
-  window_size: 1000
-  sleep_limit: 10
-  sleep_delay: 500
-  reject_limit: 99999
-  concurrent: 3
-
-rc_message:
-  per_second: 9999
-  burst_count: 9999
-
-rc_registration:
-  per_second: 9999
-  burst_count: 9999
-
-rc_login:
-  address:
-    per_second: 9999
-    burst_count: 9999
-  account:
-    per_second: 9999
-    burst_count: 9999
-  failed_attempts:
-    per_second: 9999
-    burst_count: 9999
-
-rc_admin_redaction:
-  per_second: 9999
-  burst_count: 9999
-
-rc_joins:
-  local:
-    per_second: 9999
-    burst_count: 9999
-  remote:
-    per_second: 9999
-    burst_count: 9999
-
-federation_rr_transactions_per_room_per_second: 9999
-
-## API Configuration ##
-
-# A list of application service config files to use
-#
-app_service_config_files:
-AS_REGISTRATION_FILES  
-
-## Experimental Features ##
-
-experimental_features:
-  # Enable spaces support
-  spaces_enabled: true
-  # Enable history backfilling support
-  msc2716_enabled: true
-  # server-side support for partial state in /send_join
-  msc3706_enabled: true
-  # Enable jump to date endpoint
-  msc3030_enabled: true
-
-server_notices:
-  system_mxid_localpart: _server
-  system_mxid_display_name: "Server Alert"
-  system_mxid_avatar_url: ""
-  room_name: "Server Alert"

docker/complement/conf/log_config.yaml

@@ -1,24 +0,0 @@
-version: 1
-
-formatters:
-  precise:
-   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
-
-filters:
-  context:
-    (): synapse.logging.context.LoggingContextFilter
-    request: ""
-
-handlers:
-  console:
-    class: logging.StreamHandler
-    formatter: precise
-    filters: [context]
-    # log to stdout, for easier use with 'docker logs'
-    stream: 'ext://sys.stdout'
-
-root:
-    level: INFO
-    handlers: [console]
-
-disable_existing_loggers: false

docker/complement/conf/start.sh

@@ -1,30 +0,0 @@
-#!/bin/sh
-
-set -e
-
-sed -i "s/SERVER_NAME/${SERVER_NAME}/g" /conf/homeserver.yaml
-
-# Add the application service registration files to the homeserver.yaml config
-for filename in /complement/appservice/*.yaml; do
-  [ -f "$filename" ] || break
-
-  as_id=$(basename "$filename" .yaml)
-
-  # Insert the path to the registration file and the AS_REGISTRATION_FILES marker after 
-  # so we can add the next application service in the next iteration of this for loop
-  sed -i "s/AS_REGISTRATION_FILES/  - \/complement\/appservice\/${as_id}.yaml\nAS_REGISTRATION_FILES/g" /conf/homeserver.yaml
-done
-# Remove the AS_REGISTRATION_FILES entry
-sed -i "s/AS_REGISTRATION_FILES//g" /conf/homeserver.yaml
-
-# generate an ssl key and cert for the server, signed by the complement CA
-openssl genrsa -out /conf/server.tls.key 2048
-
-openssl req -new -key /conf/server.tls.key -out /conf/server.tls.csr \
-  -subj "/CN=${SERVER_NAME}"
-openssl x509 -req -in /conf/server.tls.csr \
-  -CA /complement/ca/ca.crt -CAkey /complement/ca/ca.key -set_serial 1 \
-  -out /conf/server.tls.crt
-
-exec python -m synapse.app.homeserver -c /conf/homeserver.yaml "$@"
-

docker/conf-workers/healthcheck.sh.j2

@@ -1,6 +0,0 @@
-#!/bin/sh
-# This healthcheck script is designed to return OK when every 
-# host involved returns OK
-{%- for healthcheck_url in healthcheck_urls %}
-curl -fSs {{ healthcheck_url }} || exit 1
-{%- endfor %}

docker/conf/homeserver.yaml

@@ -148,6 +148,14 @@ bcrypt_rounds: 12
 allow_guest_access: {{ "True" if SYNAPSE_ALLOW_GUEST else "False" }}
 enable_group_creation: true
 
+# The list of identity servers trusted to verify third party
+# identifiers by this server.
+#
+# Also defines the ID server which will be called when an account is
+# deactivated (one will be picked arbitrarily).
+trusted_third_party_id_servers:
+    - matrix.org
+    - vector.im
 
 ## Metrics ###
 

docker/configure_workers_and_start.py

@@ -29,7 +29,6 @@
 import os
 import subprocess
 import sys
-from typing import Any, Dict, Set
 
 import jinja2
 import yaml
@@ -37,7 +36,7 @@ import yaml
 MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
 
 
-WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
+WORKERS_CONFIG = {
     "pusher": {
         "app": "synapse.app.pusher",
         "listener_resources": [],
@@ -49,7 +48,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
         "app": "synapse.app.user_dir",
         "listener_resources": ["client"],
         "endpoint_patterns": [
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/user_directory/search$"
+            "^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$"
         ],
         "shared_extra_conf": {"update_user_directory": False},
         "worker_extra_conf": "",
@@ -86,10 +85,10 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
         "app": "synapse.app.generic_worker",
         "listener_resources": ["client"],
         "endpoint_patterns": [
-            "^/_matrix/client/(v2_alpha|r0|v3)/sync$",
-            "^/_matrix/client/(api/v1|v2_alpha|r0|v3)/events$",
-            "^/_matrix/client/(api/v1|r0|v3)/initialSync$",
-            "^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$",
+            "^/_matrix/client/(v2_alpha|r0)/sync$",
+            "^/_matrix/client/(api/v1|v2_alpha|r0)/events$",
+            "^/_matrix/client/(api/v1|r0)/initialSync$",
+            "^/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync$",
         ],
         "shared_extra_conf": {},
         "worker_extra_conf": "",
@@ -147,11 +146,11 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
         "app": "synapse.app.generic_worker",
         "listener_resources": ["client"],
         "endpoint_patterns": [
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/redact",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/send",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/join/",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/profile/",
+            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/redact",
+            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send",
+            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$",
+            "^/_matrix/client/(api/v1|r0|unstable)/join/",
+            "^/_matrix/client/(api/v1|r0|unstable)/profile/",
         ],
         "shared_extra_conf": {},
         "worker_extra_conf": "",
@@ -159,7 +158,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
     "frontend_proxy": {
         "app": "synapse.app.frontend_proxy",
         "listener_resources": ["client", "replication"],
-        "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|v3|unstable)/keys/upload"],
+        "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|unstable)/keys/upload"],
         "shared_extra_conf": {},
         "worker_extra_conf": (
             "worker_main_http_uri: http://127.0.0.1:%d"
@@ -308,7 +307,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
 
     Args:
         environ: _Environ[str]
-        config_path: The location of the generated Synapse main worker config file.
+        config_path: Where to output the generated Synapse main worker config file.
         data_dir: The location of the synapse data directory. Where log and
             user-facing config files live.
     """
@@ -321,8 +320,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     # and adding a replication listener.
 
     # First read the original config file and extract the listeners block. Then we'll add
-    # another listener for replication. Later we'll write out the result to the shared
-    # config file.
+    # another listener for replication. Later we'll write out the result.
     listeners = [
         {
             "port": 9093,
@@ -357,7 +355,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     #   worker_type: {1234, 1235, ...}}
     # }
     # and will be used to construct 'upstream' nginx directives.
-    nginx_upstreams: Dict[str, Set[int]] = {}
+    nginx_upstreams = {}
 
     # A map of: {"endpoint": "upstream"}, where "upstream" is a str representing what will be
     # placed after the proxy_pass directive. The main benefit to representing this data as a
@@ -386,11 +384,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     # A counter of worker_type -> int. Used for determining the name for a given
     # worker type when generating its config file, as each worker's name is just
     # worker_type + instance #
-    worker_type_counter: Dict[str, int] = {}
-
-    # A list of internal endpoints to healthcheck, starting with the main process
-    # which exists even if no workers do.
-    healthcheck_urls = ["http://localhost:8080/health"]
+    worker_type_counter = {}
 
     # For each worker type specified by the user, create config values
     for worker_type in worker_types:
@@ -410,14 +404,12 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
         # e.g. federation_reader1
         worker_name = worker_type + str(new_worker_count)
         worker_config.update(
-            {"name": worker_name, "port": str(worker_port), "config_path": config_path}
+            {"name": worker_name, "port": worker_port, "config_path": config_path}
         )
 
         # Update the shared config with any worker-type specific options
         shared_config.update(worker_config["shared_extra_conf"])
 
-        healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))
-
         # Check if more than one instance of this worker type has been specified
         worker_type_total_count = worker_types.count(worker_type)
         if worker_type_total_count > 1:
@@ -482,7 +474,6 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
 
     # Determine the load-balancing upstreams to configure
     nginx_upstream_config = ""
-
     for upstream_worker_type, upstream_worker_ports in nginx_upstreams.items():
         body = ""
         for port in upstream_worker_ports:
@@ -519,13 +510,6 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
         worker_config=supervisord_config,
     )
 
-    # healthcheck config
-    convert(
-        "/conf/healthcheck.sh.j2",
-        "/healthcheck.sh",
-        healthcheck_urls=healthcheck_urls,
-    )
-
     # Ensure the logging directory exists
     log_dir = data_dir + "/logs"
     if not os.path.exists(log_dir):

docker/run_pg_tests.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# This script runs the PostgreSQL tests inside a Docker container. It expects
+# the relevant source files to be mounted into /src (done automatically by the
+# caller script). It will set up the database, run it, and then use the tox
+# configuration to run the tests.
+
+set -e
+
+# Set PGUSER so Synapse's tests know what user to connect to the database with
+export PGUSER=postgres
+
+# Start the database
+sudo -u postgres /usr/lib/postgresql/10/bin/pg_ctl -w -D /var/lib/postgresql/data start
+
+# Run the tests
+cd /src
+export TRIAL_FLAGS="-j 4"
+tox --workdir=./.tox-pg-container -e py36-postgres "$@"

docker/start.py

@@ -108,7 +108,7 @@ def generate_config_from_template(config_dir, config_path, environ, ownership):
 
     # Hopefully we already have a signing key, but generate one if not.
     args = [
-        sys.executable,
+        "python",
         "-m",
         "synapse.app.homeserver",
         "--config-path",
@@ -120,7 +120,6 @@ def generate_config_from_template(config_dir, config_path, environ, ownership):
     ]
 
     if ownership is not None:
-        log(f"Setting ownership on /data to {ownership}")
         subprocess.check_output(["chown", "-R", ownership, "/data"])
         args = ["gosu", ownership] + args
 
@@ -145,20 +144,14 @@ def run_generate_config(environ, ownership):
     config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
     data_dir = environ.get("SYNAPSE_DATA_DIR", "/data")
 
-    if ownership is not None:
-        # make sure that synapse has perms to write to the data dir.
-        log(f"Setting ownership on {data_dir} to {ownership}")
-        subprocess.check_output(["chown", ownership, data_dir])
-
     # create a suitable log config from our template
     log_config_file = "%s/%s.log.config" % (config_dir, server_name)
     if not os.path.exists(log_config_file):
         log("Creating log config %s" % (log_config_file,))
         convert("/conf/log.config", log_config_file, environ)
 
-    # generate the main config file, and a signing key.
     args = [
-        sys.executable,
+        "python",
         "-m",
         "synapse.app.homeserver",
         "--server-name",
@@ -175,23 +168,29 @@ def run_generate_config(environ, ownership):
         "--open-private-ports",
     ]
     # log("running %s" % (args, ))
-    os.execv(sys.executable, args)
+
+    if ownership is not None:
+        # make sure that synapse has perms to write to the data dir.
+        subprocess.check_output(["chown", ownership, data_dir])
+
+        args = ["gosu", ownership] + args
+        os.execv("/usr/sbin/gosu", args)
+    else:
+        os.execv("/usr/local/bin/python", args)
 
 
 def main(args, environ):
     mode = args[1] if len(args) > 1 else "run"
-
-    # if we were given an explicit user to switch to, do so
-    ownership = None
-    if "UID" in environ:
-        desired_uid = int(environ["UID"])
-        desired_gid = int(environ.get("GID", "991"))
-        ownership = f"{desired_uid}:{desired_gid}"
-    elif os.getuid() == 0:
-        # otherwise, if we are running as root, use user 991
-        ownership = "991:991"
-
+    desired_uid = int(environ.get("UID", "991"))
+    desired_gid = int(environ.get("GID", "991"))
     synapse_worker = environ.get("SYNAPSE_WORKER", "synapse.app.homeserver")
+    if (desired_uid == os.getuid()) and (desired_gid == os.getgid()):
+        ownership = None
+    else:
+        ownership = "{}:{}".format(desired_uid, desired_gid)
+
+    if ownership is None:
+        log("Will not perform chmod/gosu as UserID already matches request")
 
     # In generate mode, generate a configuration and missing keys, then exit
     if mode == "generate":
@@ -254,12 +253,12 @@ running with 'migrate_config'. See the README for more details.
 
     log("Starting synapse with args " + " ".join(args))
 
-    args = [sys.executable] + args
+    args = ["python"] + args
     if ownership is not None:
         args = ["gosu", ownership] + args
         os.execve("/usr/sbin/gosu", args, environ)
     else:
-        os.execve(sys.executable, args, environ)
+        os.execve("/usr/local/bin/python", args, environ)
 
 
 if __name__ == "__main__":

docs/CAPTCHA_SETUP.md

@@ -15,12 +15,12 @@ in `homeserver.yaml`, to the list of authorized domains. If you have not set
 1. Agree to the terms of service and submit.
 1. Copy your site key and secret key and add them to your `homeserver.yaml`
 configuration file
-    ```yaml
+    ```
     recaptcha_public_key: YOUR_SITE_KEY
     recaptcha_private_key: YOUR_SECRET_KEY
     ```
 1. Enable the CAPTCHA for new registrations
-    ```yaml
+    ```
     enable_registration_captcha: true
     ```
 1. Go to the settings page for the CAPTCHA you just created

docs/MSC1711_certificates_FAQ.md

@@ -0,0 +1,335 @@
+# MSC1711 Certificates FAQ
+
+## Historical Note
+This document was originally written to guide server admins through the upgrade
+path towards Synapse 1.0. Specifically,
+[MSC1711](https://github.com/matrix-org/matrix-doc/blob/main/proposals/1711-x509-for-federation.md)
+required that all servers present valid TLS certificates on their federation
+API. Admins were encouraged to achieve compliance from version 0.99.0 (released
+in February 2019) ahead of version 1.0 (released June 2019) enforcing the
+certificate checks.
+
+Much of what follows is now outdated since most admins will have already
+upgraded, however it may be of use to those with old installs returning to the
+project.
+
+If you are setting up a server from scratch you almost certainly should look at
+the [installation guide](setup/installation.md) instead.
+
+## Introduction
+The goal of Synapse 0.99.0 is to act as a stepping stone to Synapse 1.0.0. It
+supports the r0.1 release of the server to server specification, but is
+compatible with both the legacy Matrix federation behaviour (pre-r0.1) as well
+as post-r0.1 behaviour, in order to allow for a smooth upgrade across the
+federation.
+
+The most important thing to know is that Synapse 1.0.0 will require a valid TLS
+certificate on federation endpoints. Self signed certificates will not be
+sufficient.
+
+Synapse 0.99.0 makes it easy to configure TLS certificates and will
+interoperate with both >= 1.0.0 servers as well as existing servers yet to
+upgrade.
+
+**It is critical that all admins upgrade to 0.99.0 and configure a valid TLS
+certificate.** Admins will have 1 month to do so, after which 1.0.0 will be
+released and those servers without a valid certificate will not longer be able
+to federate with >= 1.0.0 servers.
+
+Full details on how to carry out this configuration change is given
+[below](#configuring-certificates-for-compatibility-with-synapse-100). A
+timeline and some frequently asked questions are also given below.
+
+For more details and context on the release of the r0.1 Server/Server API and
+imminent Matrix 1.0 release, you can also see our
+[main talk from FOSDEM 2019](https://matrix.org/blog/2019/02/04/matrix-at-fosdem-2019/).
+
+## Contents
+* Timeline
+* Configuring certificates for compatibility with Synapse 1.0
+* FAQ
+  * Synapse 0.99.0 has just been released, what do I need to do right now?
+  * How do I upgrade?
+  * What will happen if I do not set up a valid federation certificate
+    immediately?
+  * What will happen if I do nothing at all?
+  * When do I need a SRV record or .well-known URI?
+  * Can I still use an SRV record?
+  * I have created a .well-known URI. Do I still need an SRV record?
+  * It used to work just fine, why are you breaking everything?
+  * Can I manage my own certificates rather than having Synapse renew
+    certificates itself?
+  * Do you still recommend against using a reverse proxy on the federation port?
+  * Do I still need to give my TLS certificates to Synapse if I am using a
+    reverse proxy?
+  * Do I need the same certificate for the client and federation port?
+  * How do I tell Synapse to reload my keys/certificates after I replace them?
+
+## Timeline
+
+**5th Feb 2019  - Synapse 0.99.0 is released.**
+
+All server admins are encouraged to upgrade.
+
+0.99.0:
+
+-   provides support for ACME to make setting up Let's Encrypt certs easy, as
+    well as .well-known support.
+
+-   does not enforce that a valid CA cert is present on the federation API, but
+    rather makes it easy to set one up.
+
+-   provides support for .well-known
+
+Admins should upgrade and configure a valid CA cert. Homeservers that require a
+.well-known entry (see below), should retain their SRV record and use it
+alongside their .well-known record.
+
+**10th June 2019  - Synapse 1.0.0 is released**
+
+1.0.0 is scheduled for release on 10th June. In
+accordance with the the [S2S spec](https://matrix.org/docs/spec/server_server/r0.1.0.html)
+1.0.0 will enforce certificate validity. This means that any homeserver without a
+valid certificate after this point will no longer be able to federate with
+1.0.0 servers.
+
+## Configuring certificates for compatibility with Synapse 1.0.0
+
+### If you do not currently have an SRV record
+
+In this case, your `server_name` points to the host where your Synapse is
+running. There is no need to create a `.well-known` URI or an SRV record, but
+you will need to give Synapse a valid, signed, certificate.
+
+### If you do have an SRV record currently
+
+If you are using an SRV record, your matrix domain (`server_name`) may not
+point to the same host that your Synapse is running on (the 'target
+domain'). (If it does, you can follow the recommendation above; otherwise, read
+on.)
+
+Let's assume that your `server_name` is `example.com`, and your Synapse is
+hosted at a target domain of `customer.example.net`. Currently you should have
+an SRV record which looks like:
+
+```
+_matrix._tcp.example.com. IN SRV 10 5 8000 customer.example.net.
+```
+
+In this situation, you have three choices for how to proceed:
+
+#### Option 1: give Synapse a certificate for your matrix domain
+
+Synapse 1.0 will expect your server to present a TLS certificate for your
+`server_name` (`example.com` in the above example). You can achieve this by acquiring a
+certificate for the `server_name` yourself (for example, using `certbot`), and giving it
+and the key to Synapse via `tls_certificate_path` and `tls_private_key_path`.
+
+#### Option 2: run Synapse behind a reverse proxy
+
+If you have an existing reverse proxy set up with correct TLS certificates for
+your domain, you can simply route all traffic through the reverse proxy by
+updating the SRV record appropriately (or removing it, if the proxy listens on
+8448).
+
+See [the reverse proxy documentation](reverse_proxy.md) for information on setting up a
+reverse proxy.
+
+#### Option 3: add a .well-known file to delegate your matrix traffic
+
+This will allow you to keep Synapse on a separate domain, without having to
+give it a certificate for the matrix domain.
+
+You can do this with a `.well-known` file as follows:
+
+ 1. Keep the SRV record in place - it is needed for backwards compatibility
+    with Synapse 0.34 and earlier.
+
+  2. Give Synapse a certificate corresponding to the target domain
+    (`customer.example.net` in the above example). You can do this by acquire a 
+    certificate for the target domain and giving it to Synapse via `tls_certificate_path`
+    and `tls_private_key_path`.
+
+ 3. Restart Synapse to ensure the new certificate is loaded.
+
+ 4. Arrange for a `.well-known` file at
+    `https://<server_name>/.well-known/matrix/server` with contents:
+
+    ```json
+    {"m.server": "<target server name>"}
+    ```
+
+    where the target server name is resolved as usual (i.e. SRV lookup, falling
+    back to talking to port 8448).
+
+    In the above example, where synapse is listening on port 8000,
+    `https://example.com/.well-known/matrix/server` should have `m.server` set to one of:
+
+    1. `customer.example.net` ─ with a SRV record on
+       `_matrix._tcp.customer.example.com` pointing to port 8000, or:
+
+    2. `customer.example.net` ─ updating synapse to listen on the default port
+       8448, or:
+
+    3. `customer.example.net:8000` ─ ensuring that if there is a reverse proxy
+       on `customer.example.net:8000` it correctly handles HTTP requests with
+       Host header set to `customer.example.net:8000`.
+
+## FAQ
+
+### Synapse 0.99.0 has just been released, what do I need to do right now?
+
+Upgrade as soon as you can in preparation for Synapse 1.0.0, and update your
+TLS certificates as [above](#configuring-certificates-for-compatibility-with-synapse-100).
+
+### What will happen if I do not set up a valid federation certificate immediately?
+
+Nothing initially, but once 1.0.0 is in the wild it will not be possible to
+federate with 1.0.0 servers.
+
+### What will happen if I do nothing at all?
+
+If the admin takes no action at all, and remains on a Synapse < 0.99.0 then the
+homeserver will be unable to federate with those who have implemented
+.well-known. Then, as above, once the month upgrade window has expired the
+homeserver will not be able to federate with any Synapse >= 1.0.0
+
+### When do I need a SRV record or .well-known URI?
+
+If your homeserver listens on the default federation port (8448), and your
+`server_name` points to the host that your homeserver runs on, you do not need an
+SRV record or `.well-known/matrix/server` URI.
+
+For instance, if you registered `example.com` and pointed its DNS A record at a
+fresh Upcloud VPS or similar, you could install Synapse 0.99 on that host,
+giving it a server_name of `example.com`, and it would automatically generate a
+valid TLS certificate for you via Let's Encrypt and no SRV record or
+`.well-known` URI would be needed.
+
+This is the common case, although you can add an SRV record or
+`.well-known/matrix/server` URI for completeness if you wish.
+
+**However**, if your server does not listen on port 8448, or if your `server_name`
+does not point to the host that your homeserver runs on, you will need to let
+other servers know how to find it.
+
+In this case, you should see ["If you do have an SRV record
+currently"](#if-you-do-have-an-srv-record-currently) above.
+
+### Can I still use an SRV record?
+
+Firstly, if you didn't need an SRV record before (because your server is
+listening on port 8448 of your server_name), you certainly don't need one now:
+the defaults are still the same.
+
+If you previously had an SRV record, you can keep using it provided you are
+able to give Synapse a TLS certificate corresponding to your server name. For
+example, suppose you had the following SRV record, which directs matrix traffic
+for example.com to matrix.example.com:443:
+
+```
+_matrix._tcp.example.com. IN SRV 10 5 443 matrix.example.com
+```
+
+In this case, Synapse must be given a certificate for example.com - or be
+configured to acquire one from Let's Encrypt.
+
+If you are unable to give Synapse a certificate for your server_name, you will
+also need to use a .well-known URI instead. However, see also "I have created a
+.well-known URI. Do I still need an SRV record?".
+
+### I have created a .well-known URI. Do I still need an SRV record?
+
+As of Synapse 0.99, Synapse will first check for the existence of a `.well-known`
+URI and follow any delegation it suggests. It will only then check for the
+existence of an SRV record.
+
+That means that the SRV record will often be redundant. However, you should
+remember that there may still be older versions of Synapse in the federation
+which do not understand `.well-known` URIs, so if you removed your SRV record you
+would no longer be able to federate with them.
+
+It is therefore best to leave the SRV record in place for now. Synapse 0.34 and
+earlier will follow the SRV record (and not care about the invalid
+certificate). Synapse 0.99 and later will follow the .well-known URI, with the
+correct certificate chain.
+
+### It used to work just fine, why are you breaking everything?
+
+We have always wanted Matrix servers to be as easy to set up as possible, and
+so back when we started federation in 2014 we didn't want admins to have to go
+through the cumbersome process of buying a valid TLS certificate to run a
+server. This was before Let's Encrypt came along and made getting a free and
+valid TLS certificate straightforward. So instead, we adopted a system based on
+[Perspectives](https://en.wikipedia.org/wiki/Convergence_(SSL)): an approach
+where you check a set of "notary servers" (in practice, homeservers) to vouch
+for the validity of a certificate rather than having it signed by a CA. As long
+as enough different notaries agree on the certificate's validity, then it is
+trusted.
+
+However, in practice this has never worked properly. Most people only use the
+default notary server (matrix.org), leading to inadvertent centralisation which
+we want to eliminate. Meanwhile, we never implemented the full consensus
+algorithm to query the servers participating in a room to determine consensus
+on whether a given certificate is valid. This is fiddly to get right
+(especially in face of sybil attacks), and we found ourselves questioning
+whether it was worth the effort to finish the work and commit to maintaining a
+secure certificate validation system as opposed to focusing on core Matrix
+development.
+
+Meanwhile, Let's Encrypt came along in 2016, and put the final nail in the
+coffin of the Perspectives project (which was already pretty dead). So, the
+Spec Core Team decided that a better approach would be to mandate valid TLS
+certificates for federation alongside the rest of the Web. More details can be
+found in
+[MSC1711](https://github.com/matrix-org/matrix-doc/blob/main/proposals/1711-x509-for-federation.md#background-the-failure-of-the-perspectives-approach).
+
+This results in a breaking change, which is disruptive, but absolutely critical
+for the security model. However, the existence of Let's Encrypt as a trivial
+way to replace the old self-signed certificates with valid CA-signed ones helps
+smooth things over massively, especially as Synapse can now automate Let's
+Encrypt certificate generation if needed.
+
+### Can I manage my own certificates rather than having Synapse renew certificates itself?
+
+Yes, you are welcome to manage your certificates yourself. Synapse will only
+attempt to obtain certificates from Let's Encrypt if you configure it to do
+so.The only requirement is that there is a valid TLS cert present for
+federation end points.
+
+### Do you still recommend against using a reverse proxy on the federation port?
+
+We no longer actively recommend against using a reverse proxy. Many admins will
+find it easier to direct federation traffic to a reverse proxy and manage their
+own TLS certificates, and this is a supported configuration.
+
+See [the reverse proxy documentation](reverse_proxy.md) for information on setting up a
+reverse proxy.
+
+### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy?
+
+Practically speaking, this is no longer necessary.
+
+If you are using a reverse proxy for all of your TLS traffic, then you can set
+`no_tls: True`. In that case, the only reason Synapse needs the certificate is
+to populate a legacy 'tls_fingerprints' field in the federation API. This is
+ignored by Synapse 0.99.0 and later, and the only time pre-0.99 Synapses will
+check it is when attempting to fetch the server keys - and generally this is
+delegated via `matrix.org`, which is on 0.99.0.
+
+However, there is a bug in Synapse 0.99.0
+[4554](<https://github.com/matrix-org/synapse/issues/4554>) which prevents
+Synapse from starting if you do not give it a TLS certificate. To work around
+this, you can give it any TLS certificate at all. This will be fixed soon.
+
+### Do I need the same certificate for the client and federation port?
+
+No. There is nothing stopping you from using different certificates,
+particularly if you are using a reverse proxy. However, Synapse will use the
+same certificate on any ports where TLS is configured.
+
+### How do I tell Synapse to reload my keys/certificates after I replace them?
+
+Synapse will reload the keys and certificates when it receives a SIGHUP - for
+example `kill -HUP $(cat homeserver.pid)`. Alternatively, simply restart
+Synapse, though this will result in downtime while it restarts.

docs/README.md

@@ -50,10 +50,8 @@ build the documentation with:
 mdbook build
 ```
 
-The rendered contents will be outputted to a new `book/` directory at the root of the repository. Please note that 
-index.html is not built by default, it is created by copying over the file `welcome_and_overview.html` to `index.html`
-during deployment. Thus, when running `mdbook serve` locally the book will initially show a 404 in place of the index
-due to the above. Do not be alarmed!
+The rendered contents will be outputted to a new `book/` directory at the root of the repository. You can
+browse the book by opening `book/index.html` in a web browser.
 
 You can also have mdbook host the docs on a local webserver with hot-reload functionality via:
 

docs/SUMMARY.md

@@ -13,6 +13,7 @@
 
 # Upgrading
   - [Upgrading between Synapse Versions](upgrade.md)
+  - [Upgrading from pre-Synapse 1.0](MSC1711_certificates_FAQ.md)
 
 # Usage
   - [Federation](federate.md)
@@ -22,14 +23,13 @@
     - [Structured Logging](structured_logging.md)
     - [Templates](templates.md)
     - [User Authentication](usage/configuration/user_authentication/README.md)
-      - [Single-Sign On](usage/configuration/user_authentication/single_sign_on/README.md)
+      - [Single-Sign On]()
         - [OpenID Connect](openid.md)
-        - [SAML](usage/configuration/user_authentication/single_sign_on/saml.md)
-        - [CAS](usage/configuration/user_authentication/single_sign_on/cas.md)
+        - [SAML]()
+        - [CAS]()
         - [SSO Mapping Providers](sso_mapping_providers.md)
       - [Password Auth Providers](password_auth_providers.md)
       - [JSON Web Tokens](jwt.md)
-      - [Refresh Tokens](usage/configuration/user_authentication/refresh_tokens.md)
     - [Registration Captcha](CAPTCHA_SETUP.md)
     - [Application Services](application_services.md)
     - [Server Notices](server_notices.md)
@@ -44,8 +44,6 @@
         - [Presence router callbacks](modules/presence_router_callbacks.md)
         - [Account validity callbacks](modules/account_validity_callbacks.md)
         - [Password auth provider callbacks](modules/password_auth_provider_callbacks.md)
-        - [Background update controller callbacks](modules/background_update_controller_callbacks.md)
-        - [Account data callbacks](modules/account_data_callbacks.md)
         - [Porting a legacy module to the new interface](modules/porting_legacy_module.md)
     - [Workers](workers.md)
       - [Using `synctl` with Workers](synctl_workers.md)
@@ -53,7 +51,6 @@
   - [Administration](usage/administration/README.md)
     - [Admin API](usage/administration/admin_api/README.md)
       - [Account Validity](admin_api/account_validity.md)
-      - [Background Updates](usage/administration/admin_api/background_updates.md)
       - [Delete Group](admin_api/delete_group.md)
       - [Event Reports](admin_api/event_reports.md)
       - [Media](admin_api/media_admin_api.md)
@@ -66,24 +63,16 @@
       - [Statistics](admin_api/statistics.md)
       - [Users](admin_api/user_admin_api.md)
       - [Server Version](admin_api/version_api.md)
-      - [Federation](usage/administration/admin_api/federation.md)
     - [Manhole](manhole.md)
     - [Monitoring](metrics-howto.md)
-    - [Understanding Synapse Through Grafana Graphs](usage/administration/understanding_synapse_through_grafana_graphs.md)
-    - [Useful SQL for Admins](usage/administration/useful_sql_for_admins.md)
-    - [Database Maintenance Tools](usage/administration/database_maintenance_tools.md)
-    - [State Groups](usage/administration/state_groups.md)
     - [Request log format](usage/administration/request_log.md)
-    - [Admin FAQ](usage/administration/admin_faq.md)
     - [Scripts]()
 
 # Development
   - [Contributing Guide](development/contributing_guide.md)
   - [Code Style](code_style.md)
-  - [Release Cycle](development/releases.md)
   - [Git Usage](development/git.md)
   - [Testing]()
-    - [Demo scripts](development/demo.md)
   - [OpenTracing](opentracing.md)
   - [Database Schemas](development/database_schema.md)
   - [Experimental features](development/experimental_features.md)
@@ -104,4 +93,3 @@
 
 # Other
   - [Dependency Deprecation Policy](deprecation_policy.md)
-  - [Running Synapse on a Single-Board Computer](other/running_synapse_on_single_board_computers.md)

docs/admin_api/account_validity.md

@@ -4,9 +4,6 @@ This API allows a server administrator to manage the validity of an account. To
 use it, you must enable the account validity feature (under
 `account_validity`) in Synapse's configuration.
 
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 ## Renew account
 
 This API extends the validity of an account by as much time as configured in the

docs/admin_api/delete_group.md

@@ -4,11 +4,11 @@ This API lets a server admin delete a local group. Doing so will kick all
 users out of the group so that their clients will correctly handle the group
 being deleted.
 
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 The API is:
 
 ```
 POST /_synapse/admin/v1/delete_group/<group_id>
 ```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: see [Admin API](../usage/administration/admin_api).

docs/admin_api/event_reports.md

@@ -2,13 +2,12 @@
 
 This API returns information about reported events.
 
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 The api is:
 ```
 GET /_synapse/admin/v1/event_reports?from=0&limit=10
 ```
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: see [Admin API](../usage/administration/admin_api).
 
 It returns a JSON body like the following:
 
@@ -95,10 +94,12 @@ The api is:
 ```
 GET /_synapse/admin/v1/event_reports/<report_id>
 ```
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: see [Admin API](../usage/administration/admin_api).
 
 It returns a JSON body like the following:
 
-```json
+```jsonc
 {
     "event_id": "$bNUFCwGzWca1meCGkjp-zwslF-GfVcXukvRLI1_FaVY",
     "event_json": {
@@ -131,7 +132,7 @@ It returns a JSON body like the following:
         },
         "type": "m.room.message",
         "unsigned": {
-            "age_ts": 1592291711430
+            "age_ts": 1592291711430,
         }
     },
     "id": <report_id>,

docs/admin_api/media_admin_api.md

@@ -1,13 +1,24 @@
+# Contents
+- [Querying media](#querying-media)
+  * [List all media in a room](#list-all-media-in-a-room)
+  * [List all media uploaded by a user](#list-all-media-uploaded-by-a-user)
+- [Quarantine media](#quarantine-media)
+  * [Quarantining media by ID](#quarantining-media-by-id)
+  * [Remove media from quarantine by ID](#remove-media-from-quarantine-by-id)
+  * [Quarantining media in a room](#quarantining-media-in-a-room)
+  * [Quarantining all media of a user](#quarantining-all-media-of-a-user)
+  * [Protecting media from being quarantined](#protecting-media-from-being-quarantined)
+  * [Unprotecting media from being quarantined](#unprotecting-media-from-being-quarantined)
+- [Delete local media](#delete-local-media)
+  * [Delete a specific local media](#delete-a-specific-local-media)
+  * [Delete local media by date or size](#delete-local-media-by-date-or-size)
+  * [Delete media uploaded by a user](#delete-media-uploaded-by-a-user)
+- [Purge Remote Media API](#purge-remote-media-api)
+
 # Querying media
 
 These APIs allow extracting media information from the homeserver.
 
-Details about the format of the `media_id` and storage of the media in the file system
-are documented under [media repository](../media_repository.md).
-
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 ## List all media in a room
 
 This API gets a list of known media in a room.
@@ -17,6 +28,8 @@ The API is:
 ```
 GET /_synapse/admin/v1/room/<room_id>/media
 ```
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: see [Admin API](../usage/administration/admin_api).
 
 The API returns a JSON body like the following:
 ```json
@@ -304,5 +317,8 @@ The following fields are returned in the JSON response body:
 
 * `deleted`: integer - The number of media items successfully deleted
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: see [Admin API](../usage/administration/admin_api).
+
 If the user re-requests purged remote media, synapse will re-request the media
 from the originating server.

docs/admin_api/purge_history_api.md

@@ -10,15 +10,15 @@ paginate further back in the room from the point being purged from.
 Note that Synapse requires at least one message in each room, so it will never
 delete the last message in a room.
 
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 The API is:
 
 ```
 POST /_synapse/admin/v1/purge_history/<room_id>[/<event_id>]
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 By default, events sent by local users are not deleted, as they may represent
 the only copies of this content in existence. (Events sent by remote users are
 deleted.)
@@ -27,7 +27,7 @@ Room state data (such as joins, leaves, topic) is always preserved.
 
 To delete local message events as well, set `delete_local_events` in the body:
 
-```json
+```
 {
    "delete_local_events": true
 }
@@ -57,6 +57,9 @@ It is possible to poll for updates on recent purges with a second API;
 GET /_synapse/admin/v1/purge_history_status/<purge_id>
 ```
 
+Again, you will need to authenticate by providing an `access_token` for a
+server admin.
+
 This API returns a JSON body like the following:
 
 ```json
@@ -67,8 +70,6 @@ This API returns a JSON body like the following:
 
 The status will be one of `active`, `complete`, or `failed`.
 
-If `status` is `failed` there will be a string `error` with the error message.
-
 ## Reclaim disk space (Postgres)
 
 To reclaim the disk space and return it to the operating system, you need to run

docs/admin_api/room_membership.md

@@ -5,9 +5,6 @@ to a room with a given `room_id_or_alias`. You can only modify the membership of
 local users. The server administrator must be in the room and have permission to
 invite users.
 
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 ## Parameters
 
 The following parameters are available:
@@ -26,9 +23,12 @@ POST /_synapse/admin/v1/join/<room_id_or_alias>
 }
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: see [Admin API](../usage/administration/admin_api).
+
 Response:
 
-```json
+```
 {
   "room_id": "!636q39766251:server.com"
 }

docs/admin_api/rooms.md

@@ -1,12 +1,20 @@
+# Contents
+- [List Room API](#list-room-api)
+- [Room Details API](#room-details-api)
+- [Room Members API](#room-members-api)
+- [Room State API](#room-state-api)
+- [Delete Room API](#delete-room-api)
+  * [Undoing room shutdowns](#undoing-room-shutdowns)
+- [Make Room Admin API](#make-room-admin-api)
+- [Forward Extremities Admin API](#forward-extremities-admin-api)
+- [Event Context API](#event-context-api)
+
 # List Room API
 
 The List Room admin API allows server admins to get a list of rooms on their
 server. There are various parameters available that allow for filtering and
 sorting the returned list. This API supports pagination.
 
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 **Parameters**
 
 The following query parameters are available:
@@ -30,14 +38,9 @@ The following query parameters are available:
   - `history_visibility` - Rooms are ordered alphabetically by visibility of history of the room.
   - `state_events` - Rooms are ordered by number of state events. Largest to smallest.
 * `dir` - Direction of room order. Either `f` for forwards or `b` for backwards. Setting
-  this value to `b` will reverse the above sort order. Defaults to `f`.
-* `search_term` - Filter rooms by their room name, canonical alias and room id.
-  Specifically, rooms are selected if the search term is contained in
-  - the room's name,
-  - the local part of the room's canonical alias, or
-  - the complete (local and server part) room's id (case sensitive).
-
-  Defaults to no filtering.
+          this value to `b` will reverse the above sort order. Defaults to `f`.
+* `search_term` - Filter rooms by their room name. Search term can be contained in any
+                  part of the room name. Defaults to no filtering.
 
 **Response**
 
@@ -84,7 +87,7 @@ GET /_synapse/admin/v1/rooms
 
 A response body like the following is returned:
 
-```json
+```jsonc
 {
   "rooms": [
     {
@@ -167,7 +170,7 @@ GET /_synapse/admin/v1/rooms?order_by=size
 
 A response body like the following is returned:
 
-```json
+```jsonc
 {
   "rooms": [
     {
@@ -205,7 +208,7 @@ A response body like the following is returned:
     }
   ],
   "offset": 0,
-  "total_rooms": 150,
+  "total_rooms": 150
   "next_token": 100
 }
 ```
@@ -221,7 +224,7 @@ GET /_synapse/admin/v1/rooms?order_by=size&from=100
 
 A response body like the following is returned:
 
-```json
+```jsonc
 {
   "rooms": [
     {
@@ -375,86 +378,9 @@ A response body like the following is returned:
 }
 ```
 
-# Block Room API
-The Block Room admin API allows server admins to block and unblock rooms,
-and query to see if a given room is blocked.
-This API can be used to pre-emptively block a room, even if it's unknown to this
-homeserver. Users will be prevented from joining a blocked room.
-
-## Block or unblock a room
-
-The API is:
-
-```
-PUT /_synapse/admin/v1/rooms/<room_id>/block
-```
-
-with a body of:
-
-```json
-{
-    "block": true
-}
-```
-
-A response body like the following is returned:
-
-```json
-{
-    "block": true
-}
-```
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- `room_id` - The ID of the room.
-
-The following JSON body parameters are available:
-
-- `block` - If `true` the room will be blocked and if `false` the room will be unblocked.
-
-**Response**
-
-The following fields are possible in the JSON response body:
-
-- `block` - A boolean. `true` if the room is blocked, otherwise `false`
-
-## Get block status
-
-The API is:
-
-```
-GET /_synapse/admin/v1/rooms/<room_id>/block
-```
-
-A response body like the following is returned:
-
-```json
-{
-    "block": true,
-    "user_id": "<user_id>"
-}
-```
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- `room_id` - The ID of the room.
-
-**Response**
-
-The following fields are possible in the JSON response body:
-
-- `block` - A boolean. `true` if the room is blocked, otherwise `false`
-- `user_id` - An optional string. If the room is blocked (`block` is `true`) shows
-  the user who has add the room to blocking list. Otherwise it is not displayed.
-
 # Delete Room API
 
-The Delete Room admin API allows server admins to remove rooms from the server
+The Delete Room admin API allows server admins to remove rooms from server
 and block these rooms.
 
 Shuts down a room. Moves all local users and room aliases automatically to a
@@ -465,30 +391,18 @@ The new room will be created with the user specified by the `new_room_user_id` p
 as room administrator and will contain a message explaining what happened. Users invited
 to the new room will have power level `-10` by default, and thus be unable to speak.
 
-If `block` is `true`, users will be prevented from joining the old room.
-This option can in [Version 1](#version-1-old-version) also be used to pre-emptively
-block a room, even if it's unknown to this homeserver. In this case, the room will be
-blocked, and no further action will be taken. If `block` is `false`, attempting to
-delete an unknown room is invalid and will be rejected as a bad request.
+If `block` is `True` it prevents new joins to the old room.
 
 This API will remove all trace of the old room from your database after removing
 all local users. If `purge` is `true` (the default), all traces of the old room will
 be removed from your database after removing all local users. If you do not want
 this to happen, set `purge` to `false`.
-Depending on the amount of history being purged, a call to the API may take
+Depending on the amount of history being purged a call to the API may take
 several minutes or longer.
 
 The local server will only have the power to move local user and room aliases to
 the new room. Users on other servers will be unaffected.
 
-## Version 1 (old version)
-
-This version works synchronously. That means you only get the response once the server has
-finished the action, which may take a long time. If you request the same action
-a second time, and the server has not finished the first one, the second request will block.
-This is fixed in version 2 of this API. The parameters are the same in both APIs.
-This API will become deprecated in the future.
-
 The API is:
 
 ```
@@ -507,6 +421,9 @@ with a body of:
 }
 ```
 
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see [Admin API](../usage/administration/admin_api).
+
 A response body like the following is returned:
 
 ```json
@@ -523,44 +440,6 @@ A response body like the following is returned:
 }
 ```
 
-The parameters and response values have the same format as
-[version 2](#version-2-new-version) of the API.
-
-## Version 2 (new version)
-
-**Note**: This API is new, experimental and "subject to change".
-
-This version works asynchronously, meaning you get the response from server immediately
-while the server works on that task in background. You can then request the status of the action
-to check if it has completed.
-
-The API is:
-
-```
-DELETE /_synapse/admin/v2/rooms/<room_id>
-```
-
-with a body of:
-
-```json
-{
-    "new_room_user_id": "@someuser:example.com",
-    "room_name": "Content Violation Notification",
-    "message": "Bad Room has been shutdown due to content violations on this server. Please review our Terms of Service.",
-    "block": true,
-    "purge": true
-}
-```
-
-The API starts the shut down and purge running, and returns immediately with a JSON body with
-a purge id:
-
-```json
-{
-    "delete_id": "<opaque id>"
-}
-```
-
 **Parameters**
 
 The following parameters should be set in the URL:
@@ -580,10 +459,8 @@ The following JSON body parameters are available:
               `new_room_user_id` in the new room. Ideally this will clearly convey why the
                original room was shut down. Defaults to `Sharing illegal content on this server
                is not permitted and rooms in violation will be blocked.`
-* `block` - Optional. If set to `true`, this room will be added to a blocking list,
-            preventing future attempts to join the room. Rooms can be blocked
-            even if they're not yet known to the homeserver (only with
-            [Version 1](#version-1-old-version) of the API). Defaults to `false`.
+* `block` - Optional. If set to `true`, this room will be added to a blocking list, preventing
+            future attempts to join the room. Defaults to `false`.
 * `purge` - Optional. If set to `true`, it will remove all traces of the room from your database.
             Defaults to `true`.
 * `force_purge` - Optional, and ignored unless `purge` is `true`. If set to `true`, it
@@ -593,124 +470,16 @@ The following JSON body parameters are available:
 
 The JSON body must not be empty. The body must be at least `{}`.
 
-## Status of deleting rooms
-
-**Note**: This API is new, experimental and "subject to change".
-
-It is possible to query the status of the background task for deleting rooms.
-The status can be queried up to 24 hours after completion of the task,
-or until Synapse is restarted (whichever happens first).
-
-### Query by `room_id`
-
-With this API you can get the status of all active deletion tasks, and all those completed in the last 24h,
-for the given `room_id`.
-
-The API is:
-
-```
-GET /_synapse/admin/v2/rooms/<room_id>/delete_status
-```
-
-A response body like the following is returned:
-
-```json
-{
-    "results": [
-        {
-            "delete_id": "delete_id1",
-            "status": "failed",
-            "error": "error message",
-            "shutdown_room": {
-                "kicked_users": [],
-                "failed_to_kick_users": [],
-                "local_aliases": [],
-                "new_room_id": null
-            }
-        }, {
-            "delete_id": "delete_id2",
-            "status": "purging",
-            "shutdown_room": {
-                "kicked_users": [
-                    "@foobar:example.com"
-                ],
-                "failed_to_kick_users": [],
-                "local_aliases": [
-                    "#badroom:example.com",
-                    "#evilsaloon:example.com"
-                ],
-                "new_room_id": "!newroomid:example.com"
-            }
-        }
-    ]
-}
-```
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-* `room_id` - The ID of the room.
-
-### Query by `delete_id`
-
-With this API you can get the status of one specific task by `delete_id`.
-
-The API is:
-
-```
-GET /_synapse/admin/v2/rooms/delete_status/<delete_id>
-```
-
-A response body like the following is returned:
-
-```json
-{
-    "status": "purging",
-    "shutdown_room": {
-        "kicked_users": [
-            "@foobar:example.com"
-        ],
-        "failed_to_kick_users": [],
-        "local_aliases": [
-            "#badroom:example.com",
-            "#evilsaloon:example.com"
-        ],
-        "new_room_id": "!newroomid:example.com"
-    }
-}
-```
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-* `delete_id` - The ID for this delete.
-
-### Response
+**Response**
 
 The following fields are returned in the JSON response body:
 
-- `results` - An array of objects, each containing information about one task.
-  This field is omitted from the result when you query by `delete_id`.
-  Task objects contain the following fields:
-  - `delete_id` - The ID for this purge if you query by `room_id`.
-  - `status` - The status will be one of:
-    - `shutting_down` - The process is removing users from the room.
-    - `purging` - The process is purging the room and event data from database.
-    - `complete` - The process has completed successfully.
-    - `failed` - The process is aborted, an error has occurred.
-  - `error` - A string that shows an error message if `status` is `failed`.
-    Otherwise this field is hidden.
-  - `shutdown_room` - An object containing information about the result of shutting down the room.
-    *Note:* The result is shown after removing the room members.
-    The delete process can still be running. Please pay attention to the `status`.
-    - `kicked_users` - An array of users (`user_id`) that were kicked.
-    - `failed_to_kick_users` - An array of users (`user_id`) that that were not kicked.
-    - `local_aliases` - An array of strings representing the local aliases that were
-      migrated from the old room to the new.
-    - `new_room_id` - A string representing the room ID of the new room, or `null` if
-      no such room was created.
+* `kicked_users` - An array of users (`user_id`) that were kicked.
+* `failed_to_kick_users` - An array of users (`user_id`) that that were not kicked.
+* `local_aliases` - An array of strings representing the local aliases that were migrated from
+                    the old room to the new.
+* `new_room_id` - A string representing the room ID of the new room.
+
 
 ## Undoing room deletions
 
@@ -751,6 +520,16 @@ With all that being said, if you still want to try and recover the room:
 4. If `new_room_user_id` was given, a 'Content Violation' will have been
    created. Consider whether you want to delete that roomm.
 
+## Deprecated endpoint
+
+The previous deprecated API will be removed in a future release, it was:
+
+```
+POST /_synapse/admin/v1/rooms/<room_id>/delete
+```
+
+It behaves the same way than the current endpoint except the path and the method.
+
 # Make Room Admin API
 
 Grants another user the highest power available to a local user who is in the room.

docs/admin_api/statistics.md

@@ -3,15 +3,15 @@
 Returns information about all local media usage of users. Gives the
 possibility to filter them by time and user.
 
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 The API is:
 
 ```
 GET /_synapse/admin/v1/statistics/users/media
 ```
 
+To use it, you will need to authenticate by providing an `access_token`
+for a server admin: see [Admin API](../usage/administration/admin_api).
+
 A response body like the following is returned:
 
 ```json

docs/admin_api/user_admin_api.md

@@ -1,8 +1,5 @@
 # User Admin API
 
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 ## Query User Account
 
 This API returns information about a specific user account.
@@ -13,12 +10,14 @@ The api is:
 GET /_synapse/admin/v2/users/<user_id>
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 It returns a JSON body like the following:
 
-```jsonc
+```json
 {
-    "name": "@user:example.com",
-    "displayname": "User", // can be null if not set
+    "displayname": "User",
     "threepids": [
         {
             "medium": "email",
@@ -33,11 +32,11 @@ It returns a JSON body like the following:
             "validated_at": 1586458409743
         }
     ],
-    "avatar_url": "<avatar_url>",  // can be null if not set
-    "is_guest": 0,
+    "avatar_url": "<avatar_url>",
     "admin": 0,
     "deactivated": 0,
     "shadow_banned": 0,
+    "password_hash": "$2b$12$p9B4GkqYdRTPGD",
     "creation_ts": 1560432506,
     "appservice_id": null,
     "consent_server_notice_sent": null,
@@ -104,6 +103,9 @@ with a body of:
 }
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 Returns HTTP status code:
 - `201` - When a new user object was created.
 - `200` - When a user was modified.
@@ -126,8 +128,7 @@ Body parameters:
   [Sample Configuration File](../usage/configuration/homeserver_sample_config.html)
   section `sso` and `oidc_providers`.
   - `auth_provider` - string. ID of the external identity provider. Value of `idp_id`
-    in the homeserver configuration. Note that no error is raised if the provided
-    value is not in the homeserver configuration.
+    in homeserver configuration.
   - `external_id` - string, user ID in the external identity provider.
 - `avatar_url` - string, optional, must be a
   [MXC URI](https://matrix.org/docs/spec/client_server/r0.6.0#matrix-content-mxc-uris).
@@ -154,6 +155,9 @@ By default, the response is ordered by ascending user ID.
 GET /_synapse/admin/v2/users?from=0&limit=10&guests=false
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -273,6 +277,9 @@ GET /_matrix/client/r0/admin/whois/<userId>
 See also: [Client Server
 API Whois](https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-admin-whois-userid).
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 It returns a JSON body like the following:
 
 ```json
@@ -327,12 +334,15 @@ with a body of:
 }
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 The erase parameter is optional and defaults to `false`.
 An empty body may be passed for backwards compatibility.
 
 The following actions are performed when deactivating an user:
 
-- Try to unbind 3PIDs from the identity server
+- Try to unpind 3PIDs from the identity server
 - Remove all 3PIDs from the homeserver
 - Delete all devices and E2EE keys
 - Delete all access tokens
@@ -342,11 +352,6 @@ The following actions are performed when deactivating an user:
 - Remove the user from the user directory
 - Reject all pending invites
 - Remove all account validity information related to the user
-- Remove the arbitrary data store known as *account data*. For example, this includes:
-    - list of ignored users;
-    - push rules;
-    - secret storage keys; and
-    - cross-signing keys.
 
 The following additional actions are performed during deactivation if `erase`
 is set to `true`:
@@ -360,6 +365,7 @@ The following actions are **NOT** performed. The list may be incomplete.
 - Remove mappings of SSO IDs
 - [Delete media uploaded](#delete-media-uploaded-by-a-user) by user (included avatar images)
 - Delete sent and received messages
+- Delete E2E cross-signing keys
 - Remove the user's creation (registration) timestamp
 - [Remove rate limit overrides](#override-ratelimiting-for-users)
 - Remove from monthly active users
@@ -383,6 +389,9 @@ with a body of:
 }
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 The parameter `new_password` is required.
 The parameter `logout_devices` is optional and defaults to `true`.
 
@@ -395,6 +404,9 @@ The api is:
 GET /_synapse/admin/v1/users/<user_id>/admin
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -422,6 +434,10 @@ with a body of:
 }
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+
 ## List room memberships of a user
 
 Gets a list of all `room_id` that a specific `user_id` is member.
@@ -432,6 +448,9 @@ The API is:
 GET /_synapse/admin/v1/users/<user_id>/joined_rooms
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -461,90 +480,10 @@ The following fields are returned in the JSON response body:
 - `joined_rooms` - An array of `room_id`.
 - `total` - Number of rooms.
 
-## Account Data
-Gets information about account data for a specific `user_id`.
-
-The API is:
-
-```
-GET /_synapse/admin/v1/users/<user_id>/accountdata
-```
-
-A response body like the following is returned:
-
-```json
-{
-    "account_data": {
-        "global": {
-            "m.secret_storage.key.LmIGHTg5W": {
-                "algorithm": "m.secret_storage.v1.aes-hmac-sha2",
-                "iv": "fwjNZatxg==",
-                "mac": "eWh9kNnLWZUNOgnc="
-            },
-            "im.vector.hide_profile": {
-                "hide_profile": true
-            },
-            "org.matrix.preview_urls": {
-                "disable": false
-            },
-            "im.vector.riot.breadcrumb_rooms": {
-                "rooms": [
-                    "!LxcBDAsDUVAfJDEo:matrix.org",
-                    "!MAhRxqasbItjOqxu:matrix.org"
-                ]
-            },
-            "m.accepted_terms": {
-                "accepted": [
-                    "https://example.org/somewhere/privacy-1.2-en.html",
-                    "https://example.org/somewhere/terms-2.0-en.html"
-                ]
-            },
-            "im.vector.setting.breadcrumbs": {
-                "recent_rooms": [
-                    "!MAhRxqasbItqxuEt:matrix.org",
-                    "!ZtSaPCawyWtxiImy:matrix.org"
-                ]
-            }
-        },
-        "rooms": {
-            "!GUdfZSHUJibpiVqHYd:matrix.org": {
-                "m.fully_read": {
-                    "event_id": "$156334540fYIhZ:matrix.org"
-                }
-            },
-            "!tOZwOOiqwCYQkLhV:matrix.org": {
-                "m.fully_read": {
-                    "event_id": "$xjsIyp4_NaVl2yPvIZs_k1Jl8tsC_Sp23wjqXPno"
-                }
-            }
-        }
-    }
-}
-```
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- `user_id` - fully qualified: for example, `@user:server.com`.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- `account_data` - A map containing the account data for the user
-  - `global` - A map containing the global account data for the user
-  - `rooms` - A map containing the account data per room for the user
-
 ## User media
 
 ### List media uploaded by a user
 Gets a list of all local media that a specific `user_id` has created.
-These are media that the user has uploaded themselves
-([local media](../media_repository.md#local-media)), as well as
-[URL preview images](../media_repository.md#url-previews) requested by the user if the
-[feature is enabled](../development/url_previews.md).
-
 By default, the response is ordered by descending creation date and ascending media ID.
 The newest media is on top. You can change the order with parameters
 `order_by` and `dir`.
@@ -555,6 +494,9 @@ The API is:
 GET /_synapse/admin/v1/users/<user_id>/media
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -641,9 +583,7 @@ The following fields are returned in the JSON response body:
   Media objects contain the following fields:
   - `created_ts` - integer - Timestamp when the content was uploaded in ms.
   - `last_access_ts` - integer - Timestamp when the content was last accessed in ms.
-  - `media_id` - string - The id used to refer to the media. Details about the format
-    are documented under
-    [media repository](../media_repository.md).
+  - `media_id` - string - The id used to refer to the media.
   - `media_length` - integer - Length of the media in bytes.
   - `media_type` - string - The MIME-type of the media.
   - `quarantined_by` - string - The user ID that initiated the quarantine request
@@ -671,6 +611,9 @@ The API is:
 DELETE /_synapse/admin/v1/users/<user_id>/media
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -743,6 +686,9 @@ The API is:
 GET /_synapse/admin/v2/users/<user_id>/devices
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -808,6 +754,9 @@ POST /_synapse/admin/v2/users/<user_id>/delete_devices
 }
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 An empty JSON dict is returned.
 
 **Parameters**
@@ -829,6 +778,9 @@ The API is:
 GET /_synapse/admin/v2/users/<user_id>/devices/<device_id>
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -874,6 +826,9 @@ PUT /_synapse/admin/v2/users/<user_id>/devices/<device_id>
 }
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 An empty JSON dict is returned.
 
 **Parameters**
@@ -900,6 +855,9 @@ DELETE /_synapse/admin/v2/users/<user_id>/devices/<device_id>
 {}
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 An empty JSON dict is returned.
 
 **Parameters**
@@ -918,6 +876,9 @@ The API is:
 GET /_synapse/admin/v1/users/<user_id>/pushers
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -987,7 +948,7 @@ The following fields are returned in the JSON response body:
 See also the
 [Client-Server API Spec on pushers](https://matrix.org/docs/spec/client_server/latest#get-matrix-client-r0-pushers).
 
-## Controlling whether a user is shadow-banned
+## Shadow-banning users
 
 Shadow-banning is a useful tool for moderating malicious or egregiously abusive users.
 A shadow-banned users receives successful responses to their client-server API requests,
@@ -1000,19 +961,16 @@ or broken behaviour for the client. A shadow-banned user will not receive any
 notification and it is generally more appropriate to ban or kick abusive users.
 A shadow-banned user will be unable to contact anyone on the server.
 
-To shadow-ban a user the API is:
+The API is:
 
 ```
 POST /_synapse/admin/v1/users/<user_id>/shadow_ban
 ```
 
-To un-shadow-ban a user the API is:
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
 
-```
-DELETE /_synapse/admin/v1/users/<user_id>/shadow_ban
-```
-
-An empty JSON dict is returned in both cases.
+An empty JSON dict is returned.
 
 **Parameters**
 
@@ -1034,6 +992,9 @@ The API is:
 GET /_synapse/admin/v1/users/<user_id>/override_ratelimit
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -1073,6 +1034,9 @@ The API is:
 POST /_synapse/admin/v1/users/<user_id>/override_ratelimit
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -1115,6 +1079,9 @@ The API is:
 DELETE /_synapse/admin/v1/users/<user_id>/override_ratelimit
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 An empty JSON dict is returned.
 
 ```json
@@ -1140,8 +1107,10 @@ This endpoint will work even if registration is disabled on the server, unlike
 The API is:
 
 ```
-GET /_synapse/admin/v1/username_available?username=$localpart
+POST /_synapse/admin/v1/username_availabile?username=$localpart
 ```
 
-The request and response format is the same as the
-[/_matrix/client/r0/register/available](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available) API.
+The request and response format is the same as the [/_matrix/client/r0/register/available](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available) API.
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)

docs/admin_api/version_api.md

@@ -16,6 +16,6 @@ It returns a JSON body like the following:
 ```json
 {
     "server_version": "0.99.2rc1 (b=develop, abcdef123)",
-    "python_version": "3.7.8"
+    "python_version": "3.6.8"
 }
 ```

docs/ancient_architecture_notes.md

@@ -7,7 +7,7 @@
 
 ## Server to Server Stack
 
-To use the server to server stack, homeservers should only need to
+To use the server to server stack, home servers should only need to
 interact with the Messaging layer.
 
 The server to server side of things is designed into 4 distinct layers:
@@ -23,7 +23,7 @@ Server with a domain specific API.
 
 1. **Messaging Layer**
 
-    This is what the rest of the homeserver hits to send messages, join rooms,
+    This is what the rest of the Home Server hits to send messages, join rooms,
     etc. It also allows you to register callbacks for when it get's notified by
     lower levels that e.g. a new message has been received.
 
@@ -45,7 +45,7 @@ Server with a domain specific API.
 
     For incoming PDUs, it has to check the PDUs it references to see
     if we have missed any. If we have go and ask someone (another
-    homeserver) for it.
+    home server) for it.
 
 3. **Transaction Layer**
 

docs/changelogs/README.md

@@ -1 +0,0 @@
-This directory contains changelogs for previous years.

docs/code_style.md

@@ -10,9 +10,7 @@ The necessary tools are detailed below.
 
 First install them with:
 
-```sh
-pip install -e ".[lint,mypy]"
-```
+    pip install -e ".[lint,mypy]"
 
 -   **black**
 
@@ -23,9 +21,7 @@ pip install -e ".[lint,mypy]"
     Have `black` auto-format your code (it shouldn't change any
     functionality) with:
 
-    ```sh
-    black .
-    ```
+        black . --exclude="\.tox|build|env"
 
 -   **flake8**
 
@@ -34,9 +30,7 @@ pip install -e ".[lint,mypy]"
 
     Check all application and test code with:
 
-    ```sh
-    flake8 .
-    ```
+        flake8 synapse tests
 
 -   **isort**
 
@@ -45,9 +39,9 @@ pip install -e ".[lint,mypy]"
 
     Auto-fix imports with:
 
-    ```sh
-    isort .
-    ```
+        isort -rc synapse tests
+
+    `-rc` means to recursively search the given directories.
 
 It's worth noting that modern IDEs and text editors can run these tools
 automatically on save. It may be worth looking into whether this
@@ -72,19 +66,15 @@ save as it takes a while and is very resource intensive.
 
         Example:
 
-        ```python
-        from synapse.types import UserID
-        ...
-        user_id = UserID(local, server)
-        ```
+            from synapse.types import UserID
+            ...
+            user_id = UserID(local, server)
 
         is preferred over:
 
-        ```python
-        from synapse import types
-        ...
-        user_id = types.UserID(local, server)
-        ```
+            from synapse import types
+            ...
+            user_id = types.UserID(local, server)
 
         (or any other variant).
 
@@ -144,32 +134,30 @@ Some guidelines follow:
 
 Example:
 
-```yaml
-## Frobnication ##
+    ## Frobnication ##
 
-# The frobnicator will ensure that all requests are fully frobnicated.
-# To enable it, uncomment the following.
-#
-#frobnicator_enabled: true
+    # The frobnicator will ensure that all requests are fully frobnicated.
+    # To enable it, uncomment the following.
+    #
+    #frobnicator_enabled: true
 
-# By default, the frobnicator will frobnicate with the default frobber.
-# The following will make it use an alternative frobber.
-#
-#frobincator_frobber: special_frobber
+    # By default, the frobnicator will frobnicate with the default frobber.
+    # The following will make it use an alternative frobber.
+    #
+    #frobincator_frobber: special_frobber
 
-# Settings for the frobber
-#
-frobber:
-  # frobbing speed. Defaults to 1.
-  #
-  #speed: 10
+    # Settings for the frobber
+    #
+    frobber:
+      # frobbing speed. Defaults to 1.
+      #
+      #speed: 10
 
-  # frobbing distance. Defaults to 1000.
-  #
-  #distance: 100
-```
+      # frobbing distance. Defaults to 1000.
+      #
+      #distance: 100
 
 Note that the sample configuration is generated from the synapse code
-and is maintained by a script, `scripts-dev/generate_sample_config.sh`.
+and is maintained by a script, `scripts-dev/generate_sample_config`.
 Making sure that the output from this script matches the desired format
 is left as an exercise for the reader!

docs/consent_tracking.md

@@ -99,7 +99,7 @@ construct URIs where users can give their consent.
    see if an unauthenticated user is viewing the page. This is typically
    wrapped around the form that would be used to actually agree to the document:
 
-   ```html
+   ```
    {% if not public_version %}
      <!-- The variables used here are only provided when the 'u' param is given to the homeserver -->
      <form method="post" action="consent">

docs/delegate.md

@@ -1,8 +1,4 @@
-# Delegation of incoming federation traffic
-
-In the following documentation, we use the term `server_name` to refer to that setting
-in your homeserver configuration file. It appears at the ends of user ids, and tells
-other homeservers where they can find your server.
+# Delegation
 
 By default, other homeservers will expect to be able to reach yours via
 your `server_name`, on port 8448. For example, if you set your `server_name`
@@ -16,21 +12,13 @@ to a different server and/or port (e.g. `synapse.example.com:443`).
 
 ## .well-known delegation
 
-To use this method, you need to be able to configure the server at
-`https://<server_name>` to serve a file at
-`https://<server_name>/.well-known/matrix/server`.  There are two ways to do this, shown below.
+To use this method, you need to be able to alter the
+`server_name` 's https server to serve the `/.well-known/matrix/server`
+URL. Having an active server (with a valid TLS certificate) serving your
+`server_name` domain is out of the scope of this documentation.
 
-Note that the `.well-known` file is hosted on the default port for `https` (port 443).
-
-### External server
-
-For maximum flexibility, you need to configure an external server such as nginx, Apache
-or HAProxy to serve the `https://<server_name>/.well-known/matrix/server` file. Setting
-up such a server is out of the scope of this documentation, but note that it is often
-possible to configure your [reverse proxy](reverse_proxy.md) for this.
-
-The URL `https://<server_name>/.well-known/matrix/server` should be configured
-return a JSON structure containing the key `m.server` like this:
+The URL `https://<server_name>/.well-known/matrix/server` should
+return a JSON structure containing the key `m.server` like so:
 
 ```json
 {
@@ -38,9 +26,8 @@ return a JSON structure containing the key `m.server` like this:
 }
 ```
 
-In our example (where we want federation traffic to be routed to
-`https://synapse.example.com`, on port 443), this would mean that
-`https://example.com/.well-known/matrix/server` should return:
+In our example, this would mean that URL `https://example.com/.well-known/matrix/server`
+should return:
 
 ```json
 {
@@ -51,29 +38,16 @@ In our example (where we want federation traffic to be routed to
 Note, specifying a port is optional. If no port is specified, then it defaults
 to 8448.
 
-### Serving a `.well-known/matrix/server` file with Synapse
-
-If you are able to set up your domain so that `https://<server_name>` is routed to
-Synapse (i.e., the only change needed is to direct federation traffic to port 443
-instead of port 8448), then it is possible to configure Synapse to serve a suitable
-`.well-known/matrix/server` file. To do so, add the following to your `homeserver.yaml`
-file:
-
-```yaml
-serve_server_wellknown: true
-```
-
-**Note**: this *only* works if `https://<server_name>` is routed to Synapse, so is
-generally not suitable if Synapse is hosted at a subdomain such as
-`https://synapse.example.com`.
+With .well-known delegation, federating servers will check for a valid TLS
+certificate for the delegated hostname (in our example: `synapse.example.com`).
 
 ## SRV DNS record delegation
 
-It is also possible to do delegation using a SRV DNS record. However, that is generally
-not recommended, as it can be difficult to configure the TLS certificates correctly in
-this case, and it offers little advantage over `.well-known` delegation.
+It is also possible to do delegation using a SRV DNS record. However, that is
+considered an advanced topic since it's a bit complex to set up, and `.well-known`
+delegation is already enough in most cases.
 
-However, if you really need it, you can find some documentation on what such a
+However, if you really need it, you can find some documentation on how such a
 record should look like and how Synapse will use it in [the Matrix
 specification](https://matrix.org/docs/spec/server_server/latest#resolving-server-names).
 
@@ -94,9 +68,27 @@ wouldn't need any delegation set up.
 domain `server_name` points to, you will need to let other servers know how to
 find it using delegation.
 
-### Should I use a reverse proxy for federation traffic?
+### Do you still recommend against using a reverse proxy on the federation port?
 
-Generally, using a reverse proxy for both the federation and client traffic is a good
-idea, since it saves handling TLS traffic in Synapse. See
-[the reverse proxy documentation](reverse_proxy.md) for information on setting up a
+We no longer actively recommend against using a reverse proxy. Many admins will
+find it easier to direct federation traffic to a reverse proxy and manage their
+own TLS certificates, and this is a supported configuration.
+
+See [the reverse proxy documentation](reverse_proxy.md) for information on setting up a
 reverse proxy.
+
+### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy?
+
+This is no longer necessary. If you are using a reverse proxy for all of your
+TLS traffic, then you can set `no_tls: True` in the Synapse config.
+
+In that case, the only reason Synapse needs the certificate is to populate a legacy
+`tls_fingerprints` field in the federation API. This is ignored by Synapse 0.99.0
+and later, and the only time pre-0.99 Synapses will check it is when attempting to
+fetch the server keys - and generally this is delegated via `matrix.org`, which
+is running a modern version of Synapse.
+
+### Do I need the same certificate for the client and federation port?
+
+No. There is nothing stopping you from using different certificates,
+particularly if you are using a reverse proxy.

docs/deprecation_policy.md

@@ -14,8 +14,8 @@ i.e. when a version reaches End of Life Synapse will withdraw support for that
 version in future releases.
 
 Details on the upstream support life cycles for Python and PostgreSQL are
-documented at [https://endoflife.date/python](https://endoflife.date/python) and
-[https://endoflife.date/postgresql](https://endoflife.date/postgresql).
+documented at https://endoflife.date/python and
+https://endoflife.date/postgresql.
 
 
 Context

docs/development/cas.md

@@ -8,23 +8,23 @@ easy to run CAS implementation built on top of Django.
 1. Create a new virtualenv: `python3 -m venv <your virtualenv>`
 2. Activate your virtualenv: `source /path/to/your/virtualenv/bin/activate`
 3. Install Django and django-mama-cas:
-   ```sh
+   ```
    python -m pip install "django<3" "django-mama-cas==2.4.0"
    ```
 4. Create a Django project in the current directory:
-   ```sh
+   ```
    django-admin startproject cas_test .
    ```
 5. Follow the [install directions](https://django-mama-cas.readthedocs.io/en/latest/installation.html#configuring) for django-mama-cas
 6. Setup the SQLite database: `python manage.py migrate`
 7. Create a user:
-   ```sh
+   ```
    python manage.py createsuperuser
    ```
    1. Use whatever you want as the username and password.
    2. Leave the other fields blank.
 8. Use the built-in Django test server to serve the CAS endpoints on port 8000:
-   ```sh
+   ```
    python manage.py runserver
    ```
 

docs/development/contributing_guide.md

@@ -15,14 +15,7 @@ license - in our case, this is almost always Apache Software License v2 (see
 
 # 2. What do I need?
 
-If you are running Windows, the Windows Subsystem for Linux (WSL) is strongly
-recommended for development. More information about WSL can be found at
-<https://docs.microsoft.com/en-us/windows/wsl/install>. Running Synapse natively
-on Windows is not officially supported.
-
-The code of Synapse is written in Python 3. To do pretty much anything, you'll need [a recent version of Python 3](https://www.python.org/downloads/). Your Python also needs support for [virtual environments](https://docs.python.org/3/library/venv.html). This is usually built-in, but some Linux distributions like Debian and Ubuntu split it out into its own package. Running `sudo apt install python3-venv` should be enough.
-
-Synapse can connect to PostgreSQL via the [psycopg2](https://pypi.org/project/psycopg2/) Python library. Building this library from source requires access to PostgreSQL's C header files. On Debian or Ubuntu Linux, these can be installed with `sudo apt install libpq-dev`.
+The code of Synapse is written in Python 3. To do pretty much anything, you'll need [a recent version of Python 3](https://wiki.python.org/moin/BeginnersGuide/Download).
 
 The source code of Synapse is hosted on GitHub. You will also need [a recent version of git](https://github.com/git-guides/install-git).
 
@@ -48,6 +41,8 @@ can find many good git tutorials on the web.
 
 # 4. Install the dependencies
 
+## Under Unix (macOS, Linux, BSD, ...)
+
 Once you have installed Python 3 and added the source, please open a terminal and
 setup a *virtualenv*, as follows:
 
@@ -55,13 +50,16 @@ setup a *virtualenv*, as follows:
 cd path/where/you/have/cloned/the/repository
 python3 -m venv ./env
 source ./env/bin/activate
-pip install wheel
 pip install -e ".[all,dev]"
 pip install tox
 ```
 
 This will install the developer dependencies for the project.
 
+## Under Windows
+
+TBD
+
 
 # 5. Get in touch.
 
@@ -117,7 +115,7 @@ The linters look at your code and do two things:
 - ensure that your code follows the coding style adopted by the project;
 - catch a number of errors in your code.
 
-The linter takes no time at all to run as soon as you've [downloaded the dependencies into your python virtual environment](#4-install-the-dependencies).
+They're pretty fast, don't hesitate!
 
 ```sh
 source ./env/bin/activate
@@ -172,27 +170,6 @@ To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`:
 SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests
 ```
 
-By default, tests will use an in-memory SQLite database for test data. For additional
-help with debugging, one can use an on-disk SQLite database file instead, in order to
-review database state during and after running tests. This can be done by setting
-the `SYNAPSE_TEST_PERSIST_SQLITE_DB` environment variable. Doing so will cause the
-database state to be stored in a file named `test.db` under the trial process'
-working directory. Typically, this ends up being `_trial_temp/test.db`. For example:
-
-```sh
-SYNAPSE_TEST_PERSIST_SQLITE_DB=1 trial tests
-```
-
-The database file can then be inspected with:
-
-```sh
-sqlite3 _trial_temp/test.db
-```
-
-Note that the database file is cleared at the beginning of each test run. Thus it 
-will always only contain the data generated by the *last run test*. Though generally
-when debugging, one is only running a single test anyway.
-
 ### Running tests under PostgreSQL
 
 Invoking `trial` as above will use an in-memory SQLite database. This is great for
@@ -206,10 +183,9 @@ To do so, [configure Postgres](../postgres.md) and run `trial` with the
 following environment variables matching your configuration:
 
 - `SYNAPSE_POSTGRES` to anything nonempty
-- `SYNAPSE_POSTGRES_HOST` (optional if it's the default: UNIX socket)
-- `SYNAPSE_POSTGRES_PORT` (optional if it's the default: 5432)
-- `SYNAPSE_POSTGRES_USER` (optional if using a UNIX socket)
-- `SYNAPSE_POSTGRES_PASSWORD` (optional if using a UNIX socket)
+- `SYNAPSE_POSTGRES_HOST`
+- `SYNAPSE_POSTGRES_USER`
+- `SYNAPSE_POSTGRES_PASSWORD`
 
 For example:
 
@@ -221,12 +197,26 @@ export SYNAPSE_POSTGRES_PASSWORD=mydevenvpassword
 trial
 ```
 
-You don't need to specify the host, user, port or password if your Postgres
-server is set to authenticate you over the UNIX socket (i.e. if the `psql` command
-works without further arguments).
+#### Prebuilt container
 
-Your Postgres account needs to be able to create databases.
+Since configuring PostgreSQL can be fiddly, we can make use of a pre-made
+Docker container to set up PostgreSQL and run our tests for us. To do so, run
 
+```shell
+scripts-dev/test_postgresql.sh
+```
+
+Any extra arguments to the script will be passed to `tox` and then to `trial`,
+so we can run a specific test in this container with e.g.
+
+```shell
+scripts-dev/test_postgresql.sh tests.replication.test_sharded_event_persister.EventPersisterShardTestCase
+```
+
+The container creates a folder in your Synapse checkout called
+`.tox-pg-container` and uses this as a tox environment. The output of any
+`trial` runs goes into `_trial_temp` in your synapse source directory — the same
+as running `trial` directly on your host machine.
 
 ## Run the integration tests ([Sytest](https://github.com/matrix-org/sytest)).
 
@@ -241,14 +231,8 @@ configuration:
 ```sh
 $ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:buster
 ```
-(Note that the paths must be full paths! You could also write `$(realpath relative/path)` if needed.)
 
-This configuration should generally cover your needs.
-
-- To run with Postgres, supply the `-e POSTGRES=1 -e MULTI_POSTGRES=1` environment flags.
-- To run with Synapse in worker mode, supply the `-e WORKERS=1 -e REDIS=1` environment flags (in addition to the Postgres flags).
-
-For more details about other configurations, see the [Docker-specific documentation in the SyTest repo](https://github.com/matrix-org/sytest/blob/develop/docker/README.md).
+This configuration should generally cover  your needs. For more details about other configurations, see [documentation in the SyTest repo](https://github.com/matrix-org/sytest/blob/develop/docker/README.md).
 
 
 ## Run the integration tests ([Complement](https://github.com/matrix-org/complement)).
@@ -451,17 +435,6 @@ Git allows you to add this signoff automatically when using the `-s`
 flag to `git commit`, which uses the name and email set in your
 `user.name` and `user.email` git configs.
 
-### Private Sign off
-
-If you would like to provide your legal name privately to the Matrix.org
-Foundation (instead of in a public commit or comment), you can do so
-by emailing your legal name and a link to the pull request to
-[dco@matrix.org](mailto:dco@matrix.org?subject=Private%20sign%20off).
-It helps to include "sign off" or similar in the subject line. You will then
-be instructed further.
-
-Once private sign off is complete, doing so for future contributions will not
-be required.
 
 # 10. Turn feedback into better code.
 

docs/development/database_schema.md

@@ -89,67 +89,11 @@ To do so, use `scripts-dev/make_full_schema.sh`. This will produce new
 
 Ensure postgres is installed, then run:
 
-```sh
-./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/
-```
+    ./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/
 
 NB at the time of writing, this script predates the split into separate `state`/`main`
 databases so will require updates to handle that correctly.
 
-## Delta files
-
-Delta files define the steps required to upgrade the database from an earlier version.
-They can be written as either a file containing a series of SQL statements, or a Python
-module.
-
-Synapse remembers which delta files it has applied to a database (they are stored in the
-`applied_schema_deltas` table) and will not re-apply them (even if a given file is
-subsequently updated).
-
-Delta files should be placed in a directory named `synapse/storage/schema/<database>/delta/<version>/`.
-They are applied in alphanumeric order, so  by convention the first two characters
-of the filename should be an integer such as `01`, to put the file in the right order.
-
-### SQL delta files
-
-These should be named `*.sql`, or —  for changes which should only be applied for a
-given database engine — `*.sql.posgres` or `*.sql.sqlite`. For example, a delta which
-adds a new column to the `foo` table might be called `01add_bar_to_foo.sql`.
-
-Note that our SQL parser is a bit simple - it understands comments (`--` and `/*...*/`),
-but complex statements which require a `;` in the middle of them (such as `CREATE
-TRIGGER`) are beyond it and you'll have to use a Python delta file.
-
-### Python delta files
-
-For more flexibility, a delta file can take the form of a python module. These should
-be named `*.py`. Note that database-engine-specific modules are not supported here –
-instead you can write `if isinstance(database_engine, PostgresEngine)` or similar.
-
-A Python delta module should define either or both of the following functions:
-
-```python
-import synapse.config.homeserver
-import synapse.storage.engines
-import synapse.storage.types
-
-
-def run_create(
-    cur: synapse.storage.types.Cursor,
-    database_engine: synapse.storage.engines.BaseDatabaseEngine,
-) -> None:
-    """Called whenever an existing or new database is to be upgraded"""
-    ...
-
-def run_upgrade(
-    cur: synapse.storage.types.Cursor,
-    database_engine: synapse.storage.engines.BaseDatabaseEngine,
-    config: synapse.config.homeserver.HomeServerConfig,
-) -> None:
-    """Called whenever an existing database is to be upgraded."""
-    ...
-```
-
 ## Boolean columns
 
 Boolean columns require special treatment, since SQLite treats booleans the
@@ -158,9 +102,9 @@ same as integers.
 There are three separate aspects to this:
 
  * Any new boolean column must be added to the `BOOLEAN_COLUMNS` list in
-   `synapse/_scripts/synapse_port_db.py`. This tells the port script to cast
-   the integer value from SQLite to a boolean before writing the value to the
-   postgres database.
+   `scripts/synapse_port_db`. This tells the port script to cast the integer
+   value from SQLite to a boolean before writing the value to the postgres
+   database.
 
  * Before SQLite 3.23, `TRUE` and `FALSE` were not recognised as constants by
    SQLite, and the `IS [NOT] TRUE`/`IS [NOT] FALSE` operators were not

docs/development/demo.md

@@ -1,41 +0,0 @@
-# Synapse demo setup
-
-**DO NOT USE THESE DEMO SERVERS IN PRODUCTION**
-
-Requires you to have a [Synapse development environment setup](https://matrix-org.github.io/synapse/develop/development/contributing_guide.html#4-install-the-dependencies).
-
-The demo setup allows running three federation Synapse servers, with server
-names `localhost:8080`, `localhost:8081`, and `localhost:8082`.
-
-You can access them via any Matrix client over HTTP at `localhost:8080`,
-`localhost:8081`, and `localhost:8082` or over HTTPS at `localhost:8480`,
-`localhost:8481`, and `localhost:8482`.
-
-To enable the servers to communicate, self-signed SSL certificates are generated
-and the servers are configured in a highly insecure way, including:
-
-* Not checking certificates over federation.
-* Not verifying keys.
-
-The servers are configured to store their data under `demo/8080`, `demo/8081`, and
-`demo/8082`. This includes configuration, logs, SQLite databases, and media.
-
-Note that when joining a public room on a different HS via "#foo:bar.net", then
-you are (in the current impl) joining a room with room_id "foo". This means that
-it won't work if your HS already has a room with that name.
-
-## Using the demo scripts
-
-There's three main scripts with straightforward purposes:
-
-* `start.sh` will start the Synapse servers, generating any missing configuration.
-  * This accepts a single parameter `--no-rate-limit` to "disable" rate limits
-    (they actually still exist, but are very high).
-* `stop.sh` will stop the Synapse servers.
-* `clean.sh` will delete the configuration, databases, log files, etc.
-
-To start a completely new set of servers, run:
-
-```sh
-./demo/stop.sh; ./demo/clean.sh && ./demo/start.sh
-```

docs/development/releases.md

@@ -1,37 +0,0 @@
-# Synapse Release Cycle
-
-Releases of Synapse follow a two week release cycle with new releases usually
-occurring on Tuesdays:
-
-* Day 0: Synapse `N - 1` is released.
-* Day 7: Synapse `N` release candidate 1 is released.
-* Days 7 - 13: Synapse `N` release candidates 2+ are released, if bugs are found.
-* Day 14: Synapse `N` is released.
-
-Note that this schedule might be modified depending on the availability of the
-Synapse team, e.g. releases may be skipped to avoid holidays.
-
-Release announcements can be found in the
-[release category of the Matrix blog](https://matrix.org/blog/category/releases).
-
-## Bugfix releases
-
-If a bug is found after release that is deemed severe enough (by a combination
-of the impacted users and the impact on those users) then a bugfix release may
-be issued. This may be at any point in the release cycle.
-
-## Security releases
-
-Security will sometimes be backported to the previous version and released
-immediately before the next release candidate. An example of this might be:
-
-* Day 0: Synapse N - 1 is released.
-* Day 7: Synapse (N - 1).1 is released as Synapse N - 1 + the security fix.
-* Day 7: Synapse N release candidate 1 is released (including the security fix).
-
-Depending on the impact and complexity of security fixes, multiple fixes might
-be held to be released together.
-
-In some cases, a pre-disclosure of a security release will be issued as a notice
-to Synapse operators that there is an upcoming security release. These can be
-found in the [security category of the Matrix blog](https://matrix.org/blog/category/security).

docs/development/room-dag-concepts.md

@@ -30,73 +30,39 @@ rather than skipping any that arrived late; whereas if you're looking at a
 historical section of timeline (i.e. `/messages`), you want to see the best
 representation of the state of the room as others were seeing it at the time.
 
-## Outliers
-
-We mark an event as an `outlier` when we haven't figured out the state for the
-room at that point in the DAG yet. They are "floating" events that we haven't
-yet correlated to the DAG.
-
-Outliers typically arise when we fetch the auth chain or state for a given
-event. When that happens, we just grab the events in the state/auth chain,
-without calculating the state at those events, or backfilling their
-`prev_events`. Since we don't have the state at any events fetched in that
-way, we mark them as outliers.
-
-So, typically, we won't have the `prev_events` of an `outlier` in the database,
-(though it's entirely possible that we *might* have them for some other
-reason). Other things that make outliers different from regular events:
-
- * We don't have state for them, so there should be no entry in
-   `event_to_state_groups` for an outlier. (In practice this isn't always
-   the case, though I'm not sure why: see https://github.com/matrix-org/synapse/issues/12201).
-
- * We don't record entries for them in the `event_edges`,
-   `event_forward_extremeties` or `event_backward_extremities` tables.
-
-Since outliers are not tied into the DAG, they do not normally form part of the
-timeline sent down to clients via `/sync` or `/messages`; however there is an
-exception:
-
-### Out-of-band membership events
-
-A special case of outlier events are some membership events for federated rooms
-that we aren't full members of. For example:
-
- * invites received over federation, before we join the room
- * *rejections* for said invites
- * knock events for rooms that we would like to join but have not yet joined.
-
-In all the above cases, we don't have the state for the room, which is why they
-are treated as outliers. They are a bit special though, in that they are
-proactively sent to clients via `/sync`.
 
 ## Forward extremity
 
-Most-recent-in-time events in the DAG which are not referenced by any other
-events' `prev_events` yet. (In this definition, outliers, rejected events, and
-soft-failed events don't count.)
+Most-recent-in-time events in the DAG which are not referenced by any other events' `prev_events` yet.
 
-The forward extremities of a room (or at least, a subset of them, if there are
-more than ten) are used as the `prev_events` when the next event is sent.
+The forward extremities of a room are used as the `prev_events` when the next event is sent.
 
-The "current state" of a room (ie: the state which would be used if we
-generated a new event) is, therefore, the resolution of the room states
-at each of the forward extremities.
 
-## Backward extremity
+## Backwards extremity
 
 The current marker of where we have backfilled up to and will generally be the
-`prev_events` of the oldest-in-time events we have in the DAG. This gives a starting point when
-backfilling history.
+oldest-in-time events we know of in the DAG.
 
-Note that, unlike forward extremities, we typically don't have any backward
-extremity events themselves in the database - or, if we do, they will be "outliers" (see
-above). Either way, we don't expect to have the room state at a backward extremity.
+This is an event where we haven't fetched all of the `prev_events` for.
+
+Once we have fetched all of its `prev_events`, it's unmarked as a backwards
+extremity (although we may have formed new backwards extremities from the prev
+events during the backfilling process).
+
+
+## Outliers
+
+We mark an event as an `outlier` when we haven't figured out the state for the
+room at that point in the DAG yet.
+
+We won't *necessarily* have the `prev_events` of an `outlier` in the database,
+but it's entirely possible that we *might*. The status of whether we have all of
+the `prev_events` is marked as a [backwards extremity](#backwards-extremity).
+
+For example, when we fetch the event auth chain or state for a given event, we
+mark all of those claimed auth events as outliers because we haven't done the
+state calculation ourself.
 
-When we persist a non-outlier event, if it was previously a backward extremity,
-we clear it as a backward extremity and set all of its `prev_events` as the new
-backward extremities if they aren't already persisted as non-outliers. This
-therefore keeps the backward extremities up-to-date.
 
 ## State groups
 

docs/development/saml.md

@@ -15,7 +15,7 @@ To make Synapse (and therefore Element) use it:
      sp_config:
        allow_unknown_attributes: true  # Works around a bug with AVA Hashes: https://github.com/IdentityPython/pysaml2/issues/388
        metadata:
-         local: ["samling.xml"]
+         local: ["samling.xml"]   
    ```
 5. Ensure that your `homeserver.yaml` has a setting for `public_baseurl`:
    ```yaml

docs/development/url_previews.md

@@ -35,12 +35,7 @@ When Synapse is asked to preview a URL it does the following:
    5. If the media is HTML:
       1. Decodes the HTML via the stored file.
       2. Generates an Open Graph response from the HTML.
-      3. If a JSON oEmbed URL was found in the HTML via autodiscovery:
-         1. Downloads the URL and stores it into a file via the media storage provider
-            and saves the local media metadata.
-         2. Convert the oEmbed response to an Open Graph response.
-         3. Override any Open Graph data from the HTML with data from oEmbed.
-      4. If an image exists in the Open Graph response:
+      3. If an image exists in the Open Graph response:
          1. Downloads the URL and stores it into a file via the media storage
             provider and saves the local media metadata.
          2. Generates thumbnails.

docs/federate.md

@@ -63,5 +63,4 @@ release of Synapse.
 
 If you want to get up and running quickly with a trio of homeservers in a
 private federation, there is a script in the `demo` directory. This is mainly
-useful just for development purposes. See
-[demo scripts](https://matrix-org.github.io/synapse/develop/development/demo.html).
+useful just for development purposes. See [demo/README](https://github.com/matrix-org/synapse/tree/develop/demo/).

docs/jwt.md

@@ -22,9 +22,8 @@ will be removed in a future version of Synapse.
 
 The `token` field should include the JSON web token with the following claims:
 
-* A claim that encodes the local part of the user ID is required. By default,
-  the `sub` (subject) claim is used, or a custom claim can be set in the
-  configuration file.
+* The `sub` (subject) claim is required and should encode the local part of the
+  user ID.
 * The expiration time (`exp`), not before time (`nbf`), and issued at (`iat`)
   claims are optional, but validated if present.
 * The issuer (`iss`) claim is optional, but required and validated if configured.

docs/manhole.md

@@ -94,6 +94,6 @@ As a simple example, retrieving an event from the database:
 
 ```pycon
 >>> from twisted.internet import defer
->>> defer.ensureDeferred(hs.get_datastores().main.get_event('$1416420717069yeQaw:matrix.org'))
+>>> defer.ensureDeferred(hs.get_datastore().get_event('$1416420717069yeQaw:matrix.org'))
 <Deferred at 0x7ff253fc6998 current result: <FrozenEvent event_id='$1416420717069yeQaw:matrix.org', type='m.room.create', state_key=''>>
 ```

docs/media_repository.md

@@ -2,80 +2,29 @@
 
 *Synapse implementation-specific details for the media repository*
 
-The media repository
- * stores avatars, attachments and their thumbnails for media uploaded by local
-   users.
- * caches avatars, attachments and their thumbnails for media uploaded by remote
-   users.
- * caches resources and thumbnails used for
-   [URL previews](development/url_previews.md).
+The media repository is where attachments and avatar photos are stored.
+It stores attachment content and thumbnails for media uploaded by local users.
+It caches attachment content and thumbnails for media uploaded by remote users.
 
-All media in Matrix can be identified by a unique
-[MXC URI](https://spec.matrix.org/latest/client-server-api/#matrix-content-mxc-uris),
-consisting of a server name and media ID:
-```
-mxc://<server-name>/<media-id>
-```
+## Storage
 
-## Local Media
-Synapse generates 24 character media IDs for content uploaded by local users.
-These media IDs consist of upper and lowercase letters and are case-sensitive.
-Other homeserver implementations may generate media IDs differently.
+Each item of media is assigned a `media_id` when it is uploaded.
+The `media_id` is a randomly chosen, URL safe 24 character string.
 
-Local media is recorded in the `local_media_repository` table, which includes
-metadata such as MIME types, upload times and file sizes.
-Note that this table is shared by the URL cache, which has a different media ID
-scheme.
+Metadata such as the MIME type, upload time and length are stored in the
+sqlite3 database indexed by `media_id`.
 
-### Paths
-A file with media ID `aabbcccccccccccccccccccc` and its `128x96` `image/jpeg`
-thumbnail, created by scaling, would be stored at:
-```
-local_content/aa/bb/cccccccccccccccccccc
-local_thumbnails/aa/bb/cccccccccccccccccccc/128-96-image-jpeg-scale
-```
+Content is stored on the filesystem under a `"local_content"` directory.
 
-## Remote Media
-When media from a remote homeserver is requested from Synapse, it is assigned
-a local `filesystem_id`, with the same format as locally-generated media IDs,
-as described above.
+Thumbnails are stored under a `"local_thumbnails"` directory.
 
-A record of remote media is stored in the `remote_media_cache` table, which
-can be used to map remote MXC URIs (server names and media IDs) to local
-`filesystem_id`s.
+The item with `media_id` `"aabbccccccccdddddddddddd"` is stored under
+`"local_content/aa/bb/ccccccccdddddddddddd"`. Its thumbnail with width
+`128` and height `96` and type `"image/jpeg"` is stored under
+`"local_thumbnails/aa/bb/ccccccccdddddddddddd/128-96-image-jpeg"`
 
-### Paths
-A file from `matrix.org` with `filesystem_id` `aabbcccccccccccccccccccc` and its
-`128x96` `image/jpeg` thumbnail, created by scaling, would be stored at:
-```
-remote_content/matrix.org/aa/bb/cccccccccccccccccccc
-remote_thumbnail/matrix.org/aa/bb/cccccccccccccccccccc/128-96-image-jpeg-scale
-```
-Older thumbnails may omit the thumbnailing method:
-```
-remote_thumbnail/matrix.org/aa/bb/cccccccccccccccccccc/128-96-image-jpeg
-```
-
-Note that `remote_thumbnail/` does not have an `s`.
-
-## URL Previews
-See [URL Previews](development/url_previews.md) for documentation on the URL preview
-process.
-
-When generating previews for URLs, Synapse may download and cache various
-resources, including images. These resources are assigned temporary media IDs
-of the form `yyyy-mm-dd_aaaaaaaaaaaaaaaa`, where `yyyy-mm-dd` is the current
-date and `aaaaaaaaaaaaaaaa` is a random sequence of 16 case-sensitive letters.
-
-The metadata for these cached resources is stored in the
-`local_media_repository` and `local_media_repository_url_cache` tables.
-
-Resources for URL previews are deleted after a few days.
-
-### Paths
-The file with media ID `yyyy-mm-dd_aaaaaaaaaaaaaaaa` and its `128x96`
-`image/jpeg` thumbnail, created by scaling, would be stored at:
-```
-url_cache/yyyy-mm-dd/aaaaaaaaaaaaaaaa
-url_cache_thumbnails/yyyy-mm-dd/aaaaaaaaaaaaaaaa/128-96-image-jpeg-scale
-```
+Remote content is cached under `"remote_content"` directory. Each item of
+remote content is assigned a local `"filesystem_id"` to ensure that the
+directory structure `"remote_content/server_name/aa/bb/ccccccccdddddddddddd"`
+is appropriate. Thumbnails for remote content are stored under
+`"remote_thumbnail/server_name/..."`

docs/message_retention_policies.md

@@ -69,9 +69,9 @@ A default policy can be defined as such, in the `retention` section of
 the configuration file:
 
 ```yaml
-default_policy:
-  min_lifetime: 1d
-  max_lifetime: 1y
+  default_policy:
+    min_lifetime: 1d
+    max_lifetime: 1y
 ```
 
 Here, `min_lifetime` and `max_lifetime` have the same meaning and level
@@ -95,14 +95,14 @@ depending on an event's room's policy. This can be done by setting the
 file. An example of such configuration could be:
 
 ```yaml
-purge_jobs:
-  - longest_max_lifetime: 3d
-    interval: 12h
-  - shortest_max_lifetime: 3d
-    longest_max_lifetime: 1w
-    interval: 1d
-  - shortest_max_lifetime: 1w
-    interval: 2d
+  purge_jobs:
+    - longest_max_lifetime: 3d
+      interval: 12h
+    - shortest_max_lifetime: 3d
+      longest_max_lifetime: 1w
+      interval: 1d
+    - shortest_max_lifetime: 1w
+      interval: 2d
 ```
 
 In this example, we define three jobs:
@@ -141,8 +141,8 @@ purging old events in a room. These limits can be defined as such in the
 `retention` section of the configuration file:
 
 ```yaml
-allowed_lifetime_min: 1d
-allowed_lifetime_max: 1y
+  allowed_lifetime_min: 1d
+  allowed_lifetime_max: 1y
 ```
 
 The limits are considered when running purge jobs. If necessary, the

docs/modules/account_data_callbacks.md

@@ -1,106 +0,0 @@
-# Account data callbacks
-
-Account data callbacks allow module developers to react to changes of the account data
-of local users. Account data callbacks can be registered using the module API's
-`register_account_data_callbacks` method.
-
-## Callbacks
-
-The available account data callbacks are:
-
-### `on_account_data_updated`
-
-_First introduced in Synapse v1.57.0_
-
-```python
-async def on_account_data_updated(
-    user_id: str,
-    room_id: Optional[str],
-    account_data_type: str,
-    content: "synapse.module_api.JsonDict",
-) -> None:
-```
-
-Called after user's account data has been updated. The module is given the
-Matrix ID of the user whose account data is changing, the room ID the data is associated
-with, the type associated with the change, as well as the new content. If the account
-data is not associated with a specific room, then the room ID is `None`.
-
-This callback is triggered when new account data is added or when the data associated with
-a given type (and optionally room) changes. This includes deletion, since in Matrix,
-deleting account data consists of replacing the data associated with a given type
-(and optionally room) with an empty dictionary (`{}`).
-
-Note that this doesn't trigger when changing the tags associated with a room, as these are
-processed separately by Synapse.
-
-If multiple modules implement this callback, Synapse runs them all in order.
-
-## Example
-
-The example below is a module that implements the `on_account_data_updated` callback, and
-sends an event to an audit room when a user changes their account data.
-
-```python
-import json
-import attr
-from typing import Any, Dict, Optional
-
-from synapse.module_api import JsonDict, ModuleApi
-from synapse.module_api.errors import ConfigError
-
-
-@attr.s(auto_attribs=True)
-class CustomAccountDataConfig:
-    audit_room: str
-    sender: str
-
-
-class CustomAccountDataModule:
-    def __init__(self, config: CustomAccountDataConfig, api: ModuleApi):
-        self.api = api
-        self.config = config
-
-        self.api.register_account_data_callbacks(
-            on_account_data_updated=self.log_new_account_data,
-        )
-
-    @staticmethod
-    def parse_config(config: Dict[str, Any]) -> CustomAccountDataConfig:
-        def check_in_config(param: str):
-            if param not in config:
-                raise ConfigError(f"'{param}' is required")
-
-        check_in_config("audit_room")
-        check_in_config("sender")
-
-        return CustomAccountDataConfig(
-            audit_room=config["audit_room"],
-            sender=config["sender"],
-        )
-
-    async def log_new_account_data(
-        self,
-        user_id: str,
-        room_id: Optional[str],
-        account_data_type: str,
-        content: JsonDict,
-    ) -> None:
-        content_raw = json.dumps(content)
-        msg_content = f"{user_id} has changed their account data for type {account_data_type} to: {content_raw}"
-
-        if room_id is not None:
-            msg_content += f" (in room {room_id})"
-
-        await self.api.create_and_send_event_into_room(
-            {
-                "room_id": self.config.audit_room,
-                "sender": self.config.sender,
-                "type": "m.room.message",
-                "content": {
-                    "msgtype": "m.text",
-                    "body": msg_content
-                }
-            }
-        )
-```

docs/modules/background_update_controller_callbacks.md

@@ -1,71 +0,0 @@
-# Background update controller callbacks
-
-Background update controller callbacks allow module developers to control (e.g. rate-limit)
-how database background updates are run. A database background update is an operation
-Synapse runs on its database in the background after it starts. It's usually used to run
-database operations that would take too long if they were run at the same time as schema
-updates (which are run on startup) and delay Synapse's startup too much: populating a
-table with a big amount of data, adding an index on a big table, deleting superfluous data,
-etc.
-
-Background update controller callbacks can be registered using the module API's
-`register_background_update_controller_callbacks` method. Only the first module (in order
-of appearance in Synapse's configuration file) calling this method can register background
-update controller callbacks, subsequent calls are ignored.
-
-The available background update controller callbacks are:
-
-### `on_update`
-
-_First introduced in Synapse v1.49.0_
-
-```python
-def on_update(update_name: str, database_name: str, one_shot: bool) -> AsyncContextManager[int]
-```
-
-Called when about to do an iteration of a background update. The module is given the name
-of the update, the name of the database, and a flag to indicate whether the background
-update will happen in one go and may take a long time (e.g. creating indices). If this last
-argument is set to `False`, the update will be run in batches.
-
-The module must return an async context manager. It will be entered before Synapse runs a 
-background update; this should return the desired duration of the iteration, in
-milliseconds.
-
-The context manager will be exited when the iteration completes. Note that the duration
-returned by the context manager is a target, and an iteration may take substantially longer
-or shorter. If the `one_shot` flag is set to `True`, the duration returned is ignored.
-
-__Note__: Unlike most module callbacks in Synapse, this one is _synchronous_. This is
-because asynchronous operations are expected to be run by the async context manager.
-
-This callback is required when registering any other background update controller callback.
-
-### `default_batch_size`
-
-_First introduced in Synapse v1.49.0_
-
-```python
-async def default_batch_size(update_name: str, database_name: str) -> int
-```
-
-Called before the first iteration of a background update, with the name of the update and
-of the database. The module must return the number of elements to process in this first
-iteration.
-
-If this callback is not defined, Synapse will use a default value of 100.
-
-### `min_batch_size`
-
-_First introduced in Synapse v1.49.0_
-
-```python
-async def min_batch_size(update_name: str, database_name: str) -> int
-```
-
-Called before running a new batch for a background update, with the name of the update and
-of the database. The module must return an integer representing the minimum number of
-elements to process in this iteration. This number must be at least 1, and is used to
-ensure that progress is always made.
-
-If this callback is not defined, Synapse will use a default value of 100.

docs/modules/password_auth_provider_callbacks.md

@@ -10,8 +10,8 @@ registered by using the Module API's `register_password_auth_provider_callbacks`
 
 _First introduced in Synapse v1.46.0_
 
-```python
-auth_checkers: Dict[Tuple[str, Tuple[str, ...]], Callable]
+```
+ auth_checkers: Dict[Tuple[str,Tuple], Callable]
 ```
 
 A dict mapping from tuples of a login type identifier (such as `m.login.password`) and a
@@ -85,7 +85,7 @@ If the authentication is unsuccessful, the module must return `None`.
 If multiple modules implement this callback, they will be considered in order. If a
 callback returns `None`, Synapse falls through to the next one. The value of the first
 callback that does not return `None` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback. If every callback returns `None`,
+any of the subsequent implementations of this callback. If every callback return `None`,
 the authentication is denied.
 
 ### `on_logged_out`
@@ -105,115 +105,6 @@ device ID), and the (now deactivated) access token.
 
 If multiple modules implement this callback, Synapse runs them all in order.
 
-### `get_username_for_registration`
-
-_First introduced in Synapse v1.52.0_
-
-```python
-async def get_username_for_registration(
-    uia_results: Dict[str, Any],
-    params: Dict[str, Any],
-) -> Optional[str]
-```
-
-Called when registering a new user. The module can return a username to set for the user
-being registered by returning it as a string, or `None` if it doesn't wish to force a
-username for this user. If a username is returned, it will be used as the local part of a
-user's full Matrix ID (e.g. it's `alice` in `@alice:example.com`).
-
-This callback is called once [User-Interactive Authentication](https://spec.matrix.org/latest/client-server-api/#user-interactive-authentication-api)
-has been completed by the user. It is not called when registering a user via SSO. It is
-passed two dictionaries, which include the information that the user has provided during
-the registration process.
-
-The first dictionary contains the results of the [User-Interactive Authentication](https://spec.matrix.org/latest/client-server-api/#user-interactive-authentication-api)
-flow followed by the user. Its keys are the identifiers of every step involved in the flow,
-associated with either a boolean value indicating whether the step was correctly completed,
-or additional information (e.g. email address, phone number...). A list of most existing
-identifiers can be found in the [Matrix specification](https://spec.matrix.org/v1.1/client-server-api/#authentication-types).
-Here's an example featuring all currently supported keys:
-
-```python
-{
-    "m.login.dummy": True,  # Dummy authentication
-    "m.login.terms": True,  # User has accepted the terms of service for the homeserver
-    "m.login.recaptcha": True,  # User has completed the recaptcha challenge
-    "m.login.email.identity": {  # User has provided and verified an email address
-        "medium": "email",
-        "address": "alice@example.com",
-        "validated_at": 1642701357084,
-    },
-    "m.login.msisdn": {  # User has provided and verified a phone number
-        "medium": "msisdn",
-        "address": "33123456789",
-        "validated_at": 1642701357084,
-    },
-    "m.login.registration_token": "sometoken",  # User has registered through a registration token
-}
-```
-
-The second dictionary contains the parameters provided by the user's client in the request
-to `/_matrix/client/v3/register`. See the [Matrix specification](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3register)
-for a complete list of these parameters.
-
-If the module cannot, or does not wish to, generate a username for this user, it must
-return `None`.
-
-If multiple modules implement this callback, they will be considered in order. If a
-callback returns `None`, Synapse falls through to the next one. The value of the first
-callback that does not return `None` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback. If every callback returns `None`,
-the username provided by the user is used, if any (otherwise one is automatically
-generated).
-
-### `get_displayname_for_registration`
-
-_First introduced in Synapse v1.54.0_
-
-```python
-async def get_displayname_for_registration(
-    uia_results: Dict[str, Any],
-    params: Dict[str, Any],
-) -> Optional[str]
-```
-
-Called when registering a new user. The module can return a display name to set for the
-user being registered by returning it as a string, or `None` if it doesn't wish to force a
-display name for this user.
-
-This callback is called once [User-Interactive Authentication](https://spec.matrix.org/latest/client-server-api/#user-interactive-authentication-api)
-has been completed by the user. It is not called when registering a user via SSO. It is
-passed two dictionaries, which include the information that the user has provided during
-the registration process. These dictionaries are identical to the ones passed to
-[`get_username_for_registration`](#get_username_for_registration), so refer to the
-documentation of this callback for more information about them.
-
-If multiple modules implement this callback, they will be considered in order. If a
-callback returns `None`, Synapse falls through to the next one. The value of the first
-callback that does not return `None` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback. If every callback returns `None`,
-the username will be used (e.g. `alice` if the user being registered is `@alice:example.com`).
-
-## `is_3pid_allowed`
-
-_First introduced in Synapse v1.53.0_
-
-```python
-async def is_3pid_allowed(self, medium: str, address: str, registration: bool) -> bool
-```
-
-Called when attempting to bind a third-party identifier (i.e. an email address or a phone
-number). The module is given the medium of the third-party identifier (which is `email` if
-the identifier is an email address, or `msisdn` if the identifier is a phone number) and
-its address, as well as a boolean indicating whether the attempt to bind is happening as
-part of registering a new user. The module must return a boolean indicating whether the
-identifier can be allowed to be bound to an account on the local homeserver.
-
-If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
-
 ## Example
 
 The example module below implements authentication checkers for two different login types: 
@@ -222,7 +113,8 @@ The example module below implements authentication checkers for two different lo
     - Is checked by the method: `self.check_my_login`
 - `m.login.password` (defined in [the spec](https://matrix.org/docs/spec/client_server/latest#password-based))
     - Expects a `password` field to be sent to `/login`
-    - Is checked by the method: `self.check_pass`
+    - Is checked by the method: `self.check_pass` 
+
 
 ```python
 from typing import Awaitable, Callable, Optional, Tuple

docs/modules/spam_checker_callbacks.md

@@ -16,12 +16,10 @@ _First introduced in Synapse v1.37.0_
 async def check_event_for_spam(event: "synapse.events.EventBase") -> Union[bool, str]
 ```
 
-Called when receiving an event from a client or via federation. The callback must return
-either:
-- an error message string, to indicate the event must be rejected because of spam and 
-  give a rejection reason to forward to clients;
-- the boolean `True`, to indicate that the event is spammy, but not provide further details; or
-- the booelan `False`, to indicate that the event is not considered spammy.
+Called when receiving an event from a client or via federation. The module can return
+either a `bool` to indicate whether the event must be rejected because of spam, or a `str`
+to indicate the event must be rejected because of spam and to give a rejection reason to
+forward to clients.
 
 If multiple modules implement this callback, they will be considered in order. If a
 callback returns `False`, Synapse falls through to the next one. The value of the first
@@ -37,10 +35,7 @@ async def user_may_join_room(user: str, room: str, is_invited: bool) -> bool
 ```
 
 Called when a user is trying to join a room. The module must return a `bool` to indicate
-whether the user can join the room. Return `False` to prevent the user from joining the
-room; otherwise return `True` to permit the joining.
-
-The user is represented by their Matrix user ID (e.g.
+whether the user can join the room. The user is represented by their Matrix user ID (e.g.
 `@alice:example.com`) and the room is represented by its Matrix ID (e.g.
 `!room:example.com`). The module is also given a boolean to indicate whether the user
 currently has a pending invite in the room.
@@ -63,8 +58,7 @@ async def user_may_invite(inviter: str, invitee: str, room_id: str) -> bool
 
 Called when processing an invitation. The module must return a `bool` indicating whether
 the inviter can invite the invitee to the given room. Both inviter and invitee are
-represented by their Matrix user ID (e.g. `@alice:example.com`). Return `False` to prevent
-the invitation; otherwise return `True` to permit it.
+represented by their Matrix user ID (e.g. `@alice:example.com`).
 
 If multiple modules implement this callback, they will be considered in order. If a
 callback returns `True`, Synapse falls through to the next one. The value of the first
@@ -86,8 +80,7 @@ async def user_may_send_3pid_invite(
 
 Called when processing an invitation using a third-party identifier (also called a 3PID,
 e.g. an email address or a phone number). The module must return a `bool` indicating
-whether the inviter can invite the invitee to the given room. Return `False` to prevent
-the invitation; otherwise return `True` to permit it.
+whether the inviter can invite the invitee to the given room.
 
 The inviter is represented by their Matrix user ID (e.g. `@alice:example.com`), and the
 invitee is represented by its medium (e.g. "email") and its address
@@ -124,7 +117,42 @@ async def user_may_create_room(user: str) -> bool
 
 Called when processing a room creation request. The module must return a `bool` indicating
 whether the given user (represented by their Matrix user ID) is allowed to create a room.
-Return `False` to prevent room creation; otherwise return `True` to permit it.
+
+If multiple modules implement this callback, they will be considered in order. If a
+callback returns `True`, Synapse falls through to the next one. The value of the first
+callback that does not return `True` will be used. If this happens, Synapse will not call
+any of the subsequent implementations of this callback.
+
+### `user_may_create_room_with_invites`
+
+_First introduced in Synapse v1.44.0_
+
+```python
+async def user_may_create_room_with_invites(
+    user: str,
+    invites: List[str],
+    threepid_invites: List[Dict[str, str]],
+) -> bool
+```
+
+Called when processing a room creation request (right after `user_may_create_room`).
+The module is given the Matrix user ID of the user trying to create a room, as well as a
+list of Matrix users to invite and a list of third-party identifiers (3PID, e.g. email
+addresses) to invite.
+
+An invited Matrix user to invite is represented by their Matrix user IDs, and an invited
+3PIDs is represented by a dict that includes the 3PID medium (e.g. "email") through its
+`medium` key and its address (e.g. "alice@example.com") through its `address` key.
+
+See [the Matrix specification](https://matrix.org/docs/spec/appendices#pid-types) for more
+information regarding third-party identifiers.
+
+If no invite and/or 3PID invite were specified in the room creation request, the
+corresponding list(s) will be empty.
+
+**Note**: This callback is not called when a room is cloned (e.g. during a room upgrade)
+since no invites are sent when cloning a room. To cover this case, modules also need to
+implement `user_may_create_room`.
 
 If multiple modules implement this callback, they will be considered in order. If a
 callback returns `True`, Synapse falls through to the next one. The value of the first
@@ -141,8 +169,7 @@ async def user_may_create_room_alias(user: str, room_alias: "synapse.types.RoomA
 
 Called when trying to associate an alias with an existing room. The module must return a
 `bool` indicating whether the given user (represented by their Matrix user ID) is allowed
-to set the given alias. Return `False` to prevent the alias creation; otherwise return 
-`True` to permit it.
+to set the given alias.
 
 If multiple modules implement this callback, they will be considered in order. If a
 callback returns `True`, Synapse falls through to the next one. The value of the first
@@ -159,8 +186,7 @@ async def user_may_publish_room(user: str, room_id: str) -> bool
 
 Called when trying to publish a room to the homeserver's public rooms directory. The
 module must return a `bool` indicating whether the given user (represented by their
-Matrix user ID) is allowed to publish the given room. Return `False` to prevent the
-room from being published; otherwise return `True` to permit its publication.
+Matrix user ID) is allowed to publish the given room.
 
 If multiple modules implement this callback, they will be considered in order. If a
 callback returns `True`, Synapse falls through to the next one. The value of the first
@@ -172,21 +198,16 @@ any of the subsequent implementations of this callback.
 _First introduced in Synapse v1.37.0_
 
 ```python
-async def check_username_for_spam(user_profile: synapse.module_api.UserProfile) -> bool
+async def check_username_for_spam(user_profile: Dict[str, str]) -> bool
 ```
 
 Called when computing search results in the user directory. The module must return a
-`bool` indicating whether the given user should be excluded from user directory 
-searches. Return `True` to indicate that the user is spammy and exclude them from 
-search results; otherwise return `False`.
+`bool` indicating whether the given user profile can appear in search results. The profile
+is represented as a dictionary with the following keys:
 
-The profile is represented as a dictionary with the following keys:
-
-* `user_id: str`. The Matrix ID for this user.
-* `display_name: Optional[str]`. The user's display name, or `None` if this user
-  has not set a display name.
-* `avatar_url: Optional[str]`. The `mxc://` URL to the user's avatar, or `None`
-  if this user has not set an avatar.
+* `user_id`: The Matrix ID for this user.
+* `display_name`: The user's display name.
+* `avatar_url`: The `mxc://` URL to the user's avatar.
 
 The module is given a copy of the original dictionary, so modifying it from within the
 module cannot modify a user's profile when included in user directory search results.
@@ -240,9 +261,8 @@ async def check_media_file_for_spam(
 ) -> bool
 ```
 
-Called when storing a local or remote file. The module must return a `bool` indicating
-whether the given file should be excluded from the homeserver's media store. Return
-`True` to prevent this file from being stored; otherwise return `False`.
+Called when storing a local or remote file. The module must return a boolean indicating
+whether the given file can be stored in the homeserver's media store.
 
 If multiple modules implement this callback, they will be considered in order. If a
 callback returns `False`, Synapse falls through to the next one. The value of the first

docs/modules/third_party_rules_callbacks.md

@@ -43,14 +43,6 @@ event with new data by returning the new event's data as a dictionary. In order
 that, it is recommended the module calls `event.get_dict()` to get the current event as a
 dictionary, and modify the returned dictionary accordingly.
 
-If `check_event_allowed` raises an exception, the module is assumed to have failed.
-The event will not be accepted but is not treated as explicitly rejected, either.
-An HTTP request causing the module check will likely result in a 500 Internal
-Server Error.
-
-When the boolean returned by the module is `False`, the event is rejected.
-(Module developers should not use exceptions for rejection.)
-
 Note that replacing the event only works for events sent by local users, not for events
 received over federation.
 
@@ -127,144 +119,6 @@ callback returns `True`, Synapse falls through to the next one. The value of the
 callback that does not return `True` will be used. If this happens, Synapse will not call
 any of the subsequent implementations of this callback.
 
-### `on_new_event`
-
-_First introduced in Synapse v1.47.0_
-
-```python
-async def on_new_event(
-    event: "synapse.events.EventBase",
-    state_events: "synapse.types.StateMap",
-) -> None:
-```
-
-Called after sending an event into a room. The module is passed the event, as well
-as the state of the room _after_ the event. This means that if the event is a state event,
-it will be included in this state.
-
-Note that this callback is called when the event has already been processed and stored
-into the room, which means this callback cannot be used to deny persisting the event. To
-deny an incoming event, see [`check_event_for_spam`](spam_checker_callbacks.md#check_event_for_spam) instead.
-
-If multiple modules implement this callback, Synapse runs them all in order.
-
-### `check_can_shutdown_room`
-
-_First introduced in Synapse v1.55.0_
-
-```python
-async def check_can_shutdown_room(
-    user_id: str, room_id: str,
-) -> bool:
-```
-
-Called when an admin user requests the shutdown of a room. The module must return a
-boolean indicating whether the shutdown can go through. If the callback returns `False`,
-the shutdown will not proceed and the caller will see a `M_FORBIDDEN` error.
-
-If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
-
-### `check_can_deactivate_user`
-
-_First introduced in Synapse v1.55.0_
-
-```python
-async def check_can_deactivate_user(
-    user_id: str, by_admin: bool,
-) -> bool:
-```
-
-Called when the deactivation of a user is requested. User deactivation can be
-performed by an admin or the user themselves, so developers are encouraged to check the
-requester when implementing this callback. The module must return a
-boolean indicating whether the deactivation can go through. If the callback returns `False`,
-the deactivation will not proceed and the caller will see a `M_FORBIDDEN` error.
-
-The module is passed two parameters, `user_id` which is the ID of the user being deactivated, and `by_admin` which is `True` if the request is made by a serve admin, and `False` otherwise.
-
-If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
-
-
-### `on_profile_update`
-
-_First introduced in Synapse v1.54.0_
-
-```python
-async def on_profile_update(
-    user_id: str,
-    new_profile: "synapse.module_api.ProfileInfo",
-    by_admin: bool,
-    deactivation: bool,
-) -> None:
-```
-
-Called after updating a local user's profile. The update can be triggered either by the
-user themselves or a server admin. The update can also be triggered by a user being
-deactivated (in which case their display name is set to an empty string (`""`) and the
-avatar URL is set to `None`). The module is passed the Matrix ID of the user whose profile
-has been updated, their new profile, as well as a `by_admin` boolean that is `True` if the
-update was triggered by a server admin (and `False` otherwise), and a `deactivated`
-boolean that is `True` if the update is a result of the user being deactivated.
-
-Note that the `by_admin` boolean is also `True` if the profile change happens as a result
-of the user logging in through Single Sign-On, or if a server admin updates their own
-profile.
-
-Per-room profile changes do not trigger this callback to be called. Synapse administrators
-wishing this callback to be called on every profile change are encouraged to disable
-per-room profiles globally using the `allow_per_room_profiles` configuration setting in
-Synapse's configuration file.
-This callback is not called when registering a user, even when setting it through the
-[`get_displayname_for_registration`](https://matrix-org.github.io/synapse/latest/modules/password_auth_provider_callbacks.html#get_displayname_for_registration)
-module callback.
-
-If multiple modules implement this callback, Synapse runs them all in order.
-
-### `on_user_deactivation_status_changed`
-
-_First introduced in Synapse v1.54.0_
-
-```python
-async def on_user_deactivation_status_changed(
-    user_id: str, deactivated: bool, by_admin: bool
-) -> None:
-```
-
-Called after deactivating a local user, or reactivating them through the admin API. The
-deactivation can be triggered either by the user themselves or a server admin. The module
-is passed the Matrix ID of the user whose status is changed, as well as a `deactivated`
-boolean that is `True` if the user is being deactivated and `False` if they're being
-reactivated, and a `by_admin` boolean that is `True` if the deactivation was triggered by
-a server admin (and `False` otherwise). This latter `by_admin` boolean is always `True`
-if the user is being reactivated, as this operation can only be performed through the
-admin API.
-
-If multiple modules implement this callback, Synapse runs them all in order.
-
-### `on_threepid_bind`
-
-_First introduced in Synapse v1.56.0_
-
-```python
-async def on_threepid_bind(user_id: str, medium: str, address: str) -> None:
-```
-
-Called after creating an association between a local user and a third-party identifier
-(email address, phone number). The module is given the Matrix ID of the user the
-association is for, as well as the medium (`email` or `msisdn`) and address of the
-third-party identifier.
-
-Note that this callback is _not_ called after a successful association on an _identity
-server_.
-
-If multiple modules implement this callback, Synapse runs them all in order.
-
 ## Example
 
 The example below is a module that implements the third-party rules callback

docs/modules/writing_a_module.md

@@ -33,7 +33,7 @@ A module can implement the following static method:
 
 ```python
 @staticmethod
-def parse_config(config: dict) -> Any
+def parse_config(config: dict) -> dict
 ```
 
 This method is given a dictionary resulting from parsing the YAML configuration for the
@@ -71,15 +71,15 @@ Modules **must** register their web resources in their `__init__` method.
 ## Registering a callback
 
 Modules can use Synapse's module API to register callbacks. Callbacks are functions that
-Synapse will call when performing specific actions. Callbacks must be asynchronous (unless
-specified otherwise), and are split in categories. A single module may implement callbacks
-from multiple categories, and is under no obligation to implement all callbacks from the
-categories it registers callbacks for.
+Synapse will call when performing specific actions. Callbacks must be asynchronous, and
+are split in categories. A single module may implement callbacks from multiple categories,
+and is under no obligation to implement all callbacks from the categories it registers
+callbacks for.
 
 Modules can register callbacks using one of the module API's `register_[...]_callbacks`
 methods. The callback functions are passed to these methods as keyword arguments, with
-the callback name as the argument name and the function as its value. A
-`register_[...]_callbacks` method exists for each category.
+the callback name as the argument name and the function as its value. This is demonstrated
+in the example below. A `register_[...]_callbacks` method exists for each category.
 
 Callbacks for each category can be found on their respective page of the
 [Synapse documentation website](https://matrix-org.github.io/synapse).

docs/openid.md

@@ -21,8 +21,6 @@ such as [Github][github-idp].
 
 [google-idp]: https://developers.google.com/identity/protocols/oauth2/openid-connect
 [auth0]: https://auth0.com/
-[authentik]: https://goauthentik.io/
-[lemonldap]: https://lemonldap-ng.org/
 [okta]: https://www.okta.com/
 [dex-idp]: https://github.com/dexidp/dex
 [keycloak-idp]: https://www.keycloak.org/docs/latest/server_admin/#sso-protocols
@@ -83,7 +81,7 @@ oidc_providers:
 
 ### Dex
 
-[Dex][dex-idp] is a simple, open-source OpenID Connect Provider.
+[Dex][dex-idp] is a simple, open-source, certified OpenID Connect Provider.
 Although it is designed to help building a full-blown provider with an
 external database, it can be configured with static passwords in a config file.
 
@@ -211,78 +209,6 @@ oidc_providers:
         display_name_template: "{{ user.name }}"
 ```
 
-### Authentik
-
-[Authentik][authentik] is an open-source IdP solution.
-
-1. Create a provider in Authentik, with type OAuth2/OpenID.
-2. The parameters are:
-- Client Type: Confidential
-- JWT Algorithm: RS256
-- Scopes: OpenID, Email and Profile
-- RSA Key: Select any available key
-- Redirect URIs: `[synapse public baseurl]/_synapse/client/oidc/callback`
-3. Create an application for synapse in Authentik and link it to the provider.
-4. Note the slug of your application, Client ID and Client Secret.
-
-Note: RSA keys must be used for signing for Authentik, ECC keys do not work.
-
-Synapse config:
-```yaml
-oidc_providers:
-  - idp_id: authentik
-    idp_name: authentik
-    discover: true
-    issuer: "https://your.authentik.example.org/application/o/your-app-slug/" # TO BE FILLED: domain and slug
-    client_id: "your client id" # TO BE FILLED
-    client_secret: "your client secret" # TO BE FILLED
-    scopes:
-      - "openid"
-      - "profile"
-      - "email"
-    user_mapping_provider:
-      config:
-        localpart_template: "{{ user.preferred_username }}"
-        display_name_template: "{{ user.preferred_username|capitalize }}" # TO BE FILLED: If your users have names in Authentik and you want those in Synapse, this should be replaced with user.name|capitalize.
-```
-
-### LemonLDAP
-
-[LemonLDAP::NG][lemonldap] is an open-source IdP solution.
-
-1. Create an OpenID Connect Relying Parties in LemonLDAP::NG
-2. The parameters are:
-- Client ID under the basic menu of the new Relying Parties (`Options > Basic >
-  Client ID`)
-- Client secret (`Options > Basic > Client secret`)
-- JWT Algorithm: RS256 within the security menu of the new Relying Parties
-  (`Options > Security > ID Token signature algorithm` and `Options > Security >
-  Access Token signature algorithm`)
-- Scopes: OpenID, Email and Profile
-- Allowed redirection addresses for login (`Options > Basic > Allowed
-  redirection addresses for login` ) :
-  `[synapse public baseurl]/_synapse/client/oidc/callback`
-
-Synapse config:
-```yaml
-oidc_providers:
-  - idp_id: lemonldap
-    idp_name: lemonldap
-    discover: true
-    issuer: "https://auth.example.org/" # TO BE FILLED: replace with your domain
-    client_id: "your client id" # TO BE FILLED
-    client_secret: "your client secret" # TO BE FILLED
-    scopes:
-      - "openid"
-      - "profile"
-      - "email"
-    user_mapping_provider:
-      config:
-        localpart_template: "{{ user.preferred_username }}}"
-        # TO BE FILLED: If your users have names in LemonLDAP::NG and you want those in Synapse, this should be replaced with user.name|capitalize or any valid filter.
-        display_name_template: "{{ user.preferred_username|capitalize }}"
-```
-
 ### GitHub
 
 [GitHub][github-idp] is a bit special as it is not an OpenID Connect compliant provider, but
@@ -392,6 +318,9 @@ oidc_providers:
 
 ### Facebook
 
+Like Github, Facebook provide a custom OAuth2 API rather than an OIDC-compliant
+one so requires a little more configuration.
+
 0. You will need a Facebook developer account. You can register for one
    [here](https://developers.facebook.com/async/registration/).
 1. On the [apps](https://developers.facebook.com/apps/) page of the developer
@@ -411,28 +340,24 @@ Synapse config:
     idp_name: Facebook
     idp_brand: "facebook"  # optional: styling hint for clients
     discover: false
-    issuer: "https://www.facebook.com"
+    issuer: "https://facebook.com"
     client_id: "your-client-id" # TO BE FILLED
     client_secret: "your-client-secret" # TO BE FILLED
     scopes: ["openid", "email"]
-    authorization_endpoint: "https://facebook.com/dialog/oauth"
-    token_endpoint: "https://graph.facebook.com/v9.0/oauth/access_token"
-    jwks_uri: "https://www.facebook.com/.well-known/oauth/openid/jwks/"
+    authorization_endpoint: https://facebook.com/dialog/oauth
+    token_endpoint: https://graph.facebook.com/v9.0/oauth/access_token
+    user_profile_method: "userinfo_endpoint"
+    userinfo_endpoint: "https://graph.facebook.com/v9.0/me?fields=id,name,email,picture"
     user_mapping_provider:
       config:
+        subject_claim: "id"
         display_name_template: "{{ user.name }}"
-        email_template: "{{ '{{ user.email }}' }}"
 ```
 
 Relevant documents:
- * [Manually Build a Login Flow](https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow)
- * [Using Facebook's Graph API](https://developers.facebook.com/docs/graph-api/using-graph-api/)
- * [Reference to the User endpoint](https://developers.facebook.com/docs/graph-api/reference/user)
-
-Facebook do have an [OIDC discovery endpoint](https://www.facebook.com/.well-known/openid-configuration),
-but it has a `response_types_supported` which excludes "code" (which we rely on, and
-is even mentioned in their [documentation](https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow#login)),
-so we have to disable discovery and configure the URIs manually.
+ * https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow
+ * Using Facebook's Graph API: https://developers.facebook.com/docs/graph-api/using-graph-api/
+ * Reference to the User endpoint: https://developers.facebook.com/docs/graph-api/reference/user
 
 ### Gitea
 
@@ -526,7 +451,7 @@ The synapse config will look like this:
         email_template: "{{ user.email }}"
 ```
 
-### Django OAuth Toolkit
+## Django OAuth Toolkit
 
 [django-oauth-toolkit](https://github.com/jazzband/django-oauth-toolkit) is a
 Django application providing out of the box all the endpoints, data and logic

docs/other/running_synapse_on_single_board_computers.md

@@ -1,75 +0,0 @@
-## Summary of performance impact of running on resource constrained devices such as SBCs
-
-I've been running my homeserver on a cubietruck at home now for some time and am often replying to statements like "you need loads of ram to join large rooms" with "it works fine for me". I thought it might be useful to curate a summary of the issues you're likely to run into to help as a scaling-down guide, maybe highlight these for development work or end up as documentation. It seems that once you get up to about 4x1.5GHz arm64 4GiB these issues are no longer a problem.
-
-- **Platform**: 2x1GHz armhf 2GiB ram [Single-board computers](https://wiki.debian.org/CheapServerBoxHardware), SSD, postgres.
-
-### Presence
-
-This is the main reason people have a poor matrix experience on resource constrained homeservers. Element web will frequently be saying the server is offline while the python process will be pegged at 100% cpu. This feature is used to tell when other users are active (have a client app in the foreground) and therefore more likely to respond, but requires a lot of network activity to maintain even when nobody is talking in a room.
-
-![Screenshot_2020-10-01_19-29-46](https://user-images.githubusercontent.com/71895/94848963-a47a3580-041c-11eb-8b6e-acb772b4259e.png)
-
-While synapse does have some performance issues with presence [#3971](https://github.com/matrix-org/synapse/issues/3971), the fundamental problem is that this is an easy feature to implement for a centralised service at nearly no overhead, but federation makes it combinatorial [#8055](https://github.com/matrix-org/synapse/issues/8055). There is also a client-side config option which disables the UI and idle tracking [enable_presence_by_hs_url] to blacklist the largest instances but I didn't notice much difference, so I recommend disabling the feature entirely at the server level as well.
-
-[enable_presence_by_hs_url]: https://github.com/vector-im/element-web/blob/v1.7.8/config.sample.json#L45
-
-### Joining
-
-Joining a "large", federated room will initially fail with the below message in Element web, but waiting a while (10-60mins) and trying again will succeed without any issue. What counts as "large" is not message history, user count, connections to homeservers or even a simple count of the state events, it is instead how long the state resolution algorithm takes. However, each of those numbers are reasonable proxies, so we can use them as estimates since user count is one of the few things you see before joining.
-
-![Screenshot_2020-10-02_17-15-06](https://user-images.githubusercontent.com/71895/94945781-18771500-04d3-11eb-8419-83c2da73a341.png)
-
-This is [#1211](https://github.com/matrix-org/synapse/issues/1211) and will also hopefully be mitigated by peeking [matrix-org/matrix-doc#2753](https://github.com/matrix-org/matrix-doc/pull/2753) so at least you don't need to wait for a join to complete before finding out if it's the kind of room you want. Note that you should first disable presence, otherwise it'll just make the situation worse [#3120](https://github.com/matrix-org/synapse/issues/3120). There is a lot of database interaction too, so make sure you've [migrated your data](../postgres.md) from the default sqlite to postgresql. Personally, I recommend patience - once the initial join is complete there's rarely any issues with actually interacting with the room, but if you like you can just block "large" rooms entirely.
-
-### Sessions
-
-Anything that requires modifying the device list [#7721](https://github.com/matrix-org/synapse/issues/7721) will take a while to propagate, again taking the client "Offline" until it's complete. This includes signing in and out, editing the public name and verifying e2ee. The main mitigation I recommend is to keep long-running sessions open e.g. by using Firefox SSB "Use this site in App mode" or Chromium PWA "Install Element".
-
-### Recommended configuration
-
-Put the below in a new file at /etc/matrix-synapse/conf.d/sbc.yaml to override the defaults in homeserver.yaml.
-
-```
-# Disable presence tracking, which is currently fairly resource intensive
-# More info: https://github.com/matrix-org/synapse/issues/9478
-use_presence: false
-
-# Set a small complexity limit, preventing users from joining large rooms
-# which may be resource-intensive to remain a part of.
-#
-# Note that this will not prevent users from joining smaller rooms that
-# eventually become complex.
-limit_remote_rooms:
-  enabled: true
-  complexity: 3.0
-
-# Database configuration
-database:
-  # Use postgres for the best performance
-  name: psycopg2
-  args:
-    user: matrix-synapse
-    # Generate a long, secure password using a password manager
-    password: hunter2
-    database: matrix-synapse
-    host: localhost
-```
-
-Currently the complexity is measured by [current_state_events / 500](https://github.com/matrix-org/synapse/blob/v1.20.1/synapse/storage/databases/main/events_worker.py#L986). You can find join times and your most complex rooms like this:
-
-```
-admin@homeserver:~$ zgrep '/client/r0/join/' /var/log/matrix-synapse/homeserver.log* | awk '{print $18, $25}' | sort --human-numeric-sort
-29.922sec/-0.002sec /_matrix/client/r0/join/%23debian-fasttrack%3Apoddery.com
-182.088sec/0.003sec /_matrix/client/r0/join/%23decentralizedweb-general%3Amatrix.org
-911.625sec/-570.847sec /_matrix/client/r0/join/%23synapse%3Amatrix.org
-
-admin@homeserver:~$ sudo --user postgres psql matrix-synapse --command 'select canonical_alias, joined_members, current_state_events from room_stats_state natural join room_stats_current where canonical_alias is not null order by current_state_events desc fetch first 5 rows only'
-        canonical_alias        | joined_members | current_state_events 
--------------------------------+----------------+----------------------
- #_oftc_#debian:matrix.org             |  871   |  52355
- #matrix:matrix.org                    |  6379  |  10684
- #irc:matrix.org                       |  461   |  3751
- #decentralizedweb-general:matrix.org  |  997   |  1509
- #whatsapp:maunium.net                 |  554   |  854
-```