Add push debug logging

.buildkite/scripts/create_postgres_db.py

@@ -15,7 +15,6 @@
 # limitations under the License.
 
 import logging
-
 from synapse.storage.engines import create_engine
 
 logger = logging.getLogger("create_postgres_db")

.buildkite/scripts/test_old_deps.sh

@@ -6,11 +6,8 @@
 set -ex
 
 apt-get update
-apt-get install -y python3.5 python3.5-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox
+apt-get install -y python3.5 python3.5-dev python3-pip libxml2-dev libxslt-dev zlib1g-dev tox
 
 export LANG="C.UTF-8"
 
-# Prevent virtualenv from auto-updating pip to an incompatible version
-export VIRTUALENV_NO_DOWNLOAD=1
-
 exec tox -e py35-old,combine

.buildkite/worker-blacklist

@@ -1,10 +1,43 @@
 # This file serves as a blacklist for SyTest tests that we expect will fail in
 # Synapse when run under worker mode. For more details, see sytest-blacklist.
 
+Message history can be paginated
+
 Can re-join room if re-invited
 
+/upgrade creates a new room
+
+The only membership state included in an initial sync is for all the senders in the timeline
+
+Local device key changes get to remote servers
+
+If remote user leaves room we no longer receive device updates
+
+Forgotten room messages cannot be paginated
+
+Inbound federation can get public room list
+
+Members from the gap are included in gappy incr LL sync
+
+Leaves are present in non-gapped incremental syncs
+
+Old leaves are present in gapped incremental syncs
+
+User sees updates to presence from other users in the incremental sync.
+
+Gapped incremental syncs include all state changes
+
+Old members are included in gappy incr LL sync if they start speaking
+
 # new failures as of https://github.com/matrix-org/sytest/pull/732
 Device list doesn't change if remote server is down
+Remote servers cannot set power levels in rooms without existing powerlevels
+Remote servers should reject attempts by non-creators to set the power levels
 
 # https://buildkite.com/matrix-dot-org/synapse/builds/6134#6f67bf47-e234-474d-80e8-c6e1868b15c5
 Server correctly handles incoming m.device_list_update
+
+# this fails reliably with a torture level of 100 due to https://github.com/matrix-org/synapse/issues/6536
+Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state
+
+Can get rooms/{roomId}/members at a given point

.circleci/config.yml

@@ -1,35 +1,24 @@
-version: 2.1
+version: 2
 jobs:
   dockerhubuploadrelease:
-    docker:
+    machine: true
-      - image: docker:git
     steps:
       - checkout
-      - docker_prepare
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      # for release builds, we want to get the amd64 image out asap, so first
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
-      # we do an amd64-only build, before following up with a multiarch build.
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
-      - docker_build:
-          tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
-          platforms: linux/amd64
-      - docker_build:
-          tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
-          platforms: linux/amd64,linux/arm64
-
   dockerhubuploadlatest:
-    docker:
+    machine: true
-      - image: docker:git
     steps:
       - checkout
-      - docker_prepare
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest -t matrixdotorg/synapse:latest-py3 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      # for `latest`, we don't want the arm images to disappear, so don't update the tag
+      - run: docker push matrixdotorg/synapse:latest
-      # until all of the platforms are built.
+      - run: docker push matrixdotorg/synapse:latest-py3
-      - docker_build:
-          tag: -t matrixdotorg/synapse:latest
-          platforms: linux/amd64,linux/arm64
 
 workflows:
+  version: 2
   build:
     jobs:
       - dockerhubuploadrelease:
@@ -42,37 +31,3 @@ workflows:
           filters:
             branches:
               only: master
-
-commands:
-  docker_prepare:
-    description: Sets up a remote docker server, downloads the buildx cli plugin, and enables multiarch images
-    parameters:
-      buildx_version:
-        type: string
-        default: "v0.4.1"
-    steps:
-      - setup_remote_docker:
-          # 19.03.13 was the most recent available on circleci at the time of
-          # writing.
-          version: 19.03.13
-      - run: apk add --no-cache curl
-      - run: mkdir -vp ~/.docker/cli-plugins/ ~/dockercache
-      - run: curl --silent -L "https://github.com/docker/buildx/releases/download/<< parameters.buildx_version >>/buildx-<< parameters.buildx_version >>.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
-      - run: chmod a+x ~/.docker/cli-plugins/docker-buildx
-      # install qemu links in /proc/sys/fs/binfmt_misc on the docker instance running the circleci job
-      - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-      # create a context named `builder` for the builds
-      - run: docker context create builder
-      # create a buildx builder using the new context, and set it as the default
-      - run: docker buildx create builder --use
-
-  docker_build:
-    description: Builds and pushed images to dockerhub using buildx
-    parameters:
-      platforms:
-        type: string
-        default: linux/amd64
-      tag:
-        type: string
-    steps:
-      - run: docker buildx build -f docker/Dockerfile --push --platform << parameters.platforms >> --label gitsha1=${CIRCLE_SHA1} << parameters.tag >> --progress=plain .

.git-blame-ignore-revs

@@ -1,8 +0,0 @@
-# Black reformatting (#5482).
-32e7c9e7f20b57dd081023ac42d6931a8da9b3a3
-
-# Target Python 3.5 with black (#8664).
-aff1eb7c671b0a3813407321d2702ec46c71fa56
-
-# Update black to 20.8b1 (#9381).
-0a00b7ff14890987f09112a2ae696c61001e6cf1

.github/ISSUE_TEMPLATE.md

@@ -1,5 +0,0 @@
-**If you are looking for support** please ask in **#synapse:matrix.org**
-(using a matrix.org account if necessary). We do not use GitHub issues for
-support.
-
-**If you want to report a security issue** please see https://matrix.org/security-disclosure-policy/

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -6,11 +6,9 @@ about: Create a report to help us improve
 
 <!--
 
-**THIS IS NOT A SUPPORT CHANNEL!**
+**IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**:
-**IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**,
+You will likely get better support more quickly if you ask in ** #synapse:matrix.org ** ;)
-please ask in **#synapse:matrix.org** (using a matrix.org account if necessary)
 
-If you want to report a security issue, please see https://matrix.org/security-disclosure-policy/
 
 This is a bug report template. By following the instructions below and
 filling out the sections with your information, you will help the us to get all

.gitignore

@@ -6,25 +6,21 @@
 *.egg
 *.egg-info
 *.lock
-*.py[cod]
+*.pyc
 *.snap
 *.tac
 _trial_temp/
 _trial_temp*/
 /out
-.DS_Store
-__pycache__/
 
 # stuff that is likely to exist when you run a server locally
 /*.db
 /*.log
-/*.log.*
 /*.log.config
 /*.pid
 /.python-version
 /*.signing.key
 /env/
-/.venv*/
 /homeserver*.yaml
 /logs
 /media_store/

CONTRIBUTING.md

@@ -1,213 +1,86 @@
-Welcome to Synapse
+# Contributing code to Matrix
 
-This document aims to get you started with contributing to this repo! 
+Everyone is welcome to contribute code to Matrix
+(https://github.com/matrix-org), provided that they are willing to license
+their contributions under the same license as the project itself. We follow a
+simple 'inbound=outbound' model for contributions: the act of submitting an
+'inbound' contribution means that the contributor agrees to license the code
+under the same terms as the project's overall 'outbound' license - in our
+case, this is almost always Apache Software License v2 (see [LICENSE](LICENSE)).
 
-- [1. Who can contribute to Synapse?](#1-who-can-contribute-to-synapse)
+## How to contribute
-- [2. What do I need?](#2-what-do-i-need)
-- [3. Get the source.](#3-get-the-source)
-- [4. Install the dependencies](#4-install-the-dependencies)
-  * [Under Unix (macOS, Linux, BSD, ...)](#under-unix-macos-linux-bsd-)
-  * [Under Windows](#under-windows)
-- [5. Get in touch.](#5-get-in-touch)
-- [6. Pick an issue.](#6-pick-an-issue)
-- [7. Turn coffee and documentation into code and documentation!](#7-turn-coffee-and-documentation-into-code-and-documentation)
-- [8. Test, test, test!](#8-test-test-test)
-  * [Run the linters.](#run-the-linters)
-  * [Run the unit tests.](#run-the-unit-tests)
-  * [Run the integration tests.](#run-the-integration-tests)
-- [9. Submit your patch.](#9-submit-your-patch)
-  * [Changelog](#changelog)
-    + [How do I know what to call the changelog file before I create the PR?](#how-do-i-know-what-to-call-the-changelog-file-before-i-create-the-pr)
-    + [Debian changelog](#debian-changelog)
-  * [Sign off](#sign-off)
-- [10. Turn feedback into better code.](#10-turn-feedback-into-better-code)
-- [11. Find a new issue.](#11-find-a-new-issue)
-- [Notes for maintainers on merging PRs etc](#notes-for-maintainers-on-merging-prs-etc)
-- [Conclusion](#conclusion)
 
-# 1. Who can contribute to Synapse?
+The preferred and easiest way to contribute changes to Matrix is to fork the
+relevant project on github, and then [create a pull request](
+https://help.github.com/articles/using-pull-requests/) to ask us to pull
+your changes into our repo.
 
-Everyone is welcome to contribute code to [matrix.org
+**The single biggest thing you need to know is: please base your changes on
-projects](https://github.com/matrix-org), provided that they are willing to
+the develop branch - *not* master.**
-license their contributions under the same license as the project itself. We
-follow a simple 'inbound=outbound' model for contributions: the act of
-submitting an 'inbound' contribution means that the contributor agrees to
-license the code under the same terms as the project's overall 'outbound'
-license - in our case, this is almost always Apache Software License v2 (see
-[LICENSE](LICENSE)).
 
-# 2. What do I need?
+We use the master branch to track the most recent release, so that folks who
+blindly clone the repo and automatically check out master get something that
+works. Develop is the unstable branch where all the development actually
+happens: the workflow is that contributors should fork the develop branch to
+make a 'feature' branch for a particular contribution, and then make a pull
+request to merge this back into the matrix.org 'official' develop branch. We
+use github's pull request workflow to review the contribution, and either ask
+you to make any refinements needed or merge it and make them ourselves. The
+changes will then land on master when we next do a release.
 
-The code of Synapse is written in Python 3. To do pretty much anything, you'll need [a recent version of Python 3](https://wiki.python.org/moin/BeginnersGuide/Download).
+We use [Buildkite](https://buildkite.com/matrix-dot-org/synapse) for continuous
+integration. If your change breaks the build, this will be shown in GitHub, so
+please keep an eye on the pull request for feedback.
 
-The source code of Synapse is hosted on GitHub. You will also need [a recent version of git](https://github.com/git-guides/install-git).
+To run unit tests in a local development environment, you can use:
 
-For some tests, you will need [a recent version of Docker](https://docs.docker.com/get-docker/).
+- ``tox -e py35`` (requires tox to be installed by ``pip install tox``)
+  for SQLite-backed Synapse on Python 3.5.
+- ``tox -e py36`` for SQLite-backed Synapse on Python 3.6.
+- ``tox -e py36-postgres`` for PostgreSQL-backed Synapse on Python 3.6
+  (requires a running local PostgreSQL with access to create databases).
+- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 3.5
+  (requires Docker). Entirely self-contained, recommended if you don't want to
+  set up PostgreSQL yourself.
 
+Docker images are available for running the integration tests (SyTest) locally,
+see the [documentation in the SyTest repo](
+https://github.com/matrix-org/sytest/blob/develop/docker/README.md) for more
+information.
 
-# 3. Get the source.
+## Code style
 
-The preferred and easiest way to contribute changes is to fork the relevant
+All Matrix projects have a well-defined code-style - and sometimes we've even
-project on GitHub, and then [create a pull request](
+got as far as documenting it... For instance, synapse's code style doc lives
-https://help.github.com/articles/using-pull-requests/) to ask us to pull your
+[here](docs/code_style.md).
-changes into our repo.
 
-Please base your changes on the `develop` branch.
+To facilitate meeting these criteria you can run `scripts-dev/lint.sh`
+locally. Since this runs the tools listed in the above document, you'll need
+python 3.6 and to install each tool:
 
-```sh
-git clone git@github.com:YOUR_GITHUB_USER_NAME/synapse.git
-git checkout develop
 ```
+# Install the dependencies
+pip install -U black flake8 flake8-comprehensions isort
 
-If you need help getting started with git, this is beyond the scope of the document, but you
+# Run the linter script
-can find many good git tutorials on the web.
-
-# 4. Install the dependencies
-
-## Under Unix (macOS, Linux, BSD, ...)
-
-Once you have installed Python 3 and added the source, please open a terminal and
-setup a *virtualenv*, as follows:
-
-```sh
-cd path/where/you/have/cloned/the/repository
-python3 -m venv ./env
-source ./env/bin/activate
-pip install -e ".[all,lint,mypy,test]"
-pip install tox
-```
-
-This will install the developer dependencies for the project.
-
-## Under Windows
-
-TBD
-
-
-# 5. Get in touch.
-
-Join our developer community on Matrix: #synapse-dev:matrix.org !
-
-
-# 6. Pick an issue.
-
-Fix your favorite problem or perhaps find a [Good First Issue](https://github.com/matrix-org/synapse/issues?q=is%3Aopen+is%3Aissue+label%3A%22Good+First+Issue%22)
-to work on.
-
-
-# 7. Turn coffee and documentation into code and documentation!
-
-Synapse's code style is documented [here](docs/code_style.md). Please follow
-it, including the conventions for the [sample configuration
-file](docs/code_style.md#configuration-file-format).
-
-There is a growing amount of documentation located in the [docs](docs)
-directory. This documentation is intended primarily for sysadmins running their
-own Synapse instance, as well as developers interacting externally with
-Synapse. [docs/dev](docs/dev) exists primarily to house documentation for
-Synapse developers. [docs/admin_api](docs/admin_api) houses documentation
-regarding Synapse's Admin API, which is used mostly by sysadmins and external
-service developers.
-
-If you add new files added to either of these folders, please use [GitHub-Flavoured
-Markdown](https://guides.github.com/features/mastering-markdown/).
-
-Some documentation also exists in [Synapse's GitHub
-Wiki](https://github.com/matrix-org/synapse/wiki), although this is primarily
-contributed to by community authors.
-
-
-# 8. Test, test, test!
-<a name="test-test-test"></a>
-
-While you're developing and before submitting a patch, you'll
-want to test your code.
-
-## Run the linters.
-
-The linters look at your code and do two things:
-
-- ensure that your code follows the coding style adopted by the project;
-- catch a number of errors in your code.
-
-They're pretty fast, don't hesitate!
-
-```sh
-source ./env/bin/activate
 ./scripts-dev/lint.sh
 ```
 
-Note that this script *will modify your files* to fix styling errors.
+**Note that the script does not just test/check, but also reformats code, so you
-Make sure that you have saved all your files.
+may wish to ensure any new code is committed first**. By default this script
+checks all files and can take some time; if you alter only certain files, you
+might wish to specify paths as arguments to reduce the run-time:
 
-If you wish to restrict the linters to only the files changed since the last commit
-(much faster!), you can instead run:
-
-```sh
-source ./env/bin/activate
-./scripts-dev/lint.sh -d
 ```
-
-Or if you know exactly which files you wish to lint, you can instead run:
-
-```sh
-source ./env/bin/activate
 ./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder
 ```
 
-## Run the unit tests.
+Before pushing new changes, ensure they don't produce linting errors. Commit any
+files that were corrected.
 
-The unit tests run parts of Synapse, including your changes, to see if anything
+Please ensure your changes match the cosmetic style of the existing project,
-was broken. They are slower than the linters but will typically catch more errors.
+and **never** mix cosmetic and functional changes in the same commit, as it
-
+makes it horribly hard to review otherwise.
-```sh
-source ./env/bin/activate
-trial tests
-```
-
-If you wish to only run *some* unit tests, you may specify
-another module instead of `tests` - or a test class or a method:
-
-```sh
-source ./env/bin/activate
-trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite
-```
-
-If your tests fail, you may wish to look at the logs:
-
-```sh
-less _trial_temp/test.log
-```
-
-## Run the integration tests.
-
-The integration tests are a more comprehensive suite of tests. They
-run a full version of Synapse, including your changes, to check if
-anything was broken. They are slower than the unit tests but will
-typically catch more errors.
-
-The following command will let you run the integration test with the most common
-configuration:
-
-```sh
-$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:py37
-```
-
-This configuration should generally cover  your needs. For more details about other configurations, see [documentation in the SyTest repo](https://github.com/matrix-org/sytest/blob/develop/docker/README.md).
-
-
-# 9. Submit your patch.
-
-Once you're happy with your patch, it's time to prepare a Pull Request.
-
-To prepare a Pull Request, please:
-
-1. verify that [all the tests pass](#test-test-test), including the coding style;
-2. [sign off](#sign-off) your contribution;
-3. `git push` your commit to your fork of Synapse;
-4. on GitHub, [create the Pull Request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request);
-5. add a [changelog entry](#changelog) and push it to your Pull Request;
-6. for most contributors, that's all - however, if you are a member of the organization `matrix-org`, on GitHub, please request a review from `matrix.org / Synapse Core`.
 
 
 ## Changelog
@@ -225,55 +98,24 @@ in the format of `PRnumber.type`. The type can be one of the following:
 * `removal` (also used for deprecations)
 * `misc` (for internal-only changes)
 
-This file will become part of our [changelog](
+The content of the file is your changelog entry, which should be a short
-https://github.com/matrix-org/synapse/blob/master/CHANGES.md) at the next
+description of your change in the same style as the rest of our [changelog](
-release, so the content of the file should be a short description of your
+https://github.com/matrix-org/synapse/blob/master/CHANGES.md). The file can
-change in the same style as the rest of the changelog. The file can contain Markdown
+contain Markdown formatting, and should end with a full stop (.) or an
-formatting, and should end with a full stop (.) or an exclamation mark (!) for
+exclamation mark (!) for consistency.
-consistency.
 
 Adding credits to the changelog is encouraged, we value your
 contributions and would like to have you shouted out in the release notes!
 
 For example, a fix in PR #1234 would have its changelog entry in
-`changelog.d/1234.bugfix`, and contain content like:
+`changelog.d/1234.bugfix`, and contain content like "The security levels of
+Florbs are now validated when received over federation. Contributed by Jane
+Matrix.".
 
-> The security levels of Florbs are now validated when received
+## Debian changelog
-> via the `/federation/florb` endpoint. Contributed by Jane Matrix.
-
-If there are multiple pull requests involved in a single bugfix/feature/etc,
-then the content for each `changelog.d` file should be the same. Towncrier will
-merge the matching files together into a single changelog entry when we come to
-release.
-
-### How do I know what to call the changelog file before I create the PR?
-
-Obviously, you don't know if you should call your newsfile
-`1234.bugfix` or `5678.bugfix` until you create the PR, which leads to a
-chicken-and-egg problem.
-
-There are two options for solving this:
-
- 1. Open the PR without a changelog file, see what number you got, and *then*
-    add the changelog file to your branch (see [Updating your pull
-    request](#updating-your-pull-request)), or:
-
- 1. Look at the [list of all
-    issues/PRs](https://github.com/matrix-org/synapse/issues?q=), add one to the
-    highest number you see, and quickly open the PR before somebody else claims
-    your number.
-
-    [This
-    script](https://github.com/richvdh/scripts/blob/master/next_github_number.sh)
-    might be helpful if you find yourself doing this a lot.
-
-Sorry, we know it's a bit fiddly, but it's *really* helpful for us when we come
-to put together a release!
-
-### Debian changelog
 
 Changes which affect the debian packaging files (in `debian`) are an
-exception to the rule that all changes require a `changelog.d` file.
+exception.
 
 In this case, you will need to add an entry to the debian changelog for the
 next release. For this, run the following command:
@@ -358,36 +200,21 @@ Git allows you to add this signoff automatically when using the `-s`
 flag to `git commit`, which uses the name and email set in your
 `user.name` and `user.email` git configs.
 
+## Merge Strategy
 
-# 10. Turn feedback into better code.
+We use the commit history of develop/master extensively to identify
+when regressions were introduced and what changes have been made.
 
-Once the Pull Request is opened, you will see a few things:
+We aim to have a clean merge history, which means we normally squash-merge
+changes into develop. For small changes this means there is no need to rebase
+to clean up your PR before merging. Larger changes with an organised set of
+commits may be merged as-is, if the history is judged to be useful.
 
-1. our automated CI (Continuous Integration) pipeline will run (again) the linters, the unit tests, the integration tests and more;
+This use of squash-merging will mean PRs built on each other will be hard to 
-2. one or more of the developers will take a look at your Pull Request and offer feedback.
+merge. We suggest avoiding these where possible, and if required, ensuring
+each PR has a tidy set of commits to ease merging.
 
-From this point, you should:
+## Conclusion
-
-1. Look at the results of the CI pipeline.
-   - If there is any error, fix the error.
-2. If a developer has requested changes, make these changes and let us know if it is ready for a developer to review again.
-3. Create a new commit with the changes.
-   - Please do NOT overwrite the history. New commits make the reviewer's life easier.
-   - Push this commits to your Pull Request.
-4. Back to 1.
-
-Once both the CI and the developers are happy, the patch will be merged into Synapse and released shortly!
-
-# 11. Find a new issue.
-
-By now, you know the drill!
-
-# Notes for maintainers on merging PRs etc
-
-There are some notes for those with commit access to the project on how we
-manage git [here](docs/dev/git.md).
-
-# Conclusion
 
 That's it! Matrix is a very open and collaborative project as you might expect
 given our obsession with open communication. If we're going to successfully

INSTALL.md

@@ -1,44 +1,17 @@
-# Installation Instructions
+- [Choosing your server name](#choosing-your-server-name)
-
+- [Installing Synapse](#installing-synapse)
-There are 3 steps to follow under **Installation Instructions**.
+  - [Installing from source](#installing-from-source)
-
+    - [Platform-Specific Instructions](#platform-specific-instructions)
-- [Installation Instructions](#installation-instructions)
-  - [Choosing your server name](#choosing-your-server-name)
-  - [Installing Synapse](#installing-synapse)
-    - [Installing from source](#installing-from-source)
-      - [Platform-Specific Instructions](#platform-specific-instructions)
-        - [Debian/Ubuntu/Raspbian](#debianubunturaspbian)
-        - [ArchLinux](#archlinux)
-        - [CentOS/Fedora](#centosfedora)
-        - [macOS](#macos)
-        - [OpenSUSE](#opensuse)
-        - [OpenBSD](#openbsd)
-        - [Windows](#windows)
-    - [Prebuilt packages](#prebuilt-packages)
-      - [Docker images and Ansible playbooks](#docker-images-and-ansible-playbooks)
-      - [Debian/Ubuntu](#debianubuntu)
-        - [Matrix.org packages](#matrixorg-packages)
-        - [Downstream Debian packages](#downstream-debian-packages)
-        - [Downstream Ubuntu packages](#downstream-ubuntu-packages)
-      - [Fedora](#fedora)
-      - [OpenSUSE](#opensuse-1)
-      - [SUSE Linux Enterprise Server](#suse-linux-enterprise-server)
-      - [ArchLinux](#archlinux-1)
-      - [Void Linux](#void-linux)
-      - [FreeBSD](#freebsd)
-      - [OpenBSD](#openbsd-1)
-      - [NixOS](#nixos)
-  - [Setting up Synapse](#setting-up-synapse)
-    - [Using PostgreSQL](#using-postgresql)
-    - [TLS certificates](#tls-certificates)
-    - [Client Well-Known URI](#client-well-known-uri)
-    - [Email](#email)
-    - [Registering a user](#registering-a-user)
-    - [Setting up a TURN server](#setting-up-a-turn-server)
-    - [URL previews](#url-previews)
     - [Troubleshooting Installation](#troubleshooting-installation)
+  - [Prebuilt packages](#prebuilt-packages)
+- [Setting up Synapse](#setting-up-synapse)
+  - [TLS certificates](#tls-certificates)
+  - [Email](#email)
+  - [Registering a user](#registering-a-user)
+  - [Setting up a TURN server](#setting-up-a-turn-server)
+  - [URL previews](#url-previews)
 
-## Choosing your server name
+# Choosing your server name
 
 It is important to choose the name for your server before you install Synapse,
 because it cannot be changed later.
@@ -54,16 +27,16 @@ that your email address is probably `user@example.com` rather than
 `user@email.example.com`) - but doing so may require more advanced setup: see
 [Setting up Federation](docs/federate.md).
 
-## Installing Synapse
+# Installing Synapse
 
-### Installing from source
+## Installing from source
 
 (Prebuilt packages are available for some platforms - see [Prebuilt packages](#prebuilt-packages).)
 
 System requirements:
 
 - POSIX-compliant system (tested on Linux & OS X)
-- Python 3.5.2 or later, up to Python 3.9.
+- Python 3.5, 3.6, 3.7 or 3.8.
 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
 
 Synapse is written in Python but some of the libraries it uses are written in
@@ -74,7 +47,7 @@ these on various platforms.
 
 To install the Synapse homeserver run:
 
-```sh
+```
 mkdir -p ~/synapse
 virtualenv -p python3 ~/synapse/env
 source ~/synapse/env/bin/activate
@@ -91,15 +64,15 @@ prefer.
 This Synapse installation can then be later upgraded by using pip again with the
 update flag:
 
-```sh
+```
 source ~/synapse/env/bin/activate
 pip install -U matrix-synapse
 ```
 
 Before you can start Synapse, you will need to generate a configuration
-file. To do this, run (in your virtualenv, as before):
+file. To do this, run (in your virtualenv, as before)::
 
-```sh
+```
 cd ~/synapse
 python -m synapse.app.homeserver \
     --server-name my.domain.name \
@@ -111,60 +84,74 @@ python -m synapse.app.homeserver \
 ... substituting an appropriate value for `--server-name`.
 
 This command will generate you a config file that you can then customise, but it will
-also generate a set of keys for you. These keys will allow your homeserver to
+also generate a set of keys for you. These keys will allow your Home Server to
-identify itself to other homeserver, so don't lose or delete them. It would be
+identify itself to other Home Servers, so don't lose or delete them. It would be
 wise to back them up somewhere safe. (If, for whatever reason, you do need to
-change your homeserver's keys, you may find that other homeserver have the
+change your Home Server's keys, you may find that other Home Servers have the
 old key cached. If you update the signing key, you should change the name of the
 key in the `<server name>.signing.key` file (the second word) to something
-different. See the [spec](https://matrix.org/docs/spec/server_server/latest.html#retrieving-server-keys) for more information on key management).
+different. See the
+[spec](https://matrix.org/docs/spec/server_server/latest.html#retrieving-server-keys)
+for more information on key management.)
 
 To actually run your new homeserver, pick a working directory for Synapse to
-run (e.g. `~/synapse`), and:
+run (e.g. `~/synapse`), and::
 
-```sh
+    cd ~/synapse
-cd ~/synapse
+    source env/bin/activate
-source env/bin/activate
+    synctl start
-synctl start
-```
 
-#### Platform-Specific Instructions
+### Platform-Specific Instructions
 
-##### Debian/Ubuntu/Raspbian
+#### Debian/Ubuntu/Raspbian
 
 Installing prerequisites on Ubuntu or Debian:
 
-```sh
+```
-sudo apt install build-essential python3-dev libffi-dev \
+sudo apt-get install build-essential python3-dev libffi-dev \
                      python3-pip python3-setuptools sqlite3 \
-                     libssl-dev virtualenv libjpeg-dev libxslt1-dev
+                     libssl-dev python3-virtualenv libjpeg-dev libxslt1-dev
 ```
 
-##### ArchLinux
+#### ArchLinux
 
 Installing prerequisites on ArchLinux:
 
-```sh
+```
 sudo pacman -S base-devel python python-pip \
                python-setuptools python-virtualenv sqlite3
 ```
 
-##### CentOS/Fedora
+#### CentOS/Fedora
 
-Installing prerequisites on CentOS or Fedora Linux:
+Installing prerequisites on CentOS 8 or Fedora>26:
 
-```sh
+```
 sudo dnf install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
-                 libwebp-devel libxml2-devel libxslt-devel libpq-devel \
+                 libwebp-devel tk-devel redhat-rpm-config \
-                 python3-virtualenv libffi-devel openssl-devel python3-devel
+                 python3-virtualenv libffi-devel openssl-devel
 sudo dnf groupinstall "Development Tools"
 ```
 
-##### macOS
+Installing prerequisites on CentOS 7 or Fedora<=25:
+
+```
+sudo yum install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
+                 lcms2-devel libwebp-devel tcl-devel tk-devel redhat-rpm-config \
+                 python3-virtualenv libffi-devel openssl-devel
+sudo yum groupinstall "Development Tools"
+```
+
+Note that Synapse does not support versions of SQLite before 3.11, and CentOS 7
+uses SQLite 3.7. You may be able to work around this by installing a more
+recent SQLite version, but it is recommended that you instead use a Postgres
+database: see [docs/postgres.md](docs/postgres.md).
+
+#### macOS
 
 Installing prerequisites on macOS:
 
-```sh
+```
 xcode-select --install
 sudo easy_install pip
 sudo pip install virtualenv
@@ -174,102 +161,134 @@ brew install pkg-config libffi
 On macOS Catalina (10.15) you may need to explicitly install OpenSSL
 via brew and inform `pip` about it so that `psycopg2` builds:
 
-```sh
+```
 brew install openssl@1.1
-export LDFLAGS="-L/usr/local/opt/openssl/lib"
+export LDFLAGS=-L/usr/local/Cellar/openssl\@1.1/1.1.1d/lib/
-export CPPFLAGS="-I/usr/local/opt/openssl/include"
 ```
 
-##### OpenSUSE
+#### OpenSUSE
 
 Installing prerequisites on openSUSE:
 
-```sh
+```
 sudo zypper in -t pattern devel_basis
 sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \
                python-devel libffi-devel libopenssl-devel libjpeg62-devel
 ```
 
-##### OpenBSD
+#### OpenBSD
 
-A port of Synapse is available under `net/synapse`. The filesystem
+Installing prerequisites on OpenBSD:
-underlying the homeserver directory (defaults to `/var/synapse`) has to be
-mounted with `wxallowed` (cf. `mount(8)`), so creating a separate filesystem
-and mounting it to `/var/synapse` should be taken into consideration.
 
-To be able to build Synapse's dependency on python the `WRKOBJDIR`
+```
-(cf. `bsd.port.mk(5)`) for building python, too, needs to be on a filesystem
+doas pkg_add python libffi py-pip py-setuptools sqlite3 py-virtualenv \
-mounted with `wxallowed` (cf. `mount(8)`).
+              libxslt jpeg
-
-Creating a `WRKOBJDIR` for building python under `/usr/local` (which on a
-default OpenBSD installation is mounted with `wxallowed`):
-
-```sh
-doas mkdir /usr/local/pobj_wxallowed
 ```
 
-Assuming `PORTS_PRIVSEP=Yes` (cf. `bsd.port.mk(5)`) and `SUDO=doas` are
+There is currently no port for OpenBSD. Additionally, OpenBSD's security
-configured in `/etc/mk.conf`:
+settings require a slightly more difficult installation process.
 
-```sh
+XXX: I suspect this is out of date.
-doas chown _pbuild:_pbuild /usr/local/pobj_wxallowed
-```
 
-Setting the `WRKOBJDIR` for building python:
+1. Create a new directory in `/usr/local` called `_synapse`. Also, create a
+   new user called `_synapse` and set that directory as the new user's home.
+   This is required because, by default, OpenBSD only allows binaries which need
+   write and execute permissions on the same memory space to be run from
+   `/usr/local`.
+2. `su` to the new `_synapse` user and change to their home directory.
+3. Create a new virtualenv: `virtualenv -p python2.7 ~/.synapse`
+4. Source the virtualenv configuration located at
+   `/usr/local/_synapse/.synapse/bin/activate`. This is done in `ksh` by
+   using the `.` command, rather than `bash`'s `source`.
+5. Optionally, use `pip` to install `lxml`, which Synapse needs to parse
+   webpages for their titles.
+6. Use `pip` to install this repository: `pip install matrix-synapse`
+7. Optionally, change `_synapse`'s shell to `/bin/false` to reduce the
+   chance of a compromised Synapse server being used to take over your box.
 
-```sh
+After this, you may proceed with the rest of the install directions.
-echo WRKOBJDIR_lang/python/3.7=/usr/local/pobj_wxallowed  \\nWRKOBJDIR_lang/python/2.7=/usr/local/pobj_wxallowed >> /etc/mk.conf
-```
 
-Building Synapse:
+#### Windows
-
-```sh
-cd /usr/ports/net/synapse
-make install
-```
-
-##### Windows
 
 If you wish to run or develop Synapse on Windows, the Windows Subsystem For
 Linux provides a Linux environment on Windows 10 which is capable of using the
 Debian, Fedora, or source installation methods. More information about WSL can
-be found at <https://docs.microsoft.com/en-us/windows/wsl/install-win10> for
+be found at https://docs.microsoft.com/en-us/windows/wsl/install-win10 for
-Windows 10 and <https://docs.microsoft.com/en-us/windows/wsl/install-on-server>
+Windows 10 and https://docs.microsoft.com/en-us/windows/wsl/install-on-server
 for Windows Server.
 
-### Prebuilt packages
+### Troubleshooting Installation
+
+XXX a bunch of this is no longer relevant.
+
+Synapse requires pip 8 or later, so if your OS provides too old a version you
+may need to manually upgrade it::
+
+    sudo pip install --upgrade pip
+
+Installing may fail with `Could not find any downloads that satisfy the requirement pymacaroons-pynacl (from matrix-synapse==0.12.0)`.
+You can fix this by manually upgrading pip and virtualenv::
+
+    sudo pip install --upgrade virtualenv
+
+You can next rerun `virtualenv -p python3 synapse` to update the virtual env.
+
+Installing may fail during installing virtualenv with `InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.`
+You can fix this  by manually installing ndg-httpsclient::
+
+    pip install --upgrade ndg-httpsclient
+
+Installing may fail with `mock requires setuptools>=17.1. Aborting installation`.
+You can fix this by upgrading setuptools::
+
+    pip install --upgrade setuptools
+
+If pip crashes mid-installation for reason (e.g. lost terminal), pip may
+refuse to run until you remove the temporary installation directory it
+created. To reset the installation::
+
+    rm -rf /tmp/pip_install_matrix
+
+pip seems to leak *lots* of memory during installation.  For instance, a Linux
+host with 512MB of RAM may run out of memory whilst installing Twisted.  If this
+happens, you will have to individually install the dependencies which are
+failing, e.g.::
+
+    pip install twisted
+
+## Prebuilt packages
 
 As an alternative to installing from source, prebuilt packages are available
 for a number of platforms.
 
-#### Docker images and Ansible playbooks
+### Docker images and Ansible playbooks
 
-There is an official synapse image available at
+There is an offical synapse image available at
-<https://hub.docker.com/r/matrixdotorg/synapse> which can be used with
+https://hub.docker.com/r/matrixdotorg/synapse which can be used with
-the docker-compose file available at [contrib/docker](contrib/docker). Further
+the docker-compose file available at [contrib/docker](contrib/docker). Further information on
-information on this including configuration options is available in the README
+this including configuration options is available in the README on
-on hub.docker.com.
+hub.docker.com.
 
 Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
 Dockerfile to automate a synapse server in a single Docker image, at
-<https://hub.docker.com/r/avhost/docker-matrix/tags/>
+https://hub.docker.com/r/avhost/docker-matrix/tags/
 
 Slavi Pantaleev has created an Ansible playbook,
 which installs the offical Docker image of Matrix Synapse
-along with many other Matrix-related services (Postgres database, Element, coturn,
+along with many other Matrix-related services (Postgres database, riot-web, coturn, mxisd, SSL support, etc.).
-ma1sd, SSL support, etc.).
 For more details, see
-<https://github.com/spantaleev/matrix-docker-ansible-deploy>
+https://github.com/spantaleev/matrix-docker-ansible-deploy
 
-#### Debian/Ubuntu
 
-##### Matrix.org packages
+### Debian/Ubuntu
+
+#### Matrix.org packages
 
 Matrix.org provides Debian/Ubuntu packages of the latest stable version of
-Synapse via <https://packages.matrix.org/debian/>. They are available for Debian
+Synapse via https://packages.matrix.org/debian/. They are available for Debian
 9 (Stretch), Ubuntu 16.04 (Xenial), and later. To use them:
 
-```sh
+```
 sudo apt install -y lsb-release wget apt-transport-https
 sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
 echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" |
@@ -289,61 +308,56 @@ The fingerprint of the repository signing key (as shown by `gpg
 /usr/share/keyrings/matrix-org-archive-keyring.gpg`) is
 `AAF9AE843A7584B5A3E4CD2BCF45A512DE2DA058`.
 
-##### Downstream Debian packages
+#### Downstream Debian/Ubuntu packages
 
-We do not recommend using the packages from the default Debian `buster`
+For `buster` and `sid`, Synapse is available in the Debian repositories and
-repository at this time, as they are old and suffer from known security
+it should be possible to install it with simply:
-vulnerabilities. You can install the latest version of Synapse from
-[our repository](#matrixorg-packages) or from `buster-backports`. Please
-see the [Debian documentation](https://backports.debian.org/Instructions/)
-for information on how to use backports.
 
-If you are using Debian `sid` or testing, Synapse is available in the default
+```
-repositories and it should be possible to install it simply with:
+    sudo apt install matrix-synapse
-
-```sh
-sudo apt install matrix-synapse
 ```
 
-##### Downstream Ubuntu packages
+There is also a version of `matrix-synapse` in `stretch-backports`. Please see
+the [Debian documentation on
+backports](https://backports.debian.org/Instructions/) for information on how
+to use them.
 
-We do not recommend using the packages in the default Ubuntu repository
+We do not recommend using the packages in downstream Ubuntu at this time, as
-at this time, as they are old and suffer from known security vulnerabilities.
+they are old and suffer from known security vulnerabilities.
-The latest version of Synapse can be installed from [our repository](#matrixorg-packages).
 
-#### Fedora
+### Fedora
 
 Synapse is in the Fedora repositories as `matrix-synapse`:
 
-```sh
+```
 sudo dnf install matrix-synapse
 ```
 
 Oleg Girko provides Fedora RPMs at
-<https://obs.infoserver.lv/project/monitor/matrix-synapse>
+https://obs.infoserver.lv/project/monitor/matrix-synapse
 
-#### OpenSUSE
+### OpenSUSE
 
 Synapse is in the OpenSUSE repositories as `matrix-synapse`:
 
-```sh
+```
 sudo zypper install matrix-synapse
 ```
 
-#### SUSE Linux Enterprise Server
+### SUSE Linux Enterprise Server
 
 Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 repository at
-<https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/>
+https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/
 
-#### ArchLinux
+### ArchLinux
 
 The quickest way to get up and running with ArchLinux is probably with the community package
-<https://www.archlinux.org/packages/community/any/matrix-synapse/>, which should pull in most of
+https://www.archlinux.org/packages/community/any/matrix-synapse/, which should pull in most of
 the necessary dependencies.
 
 pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):
 
-```sh
+```
 sudo pip install --upgrade pip
 ```
 
@@ -352,65 +366,36 @@ ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly
 compile it under the right architecture. (This should not be needed if
 installing under virtualenv):
 
-```sh
+```
 sudo pip uninstall py-bcrypt
 sudo pip install py-bcrypt
 ```
 
-#### Void Linux
+### Void Linux
 
 Synapse can be found in the void repositories as 'synapse':
 
-```sh
+    xbps-install -Su
-xbps-install -Su
+    xbps-install -S synapse
-xbps-install -S synapse
-```
 
-#### FreeBSD
+### FreeBSD
 
 Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from:
 
-- Ports: `cd /usr/ports/net-im/py-matrix-synapse && make install clean`
+ - Ports: `cd /usr/ports/net-im/py-matrix-synapse && make install clean`
-- Packages: `pkg install py37-matrix-synapse`
+ - Packages: `pkg install py27-matrix-synapse`
 
-#### OpenBSD
 
-As of OpenBSD 6.7 Synapse is available as a pre-compiled binary. The filesystem
+### NixOS
-underlying the homeserver directory (defaults to `/var/synapse`) has to be
-mounted with `wxallowed` (cf. `mount(8)`), so creating a separate filesystem
-and mounting it to `/var/synapse` should be taken into consideration.
-
-Installing Synapse:
-
-```sh
-doas pkg_add synapse
-```
-
-#### NixOS
 
 Robin Lambertz has packaged Synapse for NixOS at:
-<https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-synapse.nix>
+https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-synapse.nix
 
-## Setting up Synapse
+# Setting up Synapse
 
 Once you have installed synapse as above, you will need to configure it.
 
-### Using PostgreSQL
+## TLS certificates
-
-By default Synapse uses [SQLite](https://sqlite.org/) and in doing so trades performance for convenience.
-SQLite is only recommended in Synapse for testing purposes or for servers with
-very light workloads.
-
-Almost all installations should opt to use [PostgreSQL](https://www.postgresql.org). Advantages include:
-
-- significant performance improvements due to the superior threading and
-  caching model, smarter query optimiser
-- allowing the DB to be run on separate hardware
-
-For information on how to install and use PostgreSQL in Synapse, please see
-[docs/postgres.md](docs/postgres.md)
-
-### TLS certificates
 
 The default configuration exposes a single HTTP port on the local
 interface: `http://localhost:8008`. It is suitable for local testing,
@@ -424,90 +409,36 @@ The recommended way to do so is to set up a reverse proxy on port
 Alternatively, you can configure Synapse to expose an HTTPS port. To do
 so, you will need to edit `homeserver.yaml`, as follows:
 
-- First, under the `listeners` section, uncomment the configuration for the
+* First, under the `listeners` section, uncomment the configuration for the
   TLS-enabled listener. (Remove the hash sign (`#`) at the start of
   each line). The relevant lines are like this:
 
-```yaml
-  - port: 8448
-    type: http
-    tls: true
-    resources:
-      - names: [client, federation]
   ```
-
+    - port: 8448
-- You will also need to uncomment the `tls_certificate_path` and
+      type: http
-  `tls_private_key_path` lines under the `TLS` section. You will need to manage
+      tls: true
-  provisioning of these certificates yourself — Synapse had built-in ACME
+      resources:
-  support, but the ACMEv1 protocol Synapse implements is deprecated, not
+        - names: [client, federation]
-  allowed by LetsEncrypt for new sites, and will break for existing sites in
+  ```
-  late 2020. See [ACME.md](docs/ACME.md).
+* You will also need to uncomment the `tls_certificate_path` and
-
+  `tls_private_key_path` lines under the `TLS` section. You can either
+  point these settings at an existing certificate and key, or you can
+  enable Synapse's built-in ACME (Let's Encrypt) support. Instructions
+  for having Synapse automatically provision and renew federation
+  certificates through ACME can be found at [ACME.md](docs/ACME.md).
+  Note that, as pointed out in that document, this feature will not
+  work with installs set up after November 2019. 
+  
   If you are using your own certificate, be sure to use a `.pem` file that
   includes the full certificate chain including any intermediate certificates
   (for instance, if using certbot, use `fullchain.pem` as your certificate, not
   `cert.pem`).
 
 For a more detailed guide to configuring your server for federation, see
-[federate.md](docs/federate.md).
+[federate.md](docs/federate.md)
 
-### Client Well-Known URI
 
-Setting up the client Well-Known URI is optional but if you set it up, it will
+## Email
-allow users to enter their full username (e.g. `@user:<server_name>`) into clients
-which support well-known lookup to automatically configure the homeserver and
-identity server URLs. This is useful so that users don't have to memorize or think
-about the actual homeserver URL you are using.
-
-The URL `https://<server_name>/.well-known/matrix/client` should return JSON in
-the following format.
-
-```json
-{
-  "m.homeserver": {
-    "base_url": "https://<matrix.example.com>"
-  }
-}
-```
-
-It can optionally contain identity server information as well.
-
-```json
-{
-  "m.homeserver": {
-    "base_url": "https://<matrix.example.com>"
-  },
-  "m.identity_server": {
-    "base_url": "https://<identity.example.com>"
-  }
-}
-```
-
-To work in browser based clients, the file must be served with the appropriate
-Cross-Origin Resource Sharing (CORS) headers. A recommended value would be
-`Access-Control-Allow-Origin: *` which would allow all browser based clients to
-view it.
-
-In nginx this would be something like:
-
-```nginx
-location /.well-known/matrix/client {
-    return 200 '{"m.homeserver": {"base_url": "https://<matrix.example.com>"}}';
-    default_type application/json;
-    add_header Access-Control-Allow-Origin *;
-}
-```
-
-You should also ensure the `public_baseurl` option in `homeserver.yaml` is set
-correctly. `public_baseurl` should be set to the URL that clients will use to
-connect to your server. This is the same URL you put for the `m.homeserver`
-`base_url` above.
-
-```yaml
-public_baseurl: "https://<matrix.example.com>"
-```
-
-### Email
 
 It is desirable for Synapse to have the capability to send email. This allows
 Synapse to send password reset emails, send verifications when an email address
@@ -522,15 +453,15 @@ and `notif_from` fields filled out.  You may also need to set `smtp_user`,
 If email is not configured, password reset, registration and notifications via
 email will be disabled.
 
-### Registering a user
+## Registering a user
 
-The easiest way to create a new user is to do so from a client like [Element](https://element.io/).
+The easiest way to create a new user is to do so from a client like [Riot](https://riot.im).
 
 Alternatively you can do so from the command line if you have installed via pip.
 
 This can be done as follows:
 
-```sh
+```
 $ source ~/synapse/env/bin/activate
 $ synctl start # if not already running
 $ register_new_matrix_user -c homeserver.yaml http://localhost:8008
@@ -548,35 +479,22 @@ value is generated by `--generate-config`), but it should be kept secret, as
 anyone with knowledge of it can register users, including admin accounts,
 on your server even if `enable_registration` is `false`.
 
-### Setting up a TURN server
+## Setting up a TURN server
 
 For reliable VoIP calls to be routed via this homeserver, you MUST configure
-a TURN server. See [docs/turn-howto.md](docs/turn-howto.md) for details.
+a TURN server.  See [docs/turn-howto.md](docs/turn-howto.md) for details.
 
-### URL previews
+## URL previews
 
 Synapse includes support for previewing URLs, which is disabled by default.  To
 turn it on you must enable the `url_preview_enabled: True` config parameter
 and explicitly specify the IP ranges that Synapse is not allowed to spider for
 previewing in the `url_preview_ip_range_blacklist` configuration parameter.
 This is critical from a security perspective to stop arbitrary Matrix users
-spidering 'internal' URLs on your network. At the very least we recommend that
+spidering 'internal' URLs on your network.  At the very least we recommend that
 your loopback and RFC1918 IP addresses are blacklisted.
 
-This also requires the optional `lxml` python dependency to be  installed. This
+This also requires the optional lxml and netaddr python dependencies to be
-in turn requires the `libxml2` library to be available - on  Debian/Ubuntu this
+installed.  This in turn requires the libxml2 library to be available - on
-means `apt-get install libxml2-dev`, or equivalent for your OS.
+Debian/Ubuntu this means `apt-get install libxml2-dev`, or equivalent for
-
+your OS.
-### Troubleshooting Installation
-
-`pip` seems to leak *lots* of memory during installation. For instance, a Linux
-host with 512MB of RAM may run out of memory whilst installing Twisted. If this
-happens, you will have to individually install the dependencies which are
-failing, e.g.:
-
-```sh
-pip install twisted
-```
-
-If you have any other problems, feel free to ask in
-[#synapse:matrix.org](https://matrix.to/#/#synapse:matrix.org).

MANIFEST.in

@@ -20,10 +20,9 @@ recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
 recursive-include tests *.py
-recursive-include tests *.pem
+include tests/http/ca.crt
-recursive-include tests *.p8
+include tests/http/ca.key
-recursive-include tests *.crt
+include tests/http/server.key
-recursive-include tests *.key
 
 recursive-include synapse/res *
 recursive-include synapse/static *.css
@@ -31,24 +30,23 @@ recursive-include synapse/static *.gif
 recursive-include synapse/static *.html
 recursive-include synapse/static *.js
 
-exclude .codecov.yml
-exclude .coveragerc
-exclude .dockerignore
-exclude .editorconfig
 exclude Dockerfile
-exclude mypy.ini
+exclude .dockerignore
-exclude sytest-blacklist
 exclude test_postgresql.sh
+exclude .editorconfig
+exclude sytest-blacklist
 
 include pyproject.toml
 recursive-include changelog.d *
 
 prune .buildkite
 prune .circleci
+prune .codecov.yml
+prune .coveragerc
 prune .github
-prune contrib
 prune debian
 prune demo/etc
 prune docker
+prune mypy.ini
 prune snap
 prune stubs

README.rst

@@ -1,7 +1,3 @@
-=========================================================
-Synapse |support| |development| |license| |pypi| |python|
-=========================================================
-
 .. contents::
 
 Introduction
@@ -41,7 +37,7 @@ which handle:
 - Eventually-consistent cryptographically secure synchronisation of room
   state across a global open network of federated servers and services
 - Sending and receiving extensible messages in a room with (optional)
-  end-to-end encryption
+  end-to-end encryption[1]
 - Inviting, joining, leaving, kicking, banning room members
 - Managing user accounts (registration, login, logout)
 - Using 3rd Party IDs (3PIDs) such as email addresses, phone numbers,
@@ -78,15 +74,7 @@ at the `Matrix spec <https://matrix.org/docs/spec>`_, and experiment with the
 
 Thanks for using Matrix!
 
-Support
+[1] End-to-end encryption is currently in beta: `blog post <https://matrix.org/blog/2016/11/21/matrixs-olm-end-to-end-encryption-security-assessment-released-and-implemented-cross-platform-on-riot-at-last>`_.
-=======
-
-For support installing or managing Synapse, please join |room|_ (from a matrix.org
-account if necessary) and ask questions there. We do not use GitHub issues for
-support requests, only for bug reports and feature requests.
-
-.. |room| replace:: ``#synapse:matrix.org``
-.. _room: https://matrix.to/#/#synapse:matrix.org
 
 
 Synapse Installation
@@ -108,11 +96,12 @@ Unless you are running a test instance of Synapse on your local machine, in
 general, you will need to enable TLS support before you can successfully
 connect from a client: see `<INSTALL.md#tls-certificates>`_.
 
-An easy way to get started is to login or register via Element at
+An easy way to get started is to login or register via Riot at
-https://app.element.io/#/login or https://app.element.io/#/register respectively.
+https://riot.im/app/#/login or https://riot.im/app/#/register respectively.
 You will need to change the server you are logging into from ``matrix.org``
 and instead specify a Homeserver URL of ``https://<server_name>:8448``
 (or just ``https://<server_name>`` if you are using a reverse proxy).
+(Leave the identity server as the default - see `Identity servers`_.)
 If you prefer to use another client, refer to our
 `client breakdown <https://matrix.org/docs/projects/clients-matrix>`_.
 
@@ -129,7 +118,7 @@ it, specify ``enable_registration: true`` in ``homeserver.yaml``. (It is then
 recommended to also set up CAPTCHA - see `<docs/CAPTCHA_SETUP.md>`_.)
 
 Once ``enable_registration`` is set to ``true``, it is possible to register a
-user via a Matrix client.
+user via `riot.im <https://riot.im/app/#/register>`_ or other Matrix clients.
 
 Your new user name will be formed partly from the ``server_name``, and partly
 from a localpart you specify when you create the account. Your name will take
@@ -175,6 +164,30 @@ versions of synapse.
 
 .. _UPGRADE.rst: UPGRADE.rst
 
+
+Using PostgreSQL
+================
+
+Synapse offers two database engines:
+ * `SQLite <https://sqlite.org/>`_
+ * `PostgreSQL <https://www.postgresql.org>`_
+
+By default Synapse uses SQLite in and doing so trades performance for convenience.
+SQLite is only recommended in Synapse for testing purposes or for servers with
+light workloads.
+
+Almost all installations should opt to use PostreSQL. Advantages include:
+
+* significant performance improvements due to the superior threading and
+  caching model, smarter query optimiser
+* allowing the DB to be run on separate hardware
+* allowing basic active/backup high-availability with a "hot spare" synapse
+  pointing at the same DB master, as well as enabling DB replication in
+  synapse itself.
+
+For information on how to install and use PostgreSQL, please see
+`docs/postgres.md <docs/postgres.md>`_.
+
 .. _reverse-proxy:
 
 Using a reverse proxy with Synapse
@@ -183,9 +196,8 @@ Using a reverse proxy with Synapse
 It is recommended to put a reverse proxy such as
 `nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
 `Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_,
-`Caddy <https://caddyserver.com/docs/quick-starts/reverse-proxy>`_,
+`Caddy <https://caddyserver.com/docs/proxy>`_ or
-`HAProxy <https://www.haproxy.org/>`_ or
+`HAProxy <https://www.haproxy.org/>`_ in front of Synapse. One advantage of
-`relayd <https://man.openbsd.org/relayd.8>`_ in front of Synapse. One advantage of
 doing so is that it means that you can expose the default https port (443) to
 Matrix clients without needing to run Synapse with root privileges.
 
@@ -224,9 +236,10 @@ email address.
 Password reset
 ==============
 
-Users can reset their password through their client. Alternatively, a server admin
+If a user has registered an email address to their account using an identity
-can reset a users password using the `admin API <docs/admin_api/user_admin_api.rst#reset-password>`_
+server, they can request a password-reset token via clients such as Riot.
-or by directly editing the database as shown below.
+
+A manual password reset can be done via direct database access as follows.
 
 First calculate the hash of the new password::
 
@@ -235,7 +248,7 @@ First calculate the hash of the new password::
     Confirm password:
     $2a$12$xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 
-Then update the ``users`` table in the database::
+Then update the `users` table in the database::
 
     UPDATE users SET password_hash='$2a$12$xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
         WHERE name='@test:test.com';
@@ -244,8 +257,6 @@ Then update the ``users`` table in the database::
 Synapse Development
 ===================
 
-Join our developer community on Matrix: `#synapse-dev:matrix.org <https://matrix.to/#/#synapse-dev:matrix.org>`_
-
 Before setting up a development environment for synapse, make sure you have the
 system dependencies (such as the python header files) installed - see
 `Installing from source <INSTALL.md#installing-from-source>`_.
@@ -259,48 +270,23 @@ directory of your choice::
 Synapse has a number of external dependencies, that are easiest
 to install using pip and a virtualenv::
 
-    python3 -m venv ./env
+    virtualenv -p python3 env
-    source ./env/bin/activate
+    source env/bin/activate
-    pip install -e ".[all,test]"
+    python -m pip install --no-use-pep517 -e ".[all]"
 
 This will run a process of downloading and installing all the needed
-dependencies into a virtual env. If any dependencies fail to install,
+dependencies into a virtual env.
-try installing the failing modules individually::
 
-    pip install -e "module-name"
+Once this is done, you may wish to run Synapse's unit tests, to
-
+check that everything is installed as it should be::
-Once this is done, you may wish to run Synapse's unit tests to
-check that everything is installed correctly::
 
     python -m twisted.trial tests
 
-This should end with a 'PASSED' result (note that exact numbers will
+This should end with a 'PASSED' result::
-differ)::
-
-    Ran 1337 tests in 716.064s
-
-    PASSED (skips=15, successes=1322)
-
-We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`
-
-    ./demo/start.sh
-
-(to stop, you can use `./demo/stop.sh`)
-
-If you just want to start a single instance of the app and run it directly::
-
-    # Create the homeserver.yaml config once
-    python -m synapse.app.homeserver \
-      --server-name my.domain.name \
-      --config-path homeserver.yaml \
-      --generate-config \
-      --report-stats=[yes|no]
-
-    # Start the app
-    python -m synapse.app.homeserver --config-path homeserver.yaml
-
 
+    Ran 143 tests in 0.601s
 
+    PASSED (successes=143)
 
 Running the Integration Tests
 =============================
@@ -314,12 +300,22 @@ Testing with SyTest is recommended for verifying that changes related to the
 Client-Server API are functioning correctly. See the `installation instructions
 <https://github.com/matrix-org/sytest#installing>`_ for details.
 
+Building Internal API Documentation
+===================================
+
+Before building internal API documentation install sphinx and
+sphinxcontrib-napoleon::
+
+    pip install sphinx
+    pip install sphinxcontrib-napoleon
+
+Building internal API documentation::
+
+    python setup.py build_sphinx
+
 Troubleshooting
 ===============
 
-Need help? Join our community support room on Matrix:
-`#synapse:matrix.org <https://matrix.to/#/#synapse:matrix.org>`_
-
 Running out of File Handles
 ---------------------------
 
@@ -398,23 +394,3 @@ something like the following in their logs::
 
 This is normally caused by a misconfiguration in your reverse-proxy. See
 `<docs/reverse_proxy.md>`_ and double-check that your settings are correct.
-
-.. |support| image:: https://img.shields.io/matrix/synapse:matrix.org?label=support&logo=matrix
-  :alt: (get support on #synapse:matrix.org)
-  :target: https://matrix.to/#/#synapse:matrix.org
-
-.. |development| image:: https://img.shields.io/matrix/synapse-dev:matrix.org?label=development&logo=matrix
-  :alt: (discuss development on #synapse-dev:matrix.org)
-  :target: https://matrix.to/#/#synapse-dev:matrix.org
-
-.. |license| image:: https://img.shields.io/github/license/matrix-org/synapse
-  :alt: (check license in LICENSE file)
-  :target: LICENSE
-
-.. |pypi| image:: https://img.shields.io/pypi/v/matrix-synapse
-  :alt: (latest version released on PyPi)
-  :target: https://pypi.org/project/matrix-synapse
-
-.. |python| image:: https://img.shields.io/pypi/pyversions/matrix-synapse
-  :alt: (supported python versions)
-  :target: https://pypi.org/project/matrix-synapse

UPGRADE.rst

@@ -5,16 +5,6 @@ Before upgrading check if any special steps are required to upgrade from the
 version you currently have installed to the current version of Synapse. The extra
 instructions that may be required are listed later in this document.
 
-* Check that your versions of Python and PostgreSQL are still supported.
-
-  Synapse follows upstream lifecycles for `Python`_ and `PostgreSQL`_, and
-  removes support for versions which are no longer maintained.
-
-  The website https://endoflife.date also offers convenient summaries.
-
-  .. _Python: https://devguide.python.org/devcycle/#end-of-life-branches
-  .. _PostgreSQL: https://www.postgresql.org/support/versioning/
-
 * If Synapse was installed using `prebuilt packages
   <INSTALL.md#prebuilt-packages>`_, you will need to follow the normal process
   for upgrading those packages.
@@ -85,464 +75,6 @@ for example:
      wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
      dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
 
-Upgrading to v1.29.0
-====================
-
-Requirement for X-Forwarded-Proto header
-----------------------------------------
-
-When using Synapse with a reverse proxy (in particular, when using the
-`x_forwarded` option on an HTTP listener), Synapse now expects to receive an
-`X-Forwarded-Proto` header on incoming HTTP requests. If it is not set, Synapse
-will log a warning on each received request.
-
-To avoid the warning, administrators using a reverse proxy should ensure that
-the reverse proxy sets `X-Forwarded-Proto` header to `https` or `http` to
-indicate the protocol used by the client. See the `reverse proxy documentation
-<docs/reverse_proxy.md>`_, where the example configurations have been updated to
-show how to set this header.
-
-(Users of `Caddy <https://caddyserver.com/>`_ are unaffected, since we believe it
-sets `X-Forwarded-Proto` by default.)
-
-Upgrading to v1.27.0
-====================
-
-Changes to callback URI for OAuth2 / OpenID Connect and SAML2
--------------------------------------------------------------
-
-This version changes the URI used for callbacks from OAuth2 and SAML2 identity providers:
-
-* If your server is configured for single sign-on via an OpenID Connect or OAuth2 identity
-  provider, you will need to add ``[synapse public baseurl]/_synapse/client/oidc/callback``
-  to the list of permitted "redirect URIs" at the identity provider.
-
-  See `docs/openid.md <docs/openid.md>`_ for more information on setting up OpenID
-  Connect.
-
-* If your server is configured for single sign-on via a SAML2 identity provider, you will
-  need to add ``[synapse public baseurl]/_synapse/client/saml2/authn_response`` as a permitted
-  "ACS location" (also known as "allowed callback URLs") at the identity provider.
-
-  The "Issuer" in the "AuthnRequest" to the SAML2 identity provider is also updated to
-  ``[synapse public baseurl]/_synapse/client/saml2/metadata.xml``. If your SAML2 identity
-  provider uses this property to validate or otherwise identify Synapse, its configuration
-  will need to be updated to use the new URL. Alternatively you could create a new, separate
-  "EntityDescriptor" in your SAML2 identity provider with the new URLs and leave the URLs in
-  the existing "EntityDescriptor" as they were.
-
-Changes to HTML templates
--------------------------
-
-The HTML templates for SSO and email notifications now have `Jinja2's autoescape <https://jinja.palletsprojects.com/en/2.11.x/api/#autoescaping>`_
-enabled for files ending in ``.html``, ``.htm``, and ``.xml``. If you have customised
-these templates and see issues when viewing them you might need to update them.
-It is expected that most configurations will need no changes.
-
-If you have customised the templates *names* for these templates, it is recommended
-to verify they end in ``.html`` to ensure autoescape is enabled.
-
-The above applies to the following templates:
-
-* ``add_threepid.html``
-* ``add_threepid_failure.html``
-* ``add_threepid_success.html``
-* ``notice_expiry.html``
-* ``notice_expiry.html``
-* ``notif_mail.html`` (which, by default, includes ``room.html`` and ``notif.html``)
-* ``password_reset.html``
-* ``password_reset_confirmation.html``
-* ``password_reset_failure.html``
-* ``password_reset_success.html``
-* ``registration.html``
-* ``registration_failure.html``
-* ``registration_success.html``
-* ``sso_account_deactivated.html``
-* ``sso_auth_bad_user.html``
-* ``sso_auth_confirm.html``
-* ``sso_auth_success.html``
-* ``sso_error.html``
-* ``sso_login_idp_picker.html``
-* ``sso_redirect_confirm.html``
-
-Upgrading to v1.26.0
-====================
-
-Rolling back to v1.25.0 after a failed upgrade
-----------------------------------------------
-
-v1.26.0 includes a lot of large changes. If something problematic occurs, you
-may want to roll-back to a previous version of Synapse. Because v1.26.0 also
-includes a new database schema version, reverting that version is also required
-alongside the generic rollback instructions mentioned above. In short, to roll
-back to v1.25.0 you need to:
-
-1. Stop the server
-2. Decrease the schema version in the database:
-
-   .. code:: sql
-
-      UPDATE schema_version SET version = 58;
-
-3. Delete the ignored users & chain cover data:
-
-   .. code:: sql
-
-      DROP TABLE IF EXISTS ignored_users;
-      UPDATE rooms SET has_auth_chain_index = false;
-
-   For PostgreSQL run:
-
-   .. code:: sql
-
-      TRUNCATE event_auth_chain_links;
-      TRUNCATE event_auth_chains;
-
-   For SQLite run:
-
-   .. code:: sql
-
-      DELETE FROM event_auth_chain_links;
-      DELETE FROM event_auth_chains;
-
-4. Mark the deltas as not run (so they will re-run on upgrade).
-
-   .. code:: sql
-
-      DELETE FROM applied_schema_deltas WHERE version = 59 AND file = "59/01ignored_user.py";
-      DELETE FROM applied_schema_deltas WHERE version = 59 AND file = "59/06chain_cover_index.sql";
-
-5. Downgrade Synapse by following the instructions for your installation method
-   in the "Rolling back to older versions" section above.
-
-Upgrading to v1.25.0
-====================
-
-Last release supporting Python 3.5
-----------------------------------
-
-This is the last release of Synapse which guarantees support with Python 3.5,
-which passed its upstream End of Life date several months ago.
-
-We will attempt to maintain support through March 2021, but without guarantees.
-
-In the future, Synapse will follow upstream schedules for ending support of
-older versions of Python and PostgreSQL. Please upgrade to at least Python 3.6
-and PostgreSQL 9.6 as soon as possible.
-
-Blacklisting IP ranges
-----------------------
-
-Synapse v1.25.0 includes new settings, ``ip_range_blacklist`` and
-``ip_range_whitelist``, for controlling outgoing requests from Synapse for federation,
-identity servers, push, and for checking key validity for third-party invite events.
-The previous setting, ``federation_ip_range_blacklist``, is deprecated. The new
-``ip_range_blacklist`` defaults to private IP ranges if it is not defined.
-
-If you have never customised ``federation_ip_range_blacklist`` it is recommended
-that you remove that setting.
-
-If you have customised ``federation_ip_range_blacklist`` you should update the
-setting name to ``ip_range_blacklist``.
-
-If you have a custom push server that is reached via private IP space you may
-need to customise ``ip_range_blacklist`` or ``ip_range_whitelist``.
-
-Upgrading to v1.24.0
-====================
-
-Custom OpenID Connect mapping provider breaking change
-------------------------------------------------------
-
-This release allows the OpenID Connect mapping provider to perform normalisation
-of the localpart of the Matrix ID. This allows for the mapping provider to
-specify different algorithms, instead of the [default way](https://matrix.org/docs/spec/appendices#mapping-from-other-character-sets).
-
-If your Synapse configuration uses a custom mapping provider
-(`oidc_config.user_mapping_provider.module` is specified and not equal to
-`synapse.handlers.oidc_handler.JinjaOidcMappingProvider`) then you *must* ensure
-that `map_user_attributes` of the mapping provider performs some normalisation
-of the `localpart` returned. To match previous behaviour you can use the
-`map_username_to_mxid_localpart` function provided by Synapse. An example is
-shown below:
-
-.. code-block:: python
-
-  from synapse.types import map_username_to_mxid_localpart
-
-  class MyMappingProvider:
-      def map_user_attributes(self, userinfo, token):
-          # ... your custom logic ...
-          sso_user_id = ...
-          localpart = map_username_to_mxid_localpart(sso_user_id)
-
-          return {"localpart": localpart}
-
-Removal historical Synapse Admin API
-------------------------------------
-
-Historically, the Synapse Admin API has been accessible under:
-
-* ``/_matrix/client/api/v1/admin``
-* ``/_matrix/client/unstable/admin``
-* ``/_matrix/client/r0/admin``
-* ``/_synapse/admin/v1``
-
-The endpoints with ``/_matrix/client/*`` prefixes have been removed as of v1.24.0.
-The Admin API is now only accessible under:
-
-* ``/_synapse/admin/v1``
-
-The only exception is the `/admin/whois` endpoint, which is
-`also available via the client-server API <https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-admin-whois-userid>`_.
-
-The deprecation of the old endpoints was announced with Synapse 1.20.0 (released
-on 2020-09-22) and makes it easier for homeserver admins to lock down external
-access to the Admin API endpoints.
-
-Upgrading to v1.23.0
-====================
-
-Structured logging configuration breaking changes
--------------------------------------------------
-
-This release deprecates use of the ``structured: true`` logging configuration for
-structured logging. If your logging configuration contains ``structured: true``
-then it should be modified based on the `structured logging documentation
-<https://github.com/matrix-org/synapse/blob/master/docs/structured_logging.md>`_.
-
-The ``structured`` and ``drains`` logging options are now deprecated and should
-be replaced by standard logging configuration of ``handlers`` and ``formatters``.
-
-A future will release of Synapse will make using ``structured: true`` an error.
-
-Upgrading to v1.22.0
-====================
-
-ThirdPartyEventRules breaking changes
--------------------------------------
-
-This release introduces a backwards-incompatible change to modules making use of
-``ThirdPartyEventRules`` in Synapse. If you make use of a module defined under the
-``third_party_event_rules`` config option, please make sure it is updated to handle
-the below change:
-
-The ``http_client`` argument is no longer passed to modules as they are initialised. Instead,
-modules are expected to make use of the ``http_client`` property on the ``ModuleApi`` class.
-Modules are now passed a ``module_api`` argument during initialisation, which is an instance of
-``ModuleApi``. ``ModuleApi`` instances have a ``http_client`` property which acts the same as
-the ``http_client`` argument previously passed to ``ThirdPartyEventRules`` modules.
-
-Upgrading to v1.21.0
-====================
-
-Forwarding ``/_synapse/client`` through your reverse proxy
-----------------------------------------------------------
-
-The `reverse proxy documentation
-<https://github.com/matrix-org/synapse/blob/develop/docs/reverse_proxy.md>`_ has been updated
-to include reverse proxy directives for ``/_synapse/client/*`` endpoints. As the user password
-reset flow now uses endpoints under this prefix, **you must update your reverse proxy
-configurations for user password reset to work**.
-
-Additionally, note that the `Synapse worker documentation
-<https://github.com/matrix-org/synapse/blob/develop/docs/workers.md>`_ has been updated to
- state that the ``/_synapse/client/password_reset/email/submit_token`` endpoint can be handled
-by all workers. If you make use of Synapse's worker feature, please update your reverse proxy
-configuration to reflect this change.
-
-New HTML templates
-------------------
-
-A new HTML template,
-`password_reset_confirmation.html <https://github.com/matrix-org/synapse/blob/develop/synapse/res/templates/password_reset_confirmation.html>`_,
-has been added to the ``synapse/res/templates`` directory. If you are using a
-custom template directory, you may want to copy the template over and modify it.
-
-Note that as of v1.20.0, templates do not need to be included in custom template
-directories for Synapse to start. The default templates will be used if a custom
-template cannot be found.
-
-This page will appear to the user after clicking a password reset link that has
-been emailed to them.
-
-To complete password reset, the page must include a way to make a `POST`
-request to
-``/_synapse/client/password_reset/{medium}/submit_token``
-with the query parameters from the original link, presented as a URL-encoded form. See the file
-itself for more details.
-
-Updated Single Sign-on HTML Templates
--------------------------------------
-
-The ``saml_error.html`` template was removed from Synapse and replaced with the
-``sso_error.html`` template. If your Synapse is configured to use SAML and a
-custom ``sso_redirect_confirm_template_dir`` configuration then any customisations
-of the ``saml_error.html`` template will need to be merged into the ``sso_error.html``
-template. These templates are similar, but the parameters are slightly different:
-
-* The ``msg`` parameter should be renamed to ``error_description``.
-* There is no longer a ``code`` parameter for the response code.
-* A string ``error`` parameter is available that includes a short hint of why a
-  user is seeing the error page.
-
-Upgrading to v1.18.0
-====================
-
-Docker `-py3` suffix will be removed in future versions
--------------------------------------------------------
-
-From 10th August 2020, we will no longer publish Docker images with the `-py3` tag suffix. The images tagged with the `-py3` suffix have been identical to the non-suffixed tags since release 0.99.0, and the suffix is obsolete.
-
-On 10th August, we will remove the `latest-py3` tag. Existing per-release tags (such as `v1.18.0-py3`) will not be removed, but no new `-py3` tags will be added.
-
-Scripts relying on the `-py3` suffix will need to be updated.
-
-Redis replication is now recommended in lieu of TCP replication
----------------------------------------------------------------
-
-When setting up worker processes, we now recommend the use of a Redis server for replication. **The old direct TCP connection method is deprecated and will be removed in a future release.**
-See `docs/workers.md <docs/workers.md>`_ for more details.
-
-Upgrading to v1.14.0
-====================
-
-This version includes a database update which is run as part of the upgrade,
-and which may take a couple of minutes in the case of a large server. Synapse
-will not respond to HTTP requests while this update is taking place.
-
-Upgrading to v1.13.0
-====================
-
-Incorrect database migration in old synapse versions
-----------------------------------------------------
-
-A bug was introduced in Synapse 1.4.0 which could cause the room directory to
-be incomplete or empty if Synapse was upgraded directly from v1.2.1 or
-earlier, to versions between v1.4.0 and v1.12.x.
-
-This will *not* be a problem for Synapse installations which were:
- * created at v1.4.0 or later,
- * upgraded via v1.3.x, or
- * upgraded straight from v1.2.1 or earlier to v1.13.0 or later.
-
-If completeness of the room directory is a concern, installations which are
-affected can be repaired as follows:
-
-1. Run the following sql from a `psql` or `sqlite3` console:
-
-   .. code:: sql
-
-     INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
-        ('populate_stats_process_rooms', '{}', 'current_state_events_membership');
-
-     INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
-        ('populate_stats_process_users', '{}', 'populate_stats_process_rooms');
-
-2. Restart synapse.
-
-New Single Sign-on HTML Templates
----------------------------------
-
-New templates (``sso_auth_confirm.html``, ``sso_auth_success.html``, and
-``sso_account_deactivated.html``) were added to Synapse. If your Synapse is
-configured to use SSO and a custom  ``sso_redirect_confirm_template_dir``
-configuration then these templates will need to be copied from
-`synapse/res/templates <synapse/res/templates>`_ into that directory.
-
-Synapse SSO Plugins Method Deprecation
---------------------------------------
-
-Plugins using the ``complete_sso_login`` method of
-``synapse.module_api.ModuleApi`` should update to using the async/await
-version ``complete_sso_login_async`` which includes additional checks. The
-non-async version is considered deprecated.
-
-Rolling back to v1.12.4 after a failed upgrade
-----------------------------------------------
-
-v1.13.0 includes a lot of large changes. If something problematic occurs, you
-may want to roll-back to a previous version of Synapse. Because v1.13.0 also
-includes a new database schema version, reverting that version is also required
-alongside the generic rollback instructions mentioned above. In short, to roll
-back to v1.12.4 you need to:
-
-1. Stop the server
-2. Decrease the schema version in the database:
-
-   .. code:: sql
-
-      UPDATE schema_version SET version = 57;
-
-3. Downgrade Synapse by following the instructions for your installation method
-   in the "Rolling back to older versions" section above.
-
-
-Upgrading to v1.12.0
-====================
-
-This version includes a database update which is run as part of the upgrade,
-and which may take some time (several hours in the case of a large
-server). Synapse will not respond to HTTP requests while this update is taking
-place.
-
-This is only likely to be a problem in the case of a server which is
-participating in many rooms.
-
-0. As with all upgrades, it is recommended that you have a recent backup of
-   your database which can be used for recovery in the event of any problems.
-
-1. As an initial check to see if you will be affected, you can try running the
-   following query from the `psql` or `sqlite3` console. It is safe to run it
-   while Synapse is still running.
-
-   .. code:: sql
-
-      SELECT MAX(q.v) FROM (
-        SELECT (
-          SELECT ej.json AS v
-          FROM state_events se INNER JOIN event_json ej USING (event_id)
-          WHERE se.room_id=rooms.room_id AND se.type='m.room.create' AND se.state_key=''
-          LIMIT 1
-        ) FROM rooms WHERE rooms.room_version IS NULL
-      ) q;
-
-   This query will take about the same amount of time as the upgrade process: ie,
-   if it takes 5 minutes, then it is likely that Synapse will be unresponsive for
-   5 minutes during the upgrade.
-
-   If you consider an outage of this duration to be acceptable, no further
-   action is necessary and you can simply start Synapse 1.12.0.
-
-   If you would prefer to reduce the downtime, continue with the steps below.
-
-2. The easiest workaround for this issue is to manually
-   create a new index before upgrading. On PostgreSQL, his can be done as follows:
-
-   .. code:: sql
-
-      CREATE INDEX CONCURRENTLY tmp_upgrade_1_12_0_index
-      ON state_events(room_id) WHERE type = 'm.room.create';
-
-   The above query may take some time, but is also safe to run while Synapse is
-   running.
-
-   We assume that no SQLite users have databases large enough to be
-   affected. If you *are* affected, you can run a similar query, omitting the
-   ``CONCURRENTLY`` keyword. Note however that this operation may in itself cause
-   Synapse to stop running for some time. Synapse admins are reminded that
-   `SQLite is not recommended for use outside a test
-   environment <https://github.com/matrix-org/synapse/blob/master/README.rst#using-postgresql>`_.
-
-3. Once the index has been created, the ``SELECT`` query in step 1 above should
-   complete quickly. It is therefore safe to upgrade to Synapse 1.12.0.
-
-4. Once Synapse 1.12.0 has successfully started and is responding to HTTP
-   requests, the temporary index can be removed:
-
-   .. code:: sql
-
-      DROP INDEX tmp_upgrade_1_12_0_index;
 
 Upgrading to v1.10.0
 ====================

contrib/cmdclient/console.py

@@ -15,6 +15,11 @@
 # limitations under the License.
 
 """ Starts a synapse client console. """
+from __future__ import print_function
+
+from twisted.internet import reactor, defer, threads
+from http import TwistedHttpClient
+
 import argparse
 import cmd
 import getpass
@@ -23,14 +28,12 @@ import shlex
 import sys
 import time
 import urllib
-from http import TwistedHttpClient
-
-import nacl.encoding
-import nacl.signing
 import urlparse
-from signedjson.sign import SignatureVerifyException, verify_signed_json
 
-from twisted.internet import defer, reactor, threads
+import nacl.signing
+import nacl.encoding
+
+from signedjson.sign import verify_signed_json, SignatureVerifyException
 
 CONFIG_JSON = "cmdclient_config.json"
 
@@ -92,7 +95,7 @@ class SynapseCmd(cmd.Cmd):
         return self.config["user"].split(":")[1]
 
     def do_config(self, line):
-        """Show the config for this client: "config"
+        """ Show the config for this client: "config"
         Edit a key value mapping: "config key value" e.g. "config token 1234"
         Config variables:
             user: The username to auth with.
@@ -360,7 +363,7 @@ class SynapseCmd(cmd.Cmd):
             print(e)
 
     def do_topic(self, line):
-        """ "topic [set|get] <roomid> [<newtopic>]"
+        """"topic [set|get] <roomid> [<newtopic>]"
         Set the topic for a room: topic set <roomid> <newtopic>
         Get the topic for a room: topic get <roomid>
         """
@@ -490,7 +493,7 @@ class SynapseCmd(cmd.Cmd):
         "list messages <roomid> from=END&to=START&limit=3"
         """
         args = self._parse(line, ["type", "roomid", "qp"])
-        if "type" not in args or "roomid" not in args:
+        if not "type" in args or not "roomid" in args:
             print("Must specify type and room ID.")
             return
         if args["type"] not in ["members", "messages"]:
@@ -505,7 +508,7 @@ class SynapseCmd(cmd.Cmd):
                 try:
                     key_value = key_value_str.split("=")
                     qp[key_value[0]] = key_value[1]
-                except Exception:
+                except:
                     print("Bad query param: %s" % key_value)
                     return
 
@@ -582,7 +585,7 @@ class SynapseCmd(cmd.Cmd):
                 parsed_url = urlparse.urlparse(args["path"])
                 qp.update(urlparse.parse_qs(parsed_url.query))
                 args["path"] = parsed_url.path
-            except Exception:
+            except:
                 pass
 
         reactor.callFromThread(
@@ -607,15 +610,13 @@ class SynapseCmd(cmd.Cmd):
 
     @defer.inlineCallbacks
     def _do_event_stream(self, timeout):
-        res = yield defer.ensureDeferred(
+        res = yield self.http_client.get_json(
-            self.http_client.get_json(
+            self._url() + "/events",
-                self._url() + "/events",
+            {
-                {
+                "access_token": self._tok(),
-                    "access_token": self._tok(),
+                "timeout": str(timeout),
-                    "timeout": str(timeout),
+                "from": self.event_stream_token,
-                    "from": self.event_stream_token,
+            },
-                },
-            )
         )
         print(json.dumps(res, indent=4))
 
@@ -690,7 +691,7 @@ class SynapseCmd(cmd.Cmd):
         self._do_presence_state(2, line)
 
     def _parse(self, line, keys, force_keys=False):
-        """Parses the given line.
+        """ Parses the given line.
 
         Args:
             line : The line to parse
@@ -721,7 +722,7 @@ class SynapseCmd(cmd.Cmd):
         query_params={"access_token": None},
         alt_text=None,
     ):
-        """Runs an HTTP request and pretty prints the output.
+        """ Runs an HTTP request and pretty prints the output.
 
         Args:
             method: HTTP method
@@ -771,10 +772,10 @@ def main(server_url, identity_server_url, username, token, config_path):
             syn_cmd.config = json.load(config)
             try:
                 http_client.verbose = "on" == syn_cmd.config["verbose"]
-            except Exception:
+            except:
                 pass
             print("Loaded config from %s" % config_path)
-    except Exception:
+    except:
         pass
 
     # Twisted-specific: Runs the command processor in Twisted's event loop

contrib/cmdclient/http.py

@@ -13,20 +13,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
+from __future__ import print_function
-import urllib
-from pprint import pformat
-
-from twisted.internet import defer, reactor
 from twisted.web.client import Agent, readBody
 from twisted.web.http_headers import Headers
+from twisted.internet import defer, reactor
+
+from pprint import pformat
+
+import json
+import urllib
 
 
-class HttpClient:
+class HttpClient(object):
-    """Interface for talking json over http"""
+    """ Interface for talking json over http
+    """
 
     def put_json(self, url, data):
-        """Sends the specifed json data using PUT
+        """ Sends the specifed json data using PUT
 
         Args:
             url (str): The URL to PUT data to.
@@ -40,7 +43,7 @@ class HttpClient:
         pass
 
     def get_json(self, url, args=None):
-        """Gets some json from the given host homeserver and path
+        """ Gets some json from the given host homeserver and path
 
         Args:
             url (str): The URL to GET data from.
@@ -57,7 +60,7 @@ class HttpClient:
 
 
 class TwistedHttpClient(HttpClient):
-    """Wrapper around the twisted HTTP client api.
+    """ Wrapper around the twisted HTTP client api.
 
     Attributes:
         agent (twisted.web.client.Agent): The twisted Agent used to send the
@@ -86,7 +89,8 @@ class TwistedHttpClient(HttpClient):
         defer.returnValue(json.loads(body))
 
     def _create_put_request(self, url, json_data, headers_dict={}):
-        """Wrapper of _create_request to issue a PUT request"""
+        """ Wrapper of _create_request to issue a PUT request
+        """
 
         if "Content-Type" not in headers_dict:
             raise defer.error(RuntimeError("Must include Content-Type header for PUTs"))
@@ -96,7 +100,8 @@ class TwistedHttpClient(HttpClient):
         )
 
     def _create_get_request(self, url, headers_dict={}):
-        """Wrapper of _create_request to issue a GET request"""
+        """ Wrapper of _create_request to issue a GET request
+        """
         return self._create_request("GET", url, headers_dict=headers_dict)
 
     @defer.inlineCallbacks
@@ -124,7 +129,8 @@ class TwistedHttpClient(HttpClient):
 
     @defer.inlineCallbacks
     def _create_request(self, method, url, producer=None, headers_dict={}):
-        """Creates and sends a request to the given url"""
+        """ Creates and sends a request to the given url
+        """
         headers_dict["User-Agent"] = ["Synapse Cmd Client"]
 
         retries_left = 5
@@ -163,7 +169,7 @@ class TwistedHttpClient(HttpClient):
         return d
 
 
-class _RawProducer:
+class _RawProducer(object):
     def __init__(self, data):
         self.data = data
         self.body = data
@@ -180,8 +186,9 @@ class _RawProducer:
         pass
 
 
-class _JsonProducer:
+class _JsonProducer(object):
-    """Used by the twisted http client to create the HTTP body from json"""
+    """ Used by the twisted http client to create the HTTP body from json
+    """
 
     def __init__(self, jsn):
         self.data = jsn

contrib/docker/docker-compose.yml

@@ -50,7 +50,7 @@ services:
       - traefik.http.routers.https-synapse.tls.certResolver=le-ssl
 
   db:
-    image: docker.io/postgres:12-alpine
+    image: docker.io/postgres:10-alpine
     # Change that password, of course!
     environment:
       - POSTGRES_USER=synapse

contrib/experiments/cursesio.py

@@ -63,7 +63,8 @@ class CursesStdIO:
         self.redraw()
 
     def redraw(self):
-        """method for redisplaying lines based on internal list of lines"""
+        """ method for redisplaying lines
+            based on internal list of lines """
 
         self.stdscr.clear()
         self.paintStatus(self.statusText)
@@ -140,7 +141,7 @@ class CursesStdIO:
         curses.endwin()
 
 
-class Callback:
+class Callback(object):
     def __init__(self, stdio):
         self.stdio = stdio
 

contrib/experiments/test_messaging.py

@@ -28,24 +28,27 @@ Currently assumes the local address is localhost:<port>
 """
 
 
+from synapse.federation import ReplicationHandler
+
+from synapse.federation.units import Pdu
+
+from synapse.util import origin_from_ucid
+
+from synapse.app.homeserver import SynapseHomeServer
+
+# from synapse.logging.utils import log_function
+
+from twisted.internet import reactor, defer
+from twisted.python import log
+
 import argparse
-import curses.wrapper
 import json
 import logging
 import os
 import re
 
 import cursesio
-
+import curses.wrapper
-from twisted.internet import defer, reactor
-from twisted.python import log
-
-from synapse.app.homeserver import SynapseHomeServer
-from synapse.federation import ReplicationHandler
-from synapse.federation.units import Pdu
-from synapse.util import origin_from_ucid
-
-# from synapse.logging.utils import log_function
 
 
 logger = logging.getLogger("example")
@@ -55,8 +58,8 @@ def excpetion_errback(failure):
     logging.exception(failure)
 
 
-class InputOutput:
+class InputOutput(object):
-    """This is responsible for basic I/O so that a user can interact with
+    """ This is responsible for basic I/O so that a user can interact with
     the example app.
     """
 
@@ -68,10 +71,11 @@ class InputOutput:
         self.server = server
 
     def on_line(self, line):
-        """This is where we process commands."""
+        """ This is where we process commands.
+        """
 
         try:
-            m = re.match(r"^join (\S+)$", line)
+            m = re.match("^join (\S+)$", line)
             if m:
                 # The `sender` wants to join a room.
                 (room_name,) = m.groups()
@@ -80,7 +84,7 @@ class InputOutput:
                 # self.print_line("OK.")
                 return
 
-            m = re.match(r"^invite (\S+) (\S+)$", line)
+            m = re.match("^invite (\S+) (\S+)$", line)
             if m:
                 # `sender` wants to invite someone to a room
                 room_name, invitee = m.groups()
@@ -89,7 +93,7 @@ class InputOutput:
                 # self.print_line("OK.")
                 return
 
-            m = re.match(r"^send (\S+) (.*)$", line)
+            m = re.match("^send (\S+) (.*)$", line)
             if m:
                 # `sender` wants to message a room
                 room_name, body = m.groups()
@@ -98,7 +102,7 @@ class InputOutput:
                 # self.print_line("OK.")
                 return
 
-            m = re.match(r"^backfill (\S+)$", line)
+            m = re.match("^backfill (\S+)$", line)
             if m:
                 # we want to backfill a room
                 (room_name,) = m.groups()
@@ -131,8 +135,8 @@ class IOLoggerHandler(logging.Handler):
         self.io.print_log(msg)
 
 
-class Room:
+class Room(object):
-    """Used to store (in memory) the current membership state of a room, and
+    """ Used to store (in memory) the current membership state of a room, and
     which home servers we should send PDUs associated with the room to.
     """
 
@@ -147,7 +151,8 @@ class Room:
         self.have_got_metadata = False
 
     def add_participant(self, participant):
-        """Someone has joined the room"""
+        """ Someone has joined the room
+        """
         self.participants.add(participant)
         self.invited.discard(participant)
 
@@ -158,13 +163,14 @@ class Room:
             self.oldest_server = server
 
     def add_invited(self, invitee):
-        """Someone has been invited to the room"""
+        """ Someone has been invited to the room
+        """
         self.invited.add(invitee)
         self.servers.add(origin_from_ucid(invitee))
 
 
 class HomeServer(ReplicationHandler):
-    """A very basic home server implentation that allows people to join a
+    """ A very basic home server implentation that allows people to join a
     room and then invite other people.
     """
 
@@ -178,7 +184,8 @@ class HomeServer(ReplicationHandler):
         self.output = output
 
     def on_receive_pdu(self, pdu):
-        """We just received a PDU"""
+        """ We just received a PDU
+        """
         pdu_type = pdu.pdu_type
 
         if pdu_type == "sy.room.message":
@@ -194,21 +201,34 @@ class HomeServer(ReplicationHandler):
                 % (pdu.context, pdu.pdu_type, json.dumps(pdu.content))
             )
 
+    # def on_state_change(self, pdu):
+    ##self.output.print_line("#%s (state) %s *** %s" %
+    ##(pdu.context, pdu.state_key, pdu.pdu_type)
+    ##)
+
+    # if "joinee" in pdu.content:
+    # self._on_join(pdu.context, pdu.content["joinee"])
+    # elif "invitee" in pdu.content:
+    # self._on_invite(pdu.origin, pdu.context, pdu.content["invitee"])
+
     def _on_message(self, pdu):
-        """We received a message"""
+        """ We received a message
+        """
         self.output.print_line(
             "#%s %s %s" % (pdu.context, pdu.content["sender"], pdu.content["body"])
         )
 
     def _on_join(self, context, joinee):
-        """Someone has joined a room, either a remote user or a local user"""
+        """ Someone has joined a room, either a remote user or a local user
+        """
         room = self._get_or_create_room(context)
         room.add_participant(joinee)
 
         self.output.print_line("#%s %s %s" % (context, joinee, "*** JOINED"))
 
     def _on_invite(self, origin, context, invitee):
-        """Someone has been invited"""
+        """ Someone has been invited
+        """
         room = self._get_or_create_room(context)
         room.add_invited(invitee)
 
@@ -221,7 +241,8 @@ class HomeServer(ReplicationHandler):
 
     @defer.inlineCallbacks
     def send_message(self, room_name, sender, body):
-        """Send a message to a room!"""
+        """ Send a message to a room!
+        """
         destinations = yield self.get_servers_for_context(room_name)
 
         try:
@@ -239,7 +260,8 @@ class HomeServer(ReplicationHandler):
 
     @defer.inlineCallbacks
     def join_room(self, room_name, sender, joinee):
-        """Join a room!"""
+        """ Join a room!
+        """
         self._on_join(room_name, joinee)
 
         destinations = yield self.get_servers_for_context(room_name)
@@ -260,7 +282,8 @@ class HomeServer(ReplicationHandler):
 
     @defer.inlineCallbacks
     def invite_to_room(self, room_name, sender, invitee):
-        """Invite someone to a room!"""
+        """ Invite someone to a room!
+        """
         self._on_invite(self.server_name, room_name, invitee)
 
         destinations = yield self.get_servers_for_context(room_name)
@@ -291,7 +314,7 @@ class HomeServer(ReplicationHandler):
         return self.replication_layer.backfill(dest, room_name, limit)
 
     def _get_room_remote_servers(self, room_name):
-        return list(self.joined_rooms.setdefault(room_name).servers)
+        return [i for i in self.joined_rooms.setdefault(room_name).servers]
 
     def _get_or_create_room(self, room_name):
         return self.joined_rooms.setdefault(room_name, Room(room_name))
@@ -311,7 +334,7 @@ def main(stdscr):
     user = args.user
     server_name = origin_from_ucid(user)
 
-    # Set up logging
+    ## Set up logging ##
 
     root_logger = logging.getLogger()
 
@@ -331,7 +354,7 @@ def main(stdscr):
     observer = log.PythonLoggingObserver()
     observer.start()
 
-    # Set up synapse server
+    ## Set up synapse server
 
     curses_stdio = cursesio.CursesStdIO(stdscr)
     input_output = InputOutput(curses_stdio, user)
@@ -345,16 +368,16 @@ def main(stdscr):
 
     input_output.set_home_server(hs)
 
-    # Add input_output logger
+    ## Add input_output logger
     io_logger = IOLoggerHandler(input_output)
     io_logger.setFormatter(formatter)
     root_logger.addHandler(io_logger)
 
-    # Start!
+    ## Start! ##
 
     try:
         port = int(server_name.split(":")[1])
-    except Exception:
+    except:
         port = 12345
 
     app_hs.get_http_server().start_listening(port)

contrib/grafana/README.md

@@ -3,4 +3,4 @@
 0. Set up Prometheus and Grafana. Out of scope for this readme. Useful documentation about using Grafana with Prometheus: http://docs.grafana.org/features/datasources/prometheus/
 1. Have your Prometheus scrape your Synapse. https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md
 2. Import dashboard into Grafana. Download `synapse.json`. Import it to Grafana and select the correct Prometheus datasource. http://docs.grafana.org/reference/export_import/
-3. Set up required recording rules. https://github.com/matrix-org/synapse/tree/master/contrib/prometheus
+3. Set up additional recording rules

contrib/graph/graph.py

@@ -1,10 +1,4 @@
-import argparse
+from __future__ import print_function
-import cgi
-import datetime
-import json
-
-import pydot
-import urllib2
 
 # Copyright 2014-2016 OpenMarket Ltd
 #
@@ -21,6 +15,15 @@ import urllib2
 # limitations under the License.
 
 
+import sqlite3
+import pydot
+import cgi
+import json
+import datetime
+import argparse
+import urllib2
+
+
 def make_name(pdu_id, origin):
     return "%s@%s" % (pdu_id, origin)
 
@@ -30,7 +33,7 @@ def make_graph(pdus, room, filename_prefix):
     node_map = {}
 
     origins = set()
-    colors = {"red", "green", "blue", "yellow", "purple"}
+    colors = set(("red", "green", "blue", "yellow", "purple"))
 
     for pdu in pdus:
         origins.add(pdu.get("origin"))
@@ -46,7 +49,7 @@ def make_graph(pdus, room, filename_prefix):
         try:
             c = colors.pop()
             color_map[o] = c
-        except Exception:
+        except:
             print("Run out of colours!")
             color_map[o] = "black"
 

contrib/graph/graph2.py

@@ -13,13 +13,12 @@
 # limitations under the License.
 
 
-import argparse
-import cgi
-import datetime
-import json
 import sqlite3
-
 import pydot
+import cgi
+import json
+import datetime
+import argparse
 
 from synapse.events import FrozenEvent
 from synapse.util.frozenutils import unfreeze
@@ -99,7 +98,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
         for prev_id, _ in event.prev_events:
             try:
                 end_node = node_map[prev_id]
-            except Exception:
+            except:
                 end_node = pydot.Node(name=prev_id, label="<<b>%s</b>>" % (prev_id,))
 
                 node_map[prev_id] = end_node

contrib/graph/graph3.py

@@ -1,12 +1,4 @@
-import argparse
+from __future__ import print_function
-import cgi
-import datetime
-
-import pydot
-import simplejson as json
-
-from synapse.events import FrozenEvent
-from synapse.util.frozenutils import unfreeze
 
 # Copyright 2016 OpenMarket Ltd
 #
@@ -23,6 +15,18 @@ from synapse.util.frozenutils import unfreeze
 # limitations under the License.
 
 
+import pydot
+import cgi
+import simplejson as json
+import datetime
+import argparse
+
+from synapse.events import FrozenEvent
+from synapse.util.frozenutils import unfreeze
+
+from six import string_types
+
+
 def make_graph(file_name, room_id, file_prefix, limit):
     print("Reading lines")
     with open(file_name) as f:
@@ -58,7 +62,7 @@ def make_graph(file_name, room_id, file_prefix, limit):
         for key, value in unfreeze(event.get_dict()["content"]).items():
             if value is None:
                 value = "<null>"
-            elif isinstance(value, str):
+            elif isinstance(value, string_types):
                 pass
             else:
                 value = json.dumps(value)
@@ -104,7 +108,7 @@ def make_graph(file_name, room_id, file_prefix, limit):
         for prev_id, _ in event.prev_events:
             try:
                 end_node = node_map[prev_id]
-            except Exception:
+            except:
                 end_node = pydot.Node(name=prev_id, label="<<b>%s</b>>" % (prev_id,))
 
                 node_map[prev_id] = end_node

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -10,15 +10,17 @@ the bridge.
 Requires:
 npm install jquery jsdom
 """
-import json
+from __future__ import print_function
-import subprocess
-import time
 
 import gevent
 import grequests
 from BeautifulSoup import BeautifulSoup
+import json
+import urllib
+import subprocess
+import time
 
-ACCESS_TOKEN = ""
+# ACCESS_TOKEN="" #
 
 MATRIXBASE = "https://matrix.org/_matrix/client/api/v1/"
 MYUSERNAME = "@davetest:matrix.org"
@@ -193,12 +195,15 @@ class TrivialXmppClient:
         time.sleep(7)
         print("SSRC spammer started")
         while self.running:
-            ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % {
+            ssrcMsg = (
-                "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
+                "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>"
-                "nick": self.userId,
+                % {
-                "assrc": self.ssrcs["audio"],
+                    "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
-                "vssrc": self.ssrcs["video"],
+                    "nick": self.userId,
-            }
+                    "assrc": self.ssrcs["audio"],
+                    "vssrc": self.ssrcs["video"],
+                }
+            )
             res = self.sendIq(ssrcMsg)
             print("reply from ssrc announce: ", res)
             time.sleep(10)

contrib/prometheus/README.md

@@ -20,7 +20,6 @@ Add a new job to the main prometheus.conf file:
 ```
 
 ### for Prometheus v2
-
 Add a new job to the main prometheus.yml file:
 
 ```yaml
@@ -30,17 +29,14 @@ Add a new job to the main prometheus.yml file:
     scheme: "https"
 
     static_configs:
-    - targets: ["my.server.here:port"]
+    - targets: ['SERVER.LOCATION:PORT']
 ```
 
-An example of a Prometheus configuration with workers can be found in
-[metrics-howto.md](https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md).
-
 To use `synapse.rules` add
 
 ```yaml
-  rule_files:
+    rule_files:
-    - "/PATH/TO/synapse-v2.rules"
+      - "/PATH/TO/synapse-v2.rules"
 ```
 
 Metrics are disabled by default when running synapse; they must be enabled

contrib/prometheus/consoles/synapse.html

@@ -9,7 +9,7 @@
 new PromConsole.Graph({
   node: document.querySelector("#process_resource_utime"),
   expr: "rate(process_cpu_seconds_total[2m]) * 100",
-  name: "[[job]]-[[index]]",
+  name: "[[job]]",  
   min: 0,
   max: 100,
   renderer: "line",
@@ -22,12 +22,12 @@ new PromConsole.Graph({
 </script>
 
 <h3>Memory</h3>
-<div id="process_resident_memory_bytes"></div>
+<div id="process_resource_maxrss"></div>
 <script>
 new PromConsole.Graph({
-  node: document.querySelector("#process_resident_memory_bytes"),
+  node: document.querySelector("#process_resource_maxrss"),
-  expr: "process_resident_memory_bytes",
+  expr: "process_psutil_rss:max",
-  name: "[[job]]-[[index]]",
+  name: "Maxrss",
   min: 0,
   renderer: "line",
   height: 150,
@@ -43,8 +43,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#process_fds"),
-  expr: "process_open_fds",
+  expr: "process_open_fds{job='synapse'}",
-  name: "[[job]]-[[index]]",
+  name: "FDs",
   min: 0,
   renderer: "line",
   height: 150,
@@ -62,8 +62,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#reactor_total_time"),
-  expr: "rate(python_twisted_reactor_tick_time_sum[2m])",
+  expr: "rate(python_twisted_reactor_tick_time:total[2m]) / 1000",
-  name: "[[job]]-[[index]]",
+  name: "time",
   max: 1,
   min: 0,
   renderer: "area",
@@ -80,8 +80,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#reactor_average_time"),
-  expr: "rate(python_twisted_reactor_tick_time_sum[2m]) / rate(python_twisted_reactor_tick_time_count[2m])",
+  expr: "rate(python_twisted_reactor_tick_time:total[2m]) / rate(python_twisted_reactor_tick_time:count[2m]) / 1000",
-  name: "[[job]]-[[index]]",
+  name: "time",
   min: 0,
   renderer: "line",
   height: 150,
@@ -97,14 +97,14 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#reactor_pending_calls"),
-  expr: "rate(python_twisted_reactor_pending_calls_sum[30s]) / rate(python_twisted_reactor_pending_calls_count[30s])",
+  expr: "rate(python_twisted_reactor_pending_calls:total[30s])/rate(python_twisted_reactor_pending_calls:count[30s])",
-  name: "[[job]]-[[index]]",
+  name: "calls",
   min: 0,
   renderer: "line",
   height: 150,
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
-  yTitle: "Pending Calls"
+  yTitle: "Pending Cals"
 })
 </script>
 
@@ -115,7 +115,7 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_storage_query_time"),
-  expr: "sum(rate(synapse_storage_query_time_count[2m])) by (verb)",
+  expr: "rate(synapse_storage_query_time:count[2m])",
   name: "[[verb]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
@@ -129,8 +129,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_storage_transaction_time"),
-  expr: "topk(10, rate(synapse_storage_transaction_time_count[2m]))",
+  expr: "rate(synapse_storage_transaction_time:count[2m])",
-  name: "[[job]]-[[index]] [[desc]]",
+  name: "[[desc]]",
   min: 0,
   yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
@@ -140,12 +140,12 @@ new PromConsole.Graph({
 </script>
 
 <h3>Transaction execution time</h3>
-<div id="synapse_storage_transactions_time_sec"></div>
+<div id="synapse_storage_transactions_time_msec"></div>
 <script>
 new PromConsole.Graph({
-  node: document.querySelector("#synapse_storage_transactions_time_sec"),
+  node: document.querySelector("#synapse_storage_transactions_time_msec"),
-  expr: "rate(synapse_storage_transaction_time_sum[2m])",
+  expr: "rate(synapse_storage_transaction_time:total[2m]) / 1000",
-  name: "[[job]]-[[index]] [[desc]]",
+  name: "[[desc]]",
   min: 0,
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
@@ -154,33 +154,34 @@ new PromConsole.Graph({
 })
 </script>
 
-<h3>Average time waiting for database connection</h3>
+<h3>Database scheduling latency</h3>
-<div id="synapse_storage_avg_waiting_time"></div>
+<div id="synapse_storage_schedule_time"></div>
 <script>
 new PromConsole.Graph({
-  node: document.querySelector("#synapse_storage_avg_waiting_time"),
+  node: document.querySelector("#synapse_storage_schedule_time"),
-  expr: "rate(synapse_storage_schedule_time_sum[2m]) / rate(synapse_storage_schedule_time_count[2m])",
+  expr: "rate(synapse_storage_schedule_time:total[2m]) / 1000",
-  name: "[[job]]-[[index]]",
+  name: "Total latency",
   min: 0,
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
-  yUnits: "s",
+  yUnits: "s/s",
-  yTitle: "Time"
+  yTitle: "Usage"
 })
 </script>
 
-<h3>Cache request rate</h3>
+<h3>Cache hit ratio</h3>
-<div id="synapse_cache_request_rate"></div>
+<div id="synapse_cache_ratio"></div>
 <script>
 new PromConsole.Graph({
-  node: document.querySelector("#synapse_cache_request_rate"),
+  node: document.querySelector("#synapse_cache_ratio"),
-  expr: "rate(synapse_util_caches_cache:total[2m])",
+  expr: "rate(synapse_util_caches_cache:total[2m]) * 100",
-  name: "[[job]]-[[index]] [[name]]",
+  name: "[[name]]",
   min: 0,
+  max: 100,
   yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-  yUnits: "rps",
+  yUnits: "%",
-  yTitle: "Cache request rate"
+  yTitle: "Percentage"
 })
 </script>
 
@@ -190,7 +191,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_cache_size"),
   expr: "synapse_util_caches_cache:size",
-  name: "[[job]]-[[index]] [[name]]",
+  name: "[[name]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yUnits: "",
@@ -205,8 +206,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_request_count_servlet"),
-  expr: "rate(synapse_http_server_in_flight_requests_count[2m])",
+  expr: "rate(synapse_http_server_request_count:servlet[2m])",
-  name: "[[job]]-[[index]] [[method]] [[servlet]]",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "req/s",
@@ -218,8 +219,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_request_count_servlet_minus_events"),
-  expr: "rate(synapse_http_server_in_flight_requests_count{servlet!=\"EventStreamRestServlet\", servlet!=\"SyncRestServlet\"}[2m])",
+  expr: "rate(synapse_http_server_request_count:servlet{servlet!=\"EventStreamRestServlet\", servlet!=\"SyncRestServlet\"}[2m])",
-  name: "[[job]]-[[index]] [[method]] [[servlet]]",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "req/s",
@@ -232,8 +233,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_response_time_avg"),
-  expr: "rate(synapse_http_server_response_time_seconds_sum[2m]) / rate(synapse_http_server_response_count[2m])",
+  expr: "rate(synapse_http_server_response_time_seconds[2m]) / rate(synapse_http_server_response_count[2m]) / 1000",
-  name: "[[job]]-[[index]] [[servlet]]",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "s/req",
@@ -276,7 +277,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_response_ru_utime"),
   expr: "rate(synapse_http_server_response_ru_utime_seconds[2m])",
-  name: "[[job]]-[[index]] [[servlet]]",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "s/s",
@@ -291,7 +292,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_response_db_txn_duration"),
   expr: "rate(synapse_http_server_response_db_txn_duration_seconds[2m])",
-  name: "[[job]]-[[index]] [[servlet]]",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "s/s",
@@ -305,8 +306,8 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_send_time_avg"),
-  expr: "rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet'}[2m]) / rate(synapse_http_server_response_count{servlet='RoomSendEventRestServlet'}[2m])",
+  expr: "rate(synapse_http_server_response_time_second{servlet='RoomSendEventRestServlet'}[2m]) / rate(synapse_http_server_response_count{servlet='RoomSendEventRestServlet'}[2m]) / 1000",
-  name: "[[job]]-[[index]] [[servlet]]",
+  name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "s/req",
@@ -322,7 +323,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_federation_client_sent"),
   expr: "rate(synapse_federation_client_sent[2m])",
-  name: "[[job]]-[[index]] [[type]]",
+  name: "[[type]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "req/s",
@@ -336,7 +337,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_federation_server_received"),
   expr: "rate(synapse_federation_server_received[2m])",
-  name: "[[job]]-[[index]] [[type]]",
+  name: "[[type]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "req/s",
@@ -366,7 +367,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_notifier_listeners"),
   expr: "synapse_notifier_listeners",
-  name: "[[job]]-[[index]]",
+  name: "listeners",
   min: 0,
   yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
@@ -381,7 +382,7 @@ new PromConsole.Graph({
 new PromConsole.Graph({
   node: document.querySelector("#synapse_notifier_notified_events"),
   expr: "rate(synapse_notifier_notified_events[2m])",
-  name: "[[job]]-[[index]]",
+  name: "events",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
   yUnits: "events/s",

contrib/prometheus/synapse-v2.rules

@@ -58,21 +58,3 @@ groups:
     labels:
       type: "PDU"
     expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
-
-  - record: synapse_storage_events_persisted_by_source_type
-    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_type="remote"})
-    labels:
-      type: remote
-  - record: synapse_storage_events_persisted_by_source_type
-    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_entity="*client*",origin_type="local"})
-    labels:
-      type: local
-  - record: synapse_storage_events_persisted_by_source_type
-    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_entity!="*client*",origin_type="local"})
-    labels:
-      type: bridges
-  - record: synapse_storage_events_persisted_by_event_type
-    expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep)
-  - record: synapse_storage_events_persisted_by_origin
-    expr: sum without(type) (synapse_storage_events_persisted_events_sep)
-

contrib/scripts/kick_users.py

@@ -1,11 +1,15 @@
 #!/usr/bin/env python
-
+from __future__ import print_function
+from argparse import ArgumentParser
 import json
+import requests
 import sys
 import urllib
-from argparse import ArgumentParser
 
-import requests
+try:
+    raw_input
+except NameError:  # Python 3
+    raw_input = input
 
 
 def _mkurl(template, kws):
@@ -52,7 +56,7 @@ def main(hs, room_id, access_token, user_id_prefix, why):
     print("The following user IDs will be kicked from %s" % room_name)
     for uid in kick_list:
         print(uid)
-    doit = input("Continue? [Y]es\n")
+    doit = raw_input("Continue? [Y]es\n")
     if len(doit) > 0 and doit.lower() == "y":
         print("Kicking members...")
         # encode them all

contrib/systemd-with-workers/README.md

@@ -1,2 +1,150 @@
-The documentation for using systemd to manage synapse workers is now part of
+# Setup Synapse with Workers and Systemd
-the main synapse distribution. See [docs/systemd-with-workers](../../docs/systemd-with-workers).
+
+This is a setup for managing synapse with systemd including support for
+managing workers. It provides a `matrix-synapse`, as well as a
+`matrix-synapse-worker@` service for any workers you require. Additionally to
+group the required services it sets up a `matrix.target`. You can use this to
+automatically start any bot- or bridge-services. More on this in
+[Bots and Bridges](#bots-and-bridges).
+
+See the folder [system](system) for any service and target files.
+
+The folder [workers](workers) contains an example configuration for the
+`federation_reader` worker. Pay special attention to the name of the
+configuration file. In order to work with the `matrix-synapse-worker@.service`
+service, it needs to have the exact same name as the worker app.
+
+This setup expects neither the homeserver nor any workers to fork. Forking is
+handled by systemd.
+
+## Setup
+
+1. Adjust your matrix configs. Make sure that the worker config files have the
+exact same name as the worker app. Compare `matrix-synapse-worker@.service` for
+why. You can find an example worker config in the [workers](workers) folder. See
+below for relevant settings in the `homeserver.yaml`.
+2. Copy the `*.service` and `*.target` files in [system](system) to
+`/etc/systemd/system`.
+3. `systemctl enable matrix-synapse.service` this adds the homeserver
+app to the `matrix.target`
+4. *Optional.* `systemctl enable
+matrix-synapse-worker@federation_reader.service` this adds the federation_reader
+app to the `matrix-synapse.service`
+5. *Optional.* Repeat step 4 for any additional workers you require.
+6. *Optional.* Add any bots or bridges by enabling them.
+7. Start all matrix related services via `systemctl start matrix.target`
+8. *Optional.* Enable autostart of all matrix related services on system boot
+via `systemctl enable matrix.target`
+
+## Usage
+
+After you have setup you can use the following commands to manage your synapse
+installation:
+
+```
+# Start matrix-synapse, all workers and any enabled bots or bridges.
+systemctl start matrix.target
+
+# Restart matrix-synapse and all workers (not necessarily restarting bots
+# or bridges, see "Bots and Bridges")
+systemctl restart matrix-synapse.service
+
+# Stop matrix-synapse and all workers (not necessarily restarting bots
+# or bridges, see "Bots and Bridges")
+systemctl stop matrix-synapse.service
+
+# Restart a specific worker (i. e. federation_reader), the homeserver is
+# unaffected by this.
+systemctl restart matrix-synapse-worker@federation_reader.service
+
+# Add a new worker (assuming all configs are setup already)
+systemctl enable matrix-synapse-worker@federation_writer.service
+systemctl restart matrix-synapse.service
+```
+
+## The Configs
+
+Make sure the `worker_app` is set in the `homeserver.yaml` and it does not fork.
+
+```
+worker_app: synapse.app.homeserver 
+daemonize: false
+```
+
+None of the workers should fork, as forking is handled by systemd. Hence make
+sure this is present in all worker config files.
+
+```
+worker_daemonize: false
+```
+
+The config files of all workers are expected to be located in
+`/etc/matrix-synapse/workers`. If you want to use a different location you have
+to edit the provided `*.service` files accordingly.  
+
+## Bots and Bridges
+
+Most bots and bridges do not care if the homeserver goes down or is restarted.
+Depending on the implementation this may crash them though. So look up the docs
+or ask the community of the specific bridge or bot you want to run to make sure
+you choose the correct setup.
+
+Whichever configuration you choose, after the setup the following will enable
+automatically starting (and potentially restarting) your bot/bridge with the
+`matrix.target`.
+
+```
+systemctl enable <yourBotOrBridgeName>.service
+```
+
+**Note** that from an inactive synapse the bots/bridges will only be started with
+synapse if you start the `matrix.target`, not if you start the
+`matrix-synapse.service`. This is on purpose. Think of `matrix-synapse.service`
+as *just* synapse, but `matrix.target` being anything matrix related, including
+synapse and any and all enabled bots and bridges.
+
+### Start with synapse but ignore synapse going down
+
+If the bridge can handle shutdowns of the homeserver you'll want to install the
+service in the `matrix.target` and optionally add a
+`After=matrix-synapse.service` dependency to have the bot/bridge start after
+synapse on starting everything.
+
+In this case the service file should look like this.
+
+```
+[Unit]
+# ...
+# Optional, this will only ensure that if you start everything, synapse will
+# be started before the bot/bridge will be started.
+After=matrix-synapse.service
+
+[Service]
+# ...
+
+[Install]
+WantedBy=matrix.target
+```
+
+### Stop/restart when synapse stops/restarts
+
+If the bridge can't handle shutdowns of the homeserver you'll still want to
+install the service in the `matrix.target` but also have to specify the
+`After=matrix-synapse.service` *and* `BindsTo=matrix-synapse.service`
+dependencies to have the bot/bridge stop/restart with synapse.
+
+In this case the service file should look like this.
+
+```
+[Unit]
+# ...
+# Mandatory
+After=matrix-synapse.service
+BindsTo=matrix-synapse.service
+
+[Service]
+# ...
+
+[Install]
+WantedBy=matrix.target
+```

contrib/systemd-with-workers/system/matrix-synapse-worker@.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=Synapse Matrix Worker
+After=matrix-synapse.service
+BindsTo=matrix-synapse.service
+
+[Service]
+Type=notify
+NotifyAccess=main
+User=matrix-synapse
+WorkingDirectory=/var/lib/matrix-synapse
+EnvironmentFile=/etc/default/matrix-synapse
+ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.%i --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --config-path=/etc/matrix-synapse/workers/%i.yaml
+ExecReload=/bin/kill -HUP $MAINPID
+Restart=always
+RestartSec=3
+SyslogIdentifier=matrix-synapse-%i
+
+[Install]
+WantedBy=matrix-synapse.service

contrib/systemd-with-workers/system/matrix-synapse.service

@@ -1,9 +1,5 @@
 [Unit]
-Description=Synapse master
+Description=Synapse Matrix Homeserver
-
-# This service should be restarted when the synapse target is restarted.
-PartOf=matrix-synapse.target
-ReloadPropagatedFrom=matrix-synapse.target
 
 [Service]
 Type=notify
@@ -19,4 +15,4 @@ RestartSec=3
 SyslogIdentifier=matrix-synapse
 
 [Install]
-WantedBy=matrix-synapse.target
+WantedBy=matrix.target

contrib/systemd-with-workers/system/matrix.target

@@ -0,0 +1,7 @@
+[Unit]
+Description=Contains matrix services like synapse, bridges and bots
+After=network.target
+AllowIsolate=no
+
+[Install]
+WantedBy=multi-user.target

contrib/systemd-with-workers/workers/federation_reader.yaml

@@ -1,7 +1,7 @@
 worker_app: synapse.app.federation_reader
-worker_name: federation_reader1
 
 worker_replication_host: 127.0.0.1
+worker_replication_port: 9092
 worker_replication_http_port: 9093
 
 worker_listeners:
@@ -10,4 +10,5 @@ worker_listeners:
       resources:
           - names: [federation]
 
+worker_daemonize: false
 worker_log_config: /etc/matrix-synapse/federation-reader-log.yaml

contrib/systemd/matrix-synapse.service

@@ -15,9 +15,6 @@
 
 [Unit]
 Description=Synapse Matrix homeserver
-# If you are using postgresql to persist data, uncomment this line to make sure
-# synapse starts after the postgresql service.
-# After=postgresql.service
 
 [Service]
 Type=notify

debian/build_virtualenv

@@ -33,18 +33,17 @@ esac
 # Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather
 # than the 2/3 compatible `virtualenv`.
 
-# Pin pip to 20.3.4 to fix breakage in 21.0 on py3.5 (xenial)
-
 dh_virtualenv \
     --install-suffix "matrix-synapse" \
     --builtin-venv \
+    --setuptools \
     --python "$SNAKE" \
-    --upgrade-pip-to="20.3.4" \
+    --upgrade-pip \
     --preinstall="lxml" \
     --preinstall="mock" \
     --extra-pip-arg="--no-cache-dir" \
     --extra-pip-arg="--compile" \
-    --extras="all,systemd,test"
+    --extras="all,systemd"
 
 PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
 VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"
@@ -58,10 +57,10 @@ trap "rm -r $tmpdir" EXIT
 cp -r tests "$tmpdir"
 
 PYTHONPATH="$tmpdir" \
-    "${TARGET_PYTHON}" -m twisted.trial --reporter=text -j2 tests
+    "${TARGET_PYTHON}" -B -m twisted.trial --reporter=text -j2 tests
 
 # build the config file
-"${TARGET_PYTHON}" "${VIRTUALENV_DIR}/bin/generate_config" \
+"${TARGET_PYTHON}" -B "${VIRTUALENV_DIR}/bin/generate_config" \
         --config-dir="/etc/matrix-synapse" \
         --data-dir="/var/lib/matrix-synapse" |
     perl -pe '
@@ -87,7 +86,7 @@ PYTHONPATH="$tmpdir" \
 ' > "${PACKAGE_BUILD_DIR}/etc/matrix-synapse/homeserver.yaml"
 
 # build the log config file
-"${TARGET_PYTHON}" "${VIRTUALENV_DIR}/bin/generate_log_config" \
+"${TARGET_PYTHON}" -B "${VIRTUALENV_DIR}/bin/generate_log_config" \
         --output-file="${PACKAGE_BUILD_DIR}/etc/matrix-synapse/log.yaml"
 
 # add a dependency on the right version of python to substvars.

debian/changelog

@@ -1,252 +1,3 @@
-matrix-synapse-py3 (1.30.1) stable; urgency=medium
-
-  * New synapse release 1.30.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 26 Mar 2021 12:01:28 +0000
-
-matrix-synapse-py3 (1.30.0) stable; urgency=medium
-
-  * New synapse release 1.30.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 22 Mar 2021 13:15:34 +0000
-
-matrix-synapse-py3 (1.29.0) stable; urgency=medium
-
-  [ Jonathan de Jong ]
-  * Remove the python -B flag (don't generate bytecode) in scripts and documentation.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.29.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 08 Mar 2021 13:51:50 +0000
-
-matrix-synapse-py3 (1.28.0) stable; urgency=medium
-
-  * New synapse release 1.28.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 25 Feb 2021 10:21:57 +0000
-
-matrix-synapse-py3 (1.27.0) stable; urgency=medium
-
-  [ Dan Callahan ]
-  * Fix build on Ubuntu 16.04 LTS (Xenial).
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.27.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 16 Feb 2021 13:11:28 +0000
-
-matrix-synapse-py3 (1.26.0) stable; urgency=medium
-
-  [ Richard van der Hoff ]
-  * Remove dependency on `python3-distutils`.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.26.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 27 Jan 2021 12:43:35 -0500
-
-matrix-synapse-py3 (1.25.0) stable; urgency=medium
-
-  [ Dan Callahan ]
-  * Update dependencies to account for the removal of the transitional
-    dh-systemd package from Debian Bullseye.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.25.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 13 Jan 2021 10:14:55 +0000
-
-matrix-synapse-py3 (1.24.0) stable; urgency=medium
-
-  * New synapse release 1.24.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 09 Dec 2020 10:14:30 +0000
-
-matrix-synapse-py3 (1.23.1) stable; urgency=medium
-
-  * New synapse release 1.23.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 09 Dec 2020 10:40:39 +0000
-
-matrix-synapse-py3 (1.23.0) stable; urgency=medium
-
-  * New synapse release 1.23.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 18 Nov 2020 11:41:28 +0000
-
-matrix-synapse-py3 (1.22.1) stable; urgency=medium
-
-  * New synapse release 1.22.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 30 Oct 2020 15:25:37 +0000
-
-matrix-synapse-py3 (1.22.0) stable; urgency=medium
-
-  * New synapse release 1.22.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 27 Oct 2020 12:07:12 +0000
-
-matrix-synapse-py3 (1.21.2) stable; urgency=medium
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.21.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 15 Oct 2020 09:23:27 -0400
-
-matrix-synapse-py3 (1.21.1) stable; urgency=medium
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.21.1.
-
-  [ Andrew Morgan ]
-  * Explicitly install "test" python dependencies.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Oct 2020 10:24:13 +0100
-
-matrix-synapse-py3 (1.21.0) stable; urgency=medium
-
-  * New synapse release 1.21.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 12 Oct 2020 15:47:44 +0100
-
-matrix-synapse-py3 (1.20.1) stable; urgency=medium
-
-  * New synapse release 1.20.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 24 Sep 2020 16:25:22 +0100
-
-matrix-synapse-py3 (1.20.0) stable; urgency=medium
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.20.0.
-
-  [ Dexter Chua ]
-  * Use Type=notify in systemd service
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 22 Sep 2020 15:19:32 +0100
-
-matrix-synapse-py3 (1.19.3) stable; urgency=medium
-
-  * New synapse release 1.19.3.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 18 Sep 2020 14:59:30 +0100
-
-matrix-synapse-py3 (1.19.2) stable; urgency=medium
-
-  * New synapse release 1.19.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 16 Sep 2020 12:50:30 +0100
-
-matrix-synapse-py3 (1.19.1) stable; urgency=medium
-
-  * New synapse release 1.19.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 27 Aug 2020 10:50:19 +0100
-
-matrix-synapse-py3 (1.19.0) stable; urgency=medium
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.19.0.
-
-  [ Aaron Raimist ]
-  * Fix outdated documentation for SYNAPSE_CACHE_FACTOR
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 17 Aug 2020 14:06:42 +0100
-
-matrix-synapse-py3 (1.18.0) stable; urgency=medium
-
-  * New synapse release 1.18.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 30 Jul 2020 10:55:53 +0100
-
-matrix-synapse-py3 (1.17.0) stable; urgency=medium
-
-  * New synapse release 1.17.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 13 Jul 2020 10:20:31 +0100
-
-matrix-synapse-py3 (1.16.1) stable; urgency=medium
-
-  * New synapse release 1.16.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 10 Jul 2020 12:09:24 +0100
-
-matrix-synapse-py3 (1.17.0rc1) stable; urgency=medium
-
-  * New synapse release 1.17.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 09 Jul 2020 16:53:12 +0100
-
-matrix-synapse-py3 (1.16.0) stable; urgency=medium
-
-  * New synapse release 1.16.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 08 Jul 2020 11:03:48 +0100
-
-matrix-synapse-py3 (1.15.2) stable; urgency=medium
-
-  * New synapse release 1.15.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 02 Jul 2020 10:34:00 -0400
-
-matrix-synapse-py3 (1.15.1) stable; urgency=medium
-
-  * New synapse release 1.15.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 16 Jun 2020 10:27:50 +0100
-
-matrix-synapse-py3 (1.15.0) stable; urgency=medium
-
-  * New synapse release 1.15.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 11 Jun 2020 13:27:06 +0100
-
-matrix-synapse-py3 (1.14.0) stable; urgency=medium
-
-  * New synapse release 1.14.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 28 May 2020 10:37:27 +0000
-
-matrix-synapse-py3 (1.13.0) stable; urgency=medium
-
-  [ Patrick Cloke ]
-  * Add information about .well-known files to Debian installation scripts.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.13.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 19 May 2020 09:16:56 -0400
-
-matrix-synapse-py3 (1.12.4) stable; urgency=medium
-
-  * New synapse release 1.12.4.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 23 Apr 2020 10:58:14 -0400
-
-matrix-synapse-py3 (1.12.3) stable; urgency=medium
-
-  [ Richard van der Hoff ]
-  * Update the Debian build scripts to handle the new installation paths
-   for the support libraries introduced by Pillow 7.1.1.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.12.3.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 03 Apr 2020 10:55:03 +0100
-
-matrix-synapse-py3 (1.12.2) stable; urgency=medium
-
-  * New synapse release 1.12.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 02 Apr 2020 19:02:17 +0000
-
-matrix-synapse-py3 (1.12.1) stable; urgency=medium
-
-  * New synapse release 1.12.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 02 Apr 2020 11:30:47 +0000
-
 matrix-synapse-py3 (1.12.0) stable; urgency=medium
 
   * New synapse release 1.12.0.

debian/control

@@ -3,11 +3,9 @@ Section: contrib/python
 Priority: extra
 Maintainer: Synapse Packaging team <packages@matrix.org>
 # keep this list in sync with the build dependencies in docker/Dockerfile-dhvirtualenv.
-# TODO: Remove the dependency on dh-systemd after dropping support for Ubuntu xenial
-#       On all other supported releases, it's merely a transitional package which
-#       does nothing but depends on debhelper (> 9.20160709)
 Build-Depends:
- debhelper (>= 9.20160709) | dh-systemd,
+ debhelper (>= 9),
+ dh-systemd,
  dh-virtualenv (>= 1.1),
  libsystemd-dev,
  libpq-dev,
@@ -31,6 +29,7 @@ Pre-Depends: dpkg (>= 1.16.1)
 Depends:
  adduser,
  debconf,
+ python3-distutils|libpython3-stdlib (<< 3.6),
  ${misc:Depends},
  ${shlibs:Depends},
  ${synapse:pydepends},

debian/matrix-synapse.default

@@ -1,2 +1,2 @@
 # Specify environment variables used when running Synapse
-# SYNAPSE_CACHE_FACTOR=0.5 (default)
+# SYNAPSE_CACHE_FACTOR=1 (default)

debian/matrix-synapse.service

@@ -2,7 +2,7 @@
 Description=Synapse Matrix homeserver
 
 [Service]
-Type=notify
+Type=simple
 User=matrix-synapse
 WorkingDirectory=/var/lib/matrix-synapse
 EnvironmentFile=/etc/default/matrix-synapse

debian/po/templates.pot

@@ -1,14 +1,14 @@
 # SOME DESCRIPTIVE TITLE.
 # Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
-# This file is distributed under the same license as the matrix-synapse-py3 package.
+# This file is distributed under the same license as the matrix-synapse package.
 # FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
 #
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: matrix-synapse-py3\n"
+"Project-Id-Version: matrix-synapse\n"
-"Report-Msgid-Bugs-To: matrix-synapse-py3@packages.debian.org\n"
+"Report-Msgid-Bugs-To: matrix-synapse@packages.debian.org\n"
-"POT-Creation-Date: 2020-04-06 16:39-0400\n"
+"POT-Creation-Date: 2017-02-21 07:51+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -28,10 +28,7 @@ msgstr ""
 #: ../templates:1001
 msgid ""
 "The name that this homeserver will appear as, to clients and other servers "
-"via federation. This is normally the public hostname of the server running "
+"via federation. This name should match the SRV record published in DNS."
-"synapse, but can be different if you set up delegation. Please refer to the "
-"delegation documentation in this case: https://github.com/matrix-org/synapse/"
-"blob/master/docs/delegate.md."
 msgstr ""
 
 #. Type: boolean

debian/rules

@@ -15,38 +15,17 @@ override_dh_installinit:
 # we don't really want to strip the symbols from our object files.
 override_dh_strip:
 
-# dh_shlibdeps calls dpkg-shlibdeps, which finds all the binary files
-# (executables and shared libs) in the package, and looks for the shared
-# libraries that they depend on. It then adds a dependency on the package that
-# contains that library to the package.
-#
-# We make two modifications to that process...
-#
 override_dh_shlibdeps:
-        # Firstly, postgres is not a hard dependency for us, so we want to make
+        # make the postgres package's dependencies a recommendation
-        # the things that psycopg2 depends on (such as libpq) be
+        # rather than a hard dependency.
-        # recommendations rather than hard dependencies. We do so by
-        # running dpkg-shlibdeps manually on psycopg2's libs.
-        #
 	find debian/$(PACKAGE_NAME)/ -path '*/site-packages/psycopg2/*.so' | \
 	    xargs dpkg-shlibdeps -Tdebian/$(PACKAGE_NAME).substvars \
 	        -pshlibs1 -dRecommends
 
-        # secondly, we exclude PIL's libraries from the process. They are known
+        # all the other dependencies can be normal 'Depends' requirements,
-        # to be self-contained, but they have interdependencies and
+        # except for PIL's, which is self-contained and which confuses
-        # dpkg-shlibdeps doesn't know how to resolve them.
+        # dpkg-shlibdeps.
-        #
+	dh_shlibdeps -X site-packages/PIL/.libs -X site-packages/psycopg2
-        # As of Pillow 7.1.0, these libraries are in
-        # site-packages/Pillow.libs. Previously, they were in
-        # site-packages/PIL/.libs.
-        #
-        # (we also need to exclude psycopg2, of course, since we've already
-        # dealt with that.)
-        #
-	dh_shlibdeps \
-	    -X site-packages/PIL/.libs \
-	    -X site-packages/Pillow.libs \
-	    -X site-packages/psycopg2
 
 override_dh_virtualenv:
 	./debian/build_virtualenv

debian/synctl.1

@@ -44,7 +44,7 @@ Configuration file may be generated as follows:
 .
 .nf
 
-$ python \-m synapse\.app\.homeserver \-c config\.yaml \-\-generate\-config \-\-server\-name=<server name>
+$ python \-B \-m synapse\.app\.homeserver \-c config\.yaml \-\-generate\-config \-\-server\-name=<server name>
 .
 .fi
 .

debian/synctl.ronn

@@ -41,25 +41,24 @@ process.
 
 Configuration file may be generated as follows:
 
-    $ python -m synapse.app.homeserver -c config.yaml --generate-config --server-name=<server name>
+    $ python -B -m synapse.app.homeserver -c config.yaml --generate-config --server-name=<server name>
 
 ## ENVIRONMENT
 
   * `SYNAPSE_CACHE_FACTOR`:
-    Synapse's architecture is quite RAM hungry currently - we deliberately
+    Synapse's architecture is quite RAM hungry currently - a lot of
-    cache a lot of recent room data and metadata in RAM in order to speed up
+    recent room data and metadata is deliberately cached in RAM in
-    common requests. We'll improve this in the future, but for now the easiest
+    order to speed up common requests.  This will be improved in
-    way to either reduce the RAM usage (at the risk of slowing things down)
+    future, but for now the easiest way to either reduce the RAM usage
-    is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment
+    (at the risk of slowing things down) is to set the
-    variable. The default is 0.5, which can be decreased to reduce RAM usage
+    SYNAPSE_CACHE_FACTOR environment variable. Roughly speaking, a
-    in memory constrained enviroments, or increased if performance starts to
+    SYNAPSE_CACHE_FACTOR of 1.0 will max out at around 3-4GB of
-    degrade.
+    resident memory - this is what we currently run the matrix.org
-
+    on. The default setting is currently 0.1, which is probably around
-    However, degraded performance due to a low cache factor, common on
+    a ~700MB footprint. You can dial it down further to 0.02 if
-    machines with slow disks, often leads to explosions in memory use due
+    desired, which targets roughly ~512MB. Conversely you can dial it
-    backlogged requests. In this case, reducing the cache factor will make
+    up if you need performance for lots of users and have a box with a
-    things worse. Instead, try increasing it drastically. 2.0 is a good
+    lot of RAM.
-    starting value.
 
 ## COPYRIGHT
 

debian/templates

@@ -2,10 +2,8 @@ Template: matrix-synapse/server-name
 Type: string
 _Description: Name of the server:
  The name that this homeserver will appear as, to clients and other
- servers via federation. This is normally the public hostname of the
+ servers via federation. This name should match the SRV record
- server running synapse, but can be different if you set up delegation.
+ published in DNS.
- Please refer to the delegation documentation in this case:
- https://github.com/matrix-org/synapse/blob/master/docs/delegate.md.
 
 Template: matrix-synapse/report-stats
 Type: boolean

demo/start.sh

@@ -30,8 +30,6 @@ for port in 8080 8081 8082; do
     if ! grep -F "Customisation made by demo/start.sh" -q  $DIR/etc/$port.config; then
         printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config
 
-        echo "public_baseurl: http://localhost:$port/" >> $DIR/etc/$port.config
-
         echo 'enable_registration: true' >> $DIR/etc/$port.config
 
         # Warning, this heredoc depends on the interaction of tabs and spaces. Please don't

demo/webserver.py

@@ -0,0 +1,59 @@
+import argparse
+import BaseHTTPServer
+import os
+import SimpleHTTPServer
+import cgi, logging
+
+from daemonize import Daemonize
+
+
+class SimpleHTTPRequestHandlerWithPOST(SimpleHTTPServer.SimpleHTTPRequestHandler):
+    UPLOAD_PATH = "upload"
+
+    """
+    Accept all post request as file upload
+    """
+
+    def do_POST(self):
+
+        path = os.path.join(self.UPLOAD_PATH, os.path.basename(self.path))
+        length = self.headers["content-length"]
+        data = self.rfile.read(int(length))
+
+        with open(path, "wb") as fh:
+            fh.write(data)
+
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.end_headers()
+
+        # Return the absolute path of the uploaded file
+        self.wfile.write('{"url":"/%s"}' % path)
+
+
+def setup():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("directory")
+    parser.add_argument("-p", "--port", dest="port", type=int, default=8080)
+    parser.add_argument("-P", "--pid-file", dest="pid", default="web.pid")
+    args = parser.parse_args()
+
+    # Get absolute path to directory to serve, as daemonize changes to '/'
+    os.chdir(args.directory)
+    dr = os.getcwd()
+
+    httpd = BaseHTTPServer.HTTPServer(("", args.port), SimpleHTTPRequestHandlerWithPOST)
+
+    def run():
+        os.chdir(dr)
+        httpd.serve_forever()
+
+    daemon = Daemonize(
+        app="synapse-webclient", pid=args.pid, action=run, auto_close_fds=False
+    )
+
+    daemon.start()
+
+
+if __name__ == "__main__":
+    setup()

docker/Dockerfile

@@ -11,45 +11,39 @@
 #    docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.6 .
 #
 
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=3.7
 
 ###
 ### Stage 0: builder
 ###
-FROM docker.io/python:${PYTHON_VERSION}-slim as builder
+FROM docker.io/python:${PYTHON_VERSION}-alpine3.11 as builder
 
 # install the OS build deps
-RUN apt-get update && apt-get install -y \
-        build-essential \
-        libffi-dev \
-        libjpeg-dev \
-        libpq-dev \
-        libssl-dev \
-        libwebp-dev \
-        libxml++2.6-dev \
-        libxslt1-dev \
-        openssl \
-        rustc \
-        zlib1g-dev \
-        && rm -rf /var/lib/apt/lists/*
 
-# Build dependencies that are not available as wheels, to speed up rebuilds
+RUN apk add \
+        build-base \
+        libffi-dev \
+        libjpeg-turbo-dev \
+        libressl-dev \
+        libxslt-dev \
+        linux-headers \
+        postgresql-dev \
+        zlib-dev
+
+# build things which have slow build steps, before we copy synapse, so that
+# the layer can be cached.
+#
+# (we really just care about caching a wheel here, as the "pip install" below
+# will install them again.)
+
 RUN pip install --prefix="/install" --no-warn-script-location \
         cryptography \
-        frozendict \
+        msgpack-python \
-        jaeger-client \
+        pillow \
-        opentracing \
+        pynacl
-        # Match the version constraints of Synapse
-        "prometheus_client>=0.4.0" \
-        psycopg2 \
-        pycparser \
-        pyrsistent \
-        pyyaml \
-        simplejson \
-        threadloop \
-        thrift
 
 # now install synapse and all of the python deps to /install.
+
 COPY synapse /synapse/synapse/
 COPY scripts /synapse/scripts/
 COPY MANIFEST.in README.rst setup.py synctl /synapse/
@@ -61,19 +55,19 @@ RUN pip install --prefix="/install" --no-warn-script-location \
 ### Stage 1: runtime
 ###
 
-FROM docker.io/python:${PYTHON_VERSION}-slim
+FROM docker.io/python:${PYTHON_VERSION}-alpine3.10
 
-RUN apt-get update && apt-get install -y \
+# xmlsec is required for saml support
-        curl \
+RUN apk add --no-cache --virtual .runtime_deps \
-        gosu \
+        libffi \
-        libjpeg62-turbo \
+        libjpeg-turbo \
-        libpq5 \
+        libressl \
-        libwebp6 \
+        libxslt \
-        xmlsec1 \
+        libpq \
-        libjemalloc2 \
+        zlib \
-        libssl-dev \
+        su-exec \
-        openssl \
+        tzdata \
-        && rm -rf /var/lib/apt/lists/*
+        xmlsec
 
 COPY --from=builder /install /usr/local
 COPY ./docker/start.py /start.py
@@ -84,6 +78,3 @@ VOLUME ["/data"]
 EXPOSE 8008/tcp 8009/tcp 8448/tcp
 
 ENTRYPOINT ["/start.py"]
-
-HEALTHCHECK --interval=1m --timeout=5s \
-        CMD curl -fSs http://localhost:8008/health || exit 1

docker/Dockerfile-dhvirtualenv

@@ -27,19 +27,15 @@ RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
         wget
 
 # fetch and unpack the package
-# TODO: Upgrade to 1.2.2 once xenial is dropped
+RUN wget -q -O /dh-virtuenv-1.1.tar.gz https://github.com/spotify/dh-virtualenv/archive/1.1.tar.gz
-RUN mkdir /dh-virtualenv
+RUN tar xvf /dh-virtuenv-1.1.tar.gz
-RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/spotify/dh-virtualenv/archive/ac6e1b1.tar.gz
-RUN tar -xv --strip-components=1 -C /dh-virtualenv -f /dh-virtualenv.tar.gz
 
-# install its build deps. We do another apt-cache-update here, because we might
+# install its build deps
-# be using a stale cache from docker build.
+RUN cd dh-virtualenv-1.1/ \
-RUN apt-get update -qq -o Acquire::Languages=none \
+    && env DEBIAN_FRONTEND=noninteractive mk-build-deps -ri -t "apt-get -yqq --no-install-recommends"
-    && cd /dh-virtualenv \
-    && env DEBIAN_FRONTEND=noninteractive mk-build-deps -ri -t "apt-get -y --no-install-recommends"
 
 # build it
-RUN cd /dh-virtualenv && dpkg-buildpackage -us -uc -b
+RUN cd dh-virtualenv-1.1 && dpkg-buildpackage -us -uc -b
 
 ###
 ### Stage 1
@@ -51,22 +47,17 @@ FROM ${distro}
 ARG distro=""
 ENV distro ${distro}
 
-# Python < 3.7 assumes LANG="C" means ASCII-only and throws on printing unicode
-# http://bugs.python.org/issue19846
-ENV LANG C.UTF-8
-
 # Install the build dependencies
 #
 # NB: keep this list in sync with the list of build-deps in debian/control
 # TODO: it would be nice to do that automatically.
-# TODO: Remove the dh-systemd stanza after dropping support for Ubuntu xenial
-#       it's a transitional package on all other, more recent releases
 RUN apt-get update -qq -o Acquire::Languages=none \
     && env DEBIAN_FRONTEND=noninteractive apt-get install \
         -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
         build-essential \
         debhelper \
         devscripts \
+        dh-systemd \
         libsystemd-dev \
         lsb-release \
         pkg-config \
@@ -75,18 +66,14 @@ RUN apt-get update -qq -o Acquire::Languages=none \
         python3-setuptools \
         python3-venv \
         sqlite3 \
-        libpq-dev \
+        libpq-dev
-        xmlsec1 \
-    && ( env DEBIAN_FRONTEND=noninteractive apt-get install \
-         -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
-         dh-systemd || true )
 
-COPY --from=builder /dh-virtualenv_1.2~dev-1_all.deb /
+COPY --from=builder /dh-virtualenv_1.1-1_all.deb /
 
 # install dhvirtualenv. Update the apt cache again first, in case we got a
 # cached cache from docker the first time.
 RUN apt-get update -qq -o Acquire::Languages=none \
-    && apt-get install -yq /dh-virtualenv_1.2~dev-1_all.deb
+    && apt-get install -yq /dh-virtualenv_1.1-1_all.deb
 
 WORKDIR /synapse/source
 ENTRYPOINT ["bash","/synapse/source/docker/build_debian.sh"]

docker/README.md

@@ -11,6 +11,7 @@ The image also does *not* provide a TURN server.
 By default, the image expects a single volume, located at ``/data``, that will hold:
 
 * configuration files;
+* temporary files during uploads;
 * uploaded media and thumbnails;
 * the SQLite database if you do not configure postgres;
 * the appservices configuration.
@@ -82,7 +83,7 @@ docker logs synapse
 If all is well, you should now be able to connect to http://localhost:8008 and
 see a confirmation message.
 
-The following environment variables are supported in `run` mode:
+The following environment variables are supported in run mode:
 
 * `SYNAPSE_CONFIG_DIR`: where additional config files are stored. Defaults to
   `/data`.
@@ -93,35 +94,6 @@ The following environment variables are supported in `run` mode:
 * `UID`, `GID`: the user and group id to run Synapse as. Defaults to `991`, `991`.
 * `TZ`: the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) the container will run with. Defaults to `UTC`.
 
-For more complex setups (e.g. for workers) you can also pass your args directly to synapse using `run` mode. For example like this:
-
-```
-docker run -d --name synapse \
-    --mount type=volume,src=synapse-data,dst=/data \
-    -p 8008:8008 \
-    matrixdotorg/synapse:latest run \
-    -m synapse.app.generic_worker \
-    --config-path=/data/homeserver.yaml \
-    --config-path=/data/generic_worker.yaml
-```
-
-If you do not provide `-m`, the value of the `SYNAPSE_WORKER` environment variable is used. If you do not provide at least one `--config-path` or `-c`, the value of the `SYNAPSE_CONFIG_PATH` environment variable is used instead.
-
-## Generating an (admin) user
-
-After synapse is running, you may wish to create a user via `register_new_matrix_user`.
-
-This requires a `registration_shared_secret` to be set in your config file. Synapse
-must be restarted to pick up this change.
-
-You can then call the script:
-
-```
-docker exec -it synapse register_new_matrix_user http://localhost:8008 -c /data/homeserver.yaml --help
-```
-
-Remember to remove the `registration_shared_secret` and restart if you no-longer need it.
-
 ## TLS support
 
 The default configuration exposes a single HTTP port: http://localhost:8008. It
@@ -175,37 +147,3 @@ docker build -t matrixdotorg/synapse -f docker/Dockerfile .
 
 You can choose to build a different docker image by changing the value of the `-f` flag to
 point to another Dockerfile.
-
-## Disabling the healthcheck
-
-If you are using a non-standard port or tls inside docker you can disable the healthcheck
-whilst running the above `docker run` commands. 
-
-```
-   --no-healthcheck
-```
-## Setting custom healthcheck on docker run
-
-If you wish to point the healthcheck at a different port with docker command, add the following
-
-```
-  --health-cmd 'curl -fSs http://localhost:1234/health'
-```
-
-## Setting the healthcheck in docker-compose file
-
-You can add the following to set a custom healthcheck in a docker compose file.
-You will need version >2.1 for this to work. 
-
-```
-healthcheck:
-  test: ["CMD", "curl", "-fSs", "http://localhost:8008/health"]
-  interval: 1m
-  timeout: 10s
-  retries: 3
-```
-
-## Using jemalloc
-
-Jemalloc is embedded in the image and will be used instead of the default allocator.
-You can read about jemalloc by reading the Synapse [README](../README.md)

docker/conf/homeserver.yaml

@@ -89,7 +89,8 @@ federation_rc_concurrent: 3
 ## Files ##
 
 media_store_path: "/data/media"
-max_upload_size: "{{ SYNAPSE_MAX_UPLOAD_SIZE or "50M" }}"
+uploads_path: "/data/uploads"
+max_upload_size: "{{ SYNAPSE_MAX_UPLOAD_SIZE or "10M" }}"
 max_image_pixels: "32M"
 dynamic_thumbnails: false
 
@@ -197,10 +198,12 @@ old_signing_keys: {}
 key_refresh_interval: "1d" # 1 Day.
 
 # The trusted servers to download signing keys from.
-trusted_key_servers:
+perspectives:
-  - server_name: matrix.org
+  servers:
-    verify_keys:
+    "matrix.org":
-      "ed25519:auto": "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"
+      verify_keys:
+        "ed25519:auto":
+          key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"
 
 password_config:
    enabled: true

docker/conf/log.config

@@ -4,10 +4,16 @@ formatters:
   precise:
    format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
 
+filters:
+  context:
+    (): synapse.logging.context.LoggingContextFilter
+    request: ""
+
 handlers:
   console:
     class: logging.StreamHandler
     formatter: precise
+    filters: [context]
 
 loggers:
     synapse.storage.SQL:

docker/start.py

@@ -3,7 +3,6 @@
 import codecs
 import glob
 import os
-import platform
 import subprocess
 import sys
 
@@ -121,7 +120,7 @@ def generate_config_from_template(config_dir, config_path, environ, ownership):
 
     if ownership is not None:
         subprocess.check_output(["chown", "-R", ownership, "/data"])
-        args = ["gosu", ownership] + args
+        args = ["su-exec", ownership] + args
 
     subprocess.check_output(args)
 
@@ -173,14 +172,14 @@ def run_generate_config(environ, ownership):
         # make sure that synapse has perms to write to the data dir.
         subprocess.check_output(["chown", ownership, data_dir])
 
-        args = ["gosu", ownership] + args
+        args = ["su-exec", ownership] + args
-        os.execv("/usr/sbin/gosu", args)
+        os.execv("/sbin/su-exec", args)
     else:
         os.execv("/usr/local/bin/python", args)
 
 
 def main(args, environ):
-    mode = args[1] if len(args) > 1 else "run"
+    mode = args[1] if len(args) > 1 else None
     desired_uid = int(environ.get("UID", "991"))
     desired_gid = int(environ.get("GID", "991"))
     synapse_worker = environ.get("SYNAPSE_WORKER", "synapse.app.homeserver")
@@ -190,7 +189,7 @@ def main(args, environ):
         ownership = "{}:{}".format(desired_uid, desired_gid)
 
     if ownership is None:
-        log("Will not perform chmod/gosu as UserID already matches request")
+        log("Will not perform chmod/su-exec as UserID already matches request")
 
     # In generate mode, generate a configuration and missing keys, then exit
     if mode == "generate":
@@ -206,59 +205,41 @@ def main(args, environ):
             config_dir, config_path, environ, ownership
         )
 
-    if mode != "run":
+    if mode is not None:
         error("Unknown execution mode '%s'" % (mode,))
 
-    args = args[2:]
+    config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
+    config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
 
-    if "-m" not in args:
+    if not os.path.exists(config_path):
-        args = ["-m", synapse_worker] + args
+        if "SYNAPSE_SERVER_NAME" in environ:
-
+            error(
-    jemallocpath = "/usr/lib/%s-linux-gnu/libjemalloc.so.2" % (platform.machine(),)
+                """\
-
-    if os.path.isfile(jemallocpath):
-        environ["LD_PRELOAD"] = jemallocpath
-    else:
-        log("Could not find %s, will not use" % (jemallocpath,))
-
-    # if there are no config files passed to synapse, try adding the default file
-    if not any(p.startswith("--config-path") or p.startswith("-c") for p in args):
-        config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
-        config_path = environ.get(
-            "SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml"
-        )
-
-        if not os.path.exists(config_path):
-            if "SYNAPSE_SERVER_NAME" in environ:
-                error(
-                    """\
 Config file '%s' does not exist.
 
 The synapse docker image no longer supports generating a config file on-the-fly
 based on environment variables. You can migrate to a static config file by
 running with 'migrate_config'. See the README for more details.
 """
-                    % (config_path,)
-                )
-
-            error(
-                "Config file '%s' does not exist. You should either create a new "
-                "config file by running with the `generate` argument (and then edit "
-                "the resulting file before restarting) or specify the path to an "
-                "existing config file with the SYNAPSE_CONFIG_PATH variable."
                 % (config_path,)
             )
 
-        args += ["--config-path", config_path]
+        error(
+            "Config file '%s' does not exist. You should either create a new "
+            "config file by running with the `generate` argument (and then edit "
+            "the resulting file before restarting) or specify the path to an "
+            "existing config file with the SYNAPSE_CONFIG_PATH variable."
+            % (config_path,)
+        )
 
-    log("Starting synapse with args " + " ".join(args))
+    log("Starting synapse with config file " + config_path)
 
-    args = ["python"] + args
+    args = ["python", "-m", synapse_worker, "--config-path", config_path]
     if ownership is not None:
-        args = ["gosu", ownership] + args
+        args = ["su-exec", ownership] + args
-        os.execve("/usr/sbin/gosu", args, environ)
+        os.execv("/sbin/su-exec", args)
     else:
-        os.execve("/usr/local/bin/python", args, environ)
+        os.execv("/usr/local/bin/python", args)
 
 
 if __name__ == "__main__":

docs/.sample_config_header.yaml

@@ -10,16 +10,5 @@
 # homeserver.yaml. Instead, if you are starting from scratch, please generate
 # a fresh config using Synapse by following the instructions in INSTALL.md.
 
-# Configuration options that take a time period can be set using a number
-# followed by a letter. Letters have the following meanings:
-# s = second
-# m = minute
-# h = hour
-# d = day
-# w = week
-# y = year
-# For example, setting redaction_retention_period: 5m would remove redacted
-# messages from the database after 5 minutes, rather than 5 months.
-
 ################################################################################
 

docs/ACME.md

@@ -12,14 +12,13 @@ introduced support for automatically provisioning certificates through
 In [March 2019](https://community.letsencrypt.org/t/end-of-life-plan-for-acmev1/88430),
 Let's Encrypt announced that they were deprecating version 1 of the ACME
 protocol, with the plan to disable the use of it for new accounts in
-November 2019, for new domains in June 2020, and for existing accounts and
+November 2019, and for existing accounts in June 2020.
-domains in June 2021.
 
 Synapse doesn't currently support version 2 of the ACME protocol, which
 means that:
 
 * for existing installs, Synapse's built-in ACME support will continue
-  to work until June 2021.
+  to work until June 2020.
 * for new installs, this feature will not work at all.
 
 Either way, it is recommended to move from Synapse's ACME support

docs/admin_api/README.rst

@@ -4,21 +4,17 @@ Admin APIs
 This directory includes documentation for the various synapse specific admin
 APIs available.
 
-Authenticating as a server admin
+Only users that are server admins can use these APIs. A user can be marked as a
---------------------------------
+server admin by updating the database directly, e.g.:
 
-Many of the API calls in the admin api will require an `access_token` for a
+``UPDATE users SET admin = 1 WHERE name = '@foo:bar.com'``
-server admin. (Note that a server admin is distinct from a room admin.)
 
-A user can be marked as a server admin by updating the database directly, e.g.:
+Restarting may be required for the changes to register.
 
-.. code-block:: sql
+Using an admin access_token
-
+###########################
-    UPDATE users SET admin = 1 WHERE name = '@foo:bar.com';
-
-A new server admin user can also be created using the
-``register_new_matrix_user`` script.
 
+Many of the API calls listed in the documentation here will require to include an admin `access_token`.
 Finding your user's `access_token` is client-dependent, but will usually be shown in the client's settings.
 
 Once you have your `access_token`, to include it in a request, the best option is to add the token to a request header:

docs/admin_api/delete_group.md

@@ -4,11 +4,11 @@ This API lets a server admin delete a local group. Doing so will kick all
 users out of the group so that their clients will correctly handle the group
 being deleted.
 
+
 The API is:
 
 ```
 POST /_synapse/admin/v1/delete_group/<group_id>
 ```
 
-To use it, you will need to authenticate by providing an `access_token` for a
+including an `access_token` of a server admin.
-server admin: see [README.rst](README.rst).

docs/admin_api/event_reports.md

@@ -1,172 +0,0 @@
-# Show reported events
-
-This API returns information about reported events.
-
-The api is:
-```
-GET /_synapse/admin/v1/event_reports?from=0&limit=10
-```
-To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [README.rst](README.rst).
-
-It returns a JSON body like the following:
-
-```json
-{
-    "event_reports": [
-        {
-            "event_id": "$bNUFCwGzWca1meCGkjp-zwslF-GfVcXukvRLI1_FaVY",
-            "id": 2,
-            "reason": "foo",
-            "score": -100,
-            "received_ts": 1570897107409,
-            "canonical_alias": "#alias1:matrix.org",
-            "room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org",
-            "name": "Matrix HQ",
-            "sender": "@foobar:matrix.org",
-            "user_id": "@foo:matrix.org"
-        },
-        {
-            "event_id": "$3IcdZsDaN_En-S1DF4EMCy3v4gNRKeOJs8W5qTOKj4I",
-            "id": 3,
-            "reason": "bar",
-            "score": -100,
-            "received_ts": 1598889612059,
-            "canonical_alias": "#alias2:matrix.org",
-            "room_id": "!eGvUQuTCkHGVwNMOjv:matrix.org",
-            "name": "Your room name here",
-            "sender": "@foobar:matrix.org",
-            "user_id": "@bar:matrix.org"
-        }
-    ],
-    "next_token": 2,
-    "total": 4
-}
-```
-
-To paginate, check for `next_token` and if present, call the endpoint again with `from`
-set to the value of `next_token`. This will return a new page.
-
-If the endpoint does not return a `next_token` then there are no more reports to
-paginate through.
-
-**URL parameters:**
-
-* `limit`: integer - Is optional but is used for pagination, denoting the maximum number
-  of items to return in this call. Defaults to `100`.
-* `from`: integer - Is optional but used for pagination, denoting the offset in the
-  returned results. This should be treated as an opaque value and not explicitly set to
-  anything other than the return value of `next_token` from a previous call. Defaults to `0`.
-* `dir`: string - Direction of event report order. Whether to fetch the most recent
-  first (`b`) or the oldest first (`f`). Defaults to `b`.
-* `user_id`: string - Is optional and filters to only return users with user IDs that
-  contain this value. This is the user who reported the event and wrote the reason.
-* `room_id`: string - Is optional and filters to only return rooms with room IDs that
-  contain this value.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-* `id`: integer - ID of event report.
-* `received_ts`: integer - The timestamp (in milliseconds since the unix epoch) when this
-  report was sent.
-* `room_id`: string - The ID of the room in which the event being reported is located.
-* `name`: string - The name of the room.
-* `event_id`: string - The ID of the reported event.
-* `user_id`: string - This is the user who reported the event and wrote the reason.
-* `reason`: string - Comment made by the `user_id` in this report. May be blank.
-* `score`: integer - Content is reported based upon a negative score, where -100 is
-  "most offensive" and 0 is "inoffensive".
-* `sender`: string - This is the ID of the user who sent the original message/event that
-  was reported.
-* `canonical_alias`: string - The canonical alias of the room. `null` if the room does not
-  have a canonical alias set.
-* `next_token`: integer - Indication for pagination. See above.
-* `total`: integer - Total number of event reports related to the query
-  (`user_id` and `room_id`).
-
-# Show details of a specific event report
-
-This API returns information about a specific event report.
-
-The api is:
-```
-GET /_synapse/admin/v1/event_reports/<report_id>
-```
-To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [README.rst](README.rst).
-
-It returns a JSON body like the following:
-
-```jsonc
-{
-    "event_id": "$bNUFCwGzWca1meCGkjp-zwslF-GfVcXukvRLI1_FaVY",
-    "event_json": {
-        "auth_events": [
-            "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M",
-            "$oggsNXxzPFRE3y53SUNd7nsj69-QzKv03a1RucHu-ws"
-        ],
-        "content": {
-            "body": "matrix.org: This Week in Matrix",
-            "format": "org.matrix.custom.html",
-            "formatted_body": "<strong>matrix.org</strong>:<br><a href=\"https://matrix.org/blog/\"><strong>This Week in Matrix</strong></a>",
-            "msgtype": "m.notice"
-        },
-        "depth": 546,
-        "hashes": {
-            "sha256": "xK1//xnmvHJIOvbgXlkI8eEqdvoMmihVDJ9J4SNlsAw"
-        },
-        "origin": "matrix.org",
-        "origin_server_ts": 1592291711430,
-        "prev_events": [
-            "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M"
-        ],
-        "prev_state": [],
-        "room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org",
-        "sender": "@foobar:matrix.org",
-        "signatures": {
-            "matrix.org": {
-                "ed25519:a_JaEG": "cs+OUKW/iHx5pEidbWxh0UiNNHwe46Ai9LwNz+Ah16aWDNszVIe2gaAcVZfvNsBhakQTew51tlKmL2kspXk/Dg"
-            }
-        },
-        "type": "m.room.message",
-        "unsigned": {
-            "age_ts": 1592291711430,
-        }
-    },
-    "id": <report_id>,
-    "reason": "foo",
-    "score": -100,
-    "received_ts": 1570897107409,
-    "canonical_alias": "#alias1:matrix.org",
-    "room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org",
-    "name": "Matrix HQ",
-    "sender": "@foobar:matrix.org",
-    "user_id": "@foo:matrix.org"
-}
-```
-
-**URL parameters:**
-
-* `report_id`: string - The ID of the event report.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-* `id`: integer - ID of event report.
-* `received_ts`: integer - The timestamp (in milliseconds since the unix epoch) when this
-  report was sent.
-* `room_id`: string - The ID of the room in which the event being reported is located.
-* `name`: string - The name of the room.
-* `event_id`: string - The ID of the reported event.
-* `user_id`: string - This is the user who reported the event and wrote the reason.
-* `reason`: string - Comment made by the `user_id` in this report. May be blank.
-* `score`: integer - Content is reported based upon a negative score, where -100 is
-  "most offensive" and 0 is "inoffensive".
-* `sender`: string - This is the ID of the user who sent the original message/event that
-  was reported.
-* `canonical_alias`: string - The canonical alias of the room. `null` if the room does not
-  have a canonical alias set.
-* `event_json`: object - Details of the original event that was reported.

docs/admin_api/media_admin_api.md

@@ -1,53 +1,27 @@
-# Contents
+# List all media in a room
-- [Querying media](#querying-media)
-  * [List all media in a room](#list-all-media-in-a-room)
-  * [List all media uploaded by a user](#list-all-media-uploaded-by-a-user)
-- [Quarantine media](#quarantine-media)
-  * [Quarantining media by ID](#quarantining-media-by-id)
-  * [Quarantining media in a room](#quarantining-media-in-a-room)
-  * [Quarantining all media of a user](#quarantining-all-media-of-a-user)
-  * [Protecting media from being quarantined](#protecting-media-from-being-quarantined)
-- [Delete local media](#delete-local-media)
-  * [Delete a specific local media](#delete-a-specific-local-media)
-  * [Delete local media by date or size](#delete-local-media-by-date-or-size)
-- [Purge Remote Media API](#purge-remote-media-api)
-
-# Querying media
-
-These APIs allow extracting media information from the homeserver.
-
-## List all media in a room
 
 This API gets a list of known media in a room.
-However, it only shows media from unencrypted events or rooms.
 
 The API is:
 ```
 GET /_synapse/admin/v1/room/<room_id>/media
 ```
-To use it, you will need to authenticate by providing an `access_token` for a
+including an `access_token` of a server admin.
-server admin: see [README.rst](README.rst).
 
-The API returns a JSON body like the following:
+It returns a JSON body like the following:
-```json
+```
 {
-  "local": [
+    "local": [
-    "mxc://localhost/xwvutsrqponmlkjihgfedcba",
+        "mxc://localhost/xwvutsrqponmlkjihgfedcba",
-    "mxc://localhost/abcdefghijklmnopqrstuvwx"
+        "mxc://localhost/abcdefghijklmnopqrstuvwx"
-  ],
+    ],
-  "remote": [
+    "remote": [
-    "mxc://matrix.org/xwvutsrqponmlkjihgfedcba",
+        "mxc://matrix.org/xwvutsrqponmlkjihgfedcba",
-    "mxc://matrix.org/abcdefghijklmnopqrstuvwx"
+        "mxc://matrix.org/abcdefghijklmnopqrstuvwx"
-  ]
+    ]
 }
 ```
 
-## List all media uploaded by a user
-
-Listing all media that has been uploaded by a local user can be achieved through
-the use of the [List media of a user](user_admin_api.rst#list-media-of-a-user)
-Admin API.
-
 # Quarantine media
 
 Quarantining media means that it is marked as inaccessible by users. It applies
@@ -72,7 +46,7 @@ form of `abcdefg12345...`.
 
 Response:
 
-```json
+```
 {}
 ```
 
@@ -92,18 +66,14 @@ Where `room_id` is in the form of `!roomid12345:example.org`.
 
 Response:
 
-```json
+```
 {
-  "num_quarantined": 10
+  "num_quarantined": 10  # The number of media items successfully quarantined
 }
 ```
 
-The following fields are returned in the JSON response body:
-
-* `num_quarantined`: integer - The number of media items successfully quarantined
-
 Note that there is a legacy endpoint, `POST
-/_synapse/admin/v1/quarantine_media/<room_id>`, that operates the same.
+/_synapse/admin/v1/quarantine_media/<room_id >`, that operates the same.
 However, it is deprecated and may be removed in a future release.
 
 ## Quarantining all media of a user
@@ -120,155 +90,13 @@ POST /_synapse/admin/v1/user/<user_id>/media/quarantine
 {}
 ```
 
-URL Parameters
+Where `user_id` is in the form of `@bob:example.org`.
-
-* `user_id`: string - User ID in the form of `@bob:example.org`
 
 Response:
 
-```json
+```
 {
-  "num_quarantined": 10
+  "num_quarantined": 10  # The number of media items successfully quarantined
 }
 ```
 
-The following fields are returned in the JSON response body:
-
-* `num_quarantined`: integer - The number of media items successfully quarantined
-
-## Protecting media from being quarantined
-
-This API protects a single piece of local media from being quarantined using the
-above APIs. This is useful for sticker packs and other shared media which you do
-not want to get quarantined, especially when
-[quarantining media in a room](#quarantining-media-in-a-room).
-
-Request:
-
-```
-POST /_synapse/admin/v1/media/protect/<media_id>
-
-{}
-```
-
-Where `media_id` is in the  form of `abcdefg12345...`.
-
-Response:
-
-```json
-{}
-```
-
-# Delete local media
-This API deletes the *local* media from the disk of your own server.
-This includes any local thumbnails and copies of media downloaded from
-remote homeservers.
-This API will not affect media that has been uploaded to external
-media repositories (e.g https://github.com/turt2live/matrix-media-repo/).
-See also [Purge Remote Media API](#purge-remote-media-api).
-
-## Delete a specific local media
-Delete a specific `media_id`.
-
-Request:
-
-```
-DELETE /_synapse/admin/v1/media/<server_name>/<media_id>
-
-{}
-```
-
-URL Parameters
-
-* `server_name`: string - The name of your local server (e.g `matrix.org`)
-* `media_id`: string - The ID of the media (e.g `abcdefghijklmnopqrstuvwx`)
-
-Response:
-
-```json
-{
-  "deleted_media": [
-    "abcdefghijklmnopqrstuvwx"
-  ],
-  "total": 1
-}
-```
-
-The following fields are returned in the JSON response body:
-
-* `deleted_media`: an array of strings - List of deleted `media_id`
-* `total`: integer - Total number of deleted `media_id`
-
-## Delete local media by date or size
-
-Request:
-
-```
-POST /_synapse/admin/v1/media/<server_name>/delete?before_ts=<before_ts>
-
-{}
-```
-
-URL Parameters
-
-* `server_name`: string - The name of your local server (e.g `matrix.org`).
-* `before_ts`: string representing a positive integer - Unix timestamp in ms.
-Files that were last used before this timestamp will be deleted. It is the timestamp of
-last access and not the timestamp creation. 
-* `size_gt`: Optional - string representing a positive integer - Size of the media in bytes.
-Files that are larger will be deleted. Defaults to `0`.
-* `keep_profiles`: Optional - string representing a boolean - Switch to also delete files
-that are still used in image data (e.g user profile, room avatar).
-If `false` these files will be deleted. Defaults to `true`.
-
-Response:
-
-```json
-{
-  "deleted_media": [
-    "abcdefghijklmnopqrstuvwx",
-    "abcdefghijklmnopqrstuvwz"
-  ],
-  "total": 2
-}
-```
-
-The following fields are returned in the JSON response body:
-
-* `deleted_media`: an array of strings - List of deleted `media_id`
-* `total`: integer - Total number of deleted `media_id`
-
-# Purge Remote Media API
-
-The purge remote media API allows server admins to purge old cached remote media.
-
-The API is:
-
-```
-POST /_synapse/admin/v1/purge_media_cache?before_ts=<unix_timestamp_in_ms>
-
-{}
-```
-
-URL Parameters
-
-* `unix_timestamp_in_ms`: string representing a positive integer - Unix timestamp in ms.
-All cached media that was last accessed before this timestamp will be removed.
-
-Response:
-
-```json
-{
-  "deleted": 10
-}
-```
-
-The following fields are returned in the JSON response body:
-
-* `deleted`: integer - The number of media items successfully deleted
-
-To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [README.rst](README.rst).
-
-If the user re-requests purged remote media, synapse will re-request the media
-from the originating server.

docs/admin_api/purge_history_api.rst

@@ -15,8 +15,7 @@ The API is:
 
 ``POST /_synapse/admin/v1/purge_history/<room_id>[/<event_id>]``
 
-To use it, you will need to authenticate by providing an ``access_token`` for a
+including an ``access_token`` of a server admin.
-server admin: see `README.rst <README.rst>`_.
 
 By default, events sent by local users are not deleted, as they may represent
 the only copies of this content in existence. (Events sent by remote users are
@@ -55,10 +54,8 @@ It is possible to poll for updates on recent purges with a second API;
 
 ``GET /_synapse/admin/v1/purge_history_status/<purge_id>``
 
-Again, you will need to authenticate by providing an ``access_token`` for a
+(again, with a suitable ``access_token``). This API returns a JSON body like
-server admin.
+the following:
-
-This API returns a JSON body like the following:
 
 .. code:: json
 

docs/admin_api/purge_remote_media.rst

@@ -0,0 +1,17 @@
+Purge Remote Media API
+======================
+
+The purge remote media API allows server admins to purge old cached remote
+media.
+
+The API is::
+
+    POST /_synapse/admin/v1/purge_media_cache?before_ts=<unix_timestamp_in_ms>&access_token=<access_token>
+
+    {}
+
+Which will remove all cached media that was last accessed before
+``<unix_timestamp_in_ms>``.
+
+If the user re-requests purged remote media, synapse will re-request the media
+from the originating server.

docs/admin_api/purge_room.md

@@ -1,8 +1,5 @@
-Deprecated: Purge room API
+Purge room API
-==========================
+==============
-
-**The old Purge room API is deprecated and will be removed in a future release.
-See the new [Delete Room API](rooms.md#delete-room-api) for more details.**
 
 This API will remove all trace of a room from your database.
 

docs/admin_api/register_api.rst

@@ -18,8 +18,7 @@ To fetch the nonce, you need to request one from the API::
 
 Once you have the nonce, you can make a ``POST`` to the same URL with a JSON
 body containing the nonce, username, password, whether they are an admin
-(optional, False by default), and a HMAC digest of the content. Also you can
+(optional, False by default), and a HMAC digest of the content.
-set the displayname (optional, ``username`` by default).
 
 As an example::
 
@@ -27,7 +26,6 @@ As an example::
   > {
      "nonce": "thisisanonce",
      "username": "pepper_roni",
-     "displayname": "Pepper Roni",
      "password": "pizza",
      "admin": true,
      "mac": "mac_digest_here"

docs/admin_api/room_membership.md

@@ -1,35 +0,0 @@
-# Edit Room Membership API
-
-This API allows an administrator to join an user account with a given `user_id`
-to a room with a given `room_id_or_alias`. You can only modify the membership of
-local users. The server administrator must be in the room and have permission to
-invite users.
-
-## Parameters
-
-The following parameters are available:
-
-* `user_id` - Fully qualified user: for example, `@user:server.com`.
-* `room_id_or_alias` - The room identifier or alias to join: for example,
-  `!636q39766251:server.com`.
-
-## Usage
-
-```
-POST /_synapse/admin/v1/join/<room_id_or_alias>
-
-{
-  "user_id": "@user:server.com"
-}
-```
-
-To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [README.rst](README.rst).
-
-Response:
-
-```
-{
-  "room_id": "!636q39766251:server.com"
-}
-```

docs/admin_api/rooms.md

@@ -1,17 +1,3 @@
-# Contents
-- [List Room API](#list-room-api)
-  * [Parameters](#parameters)
-  * [Usage](#usage)
-- [Room Details API](#room-details-api)
-- [Room Members API](#room-members-api)
-- [Delete Room API](#delete-room-api)
-  * [Parameters](#parameters-1)
-  * [Response](#response)
-  * [Undoing room shutdowns](#undoing-room-shutdowns)
-- [Make Room Admin API](#make-room-admin-api)
-- [Forward Extremities Admin API](#forward-extremities-admin-api)
-- [Event Context API](#event-context-api)
-
 # List Room API
 
 The List Room admin API allows server admins to get a list of rooms on their
@@ -25,21 +11,8 @@ The following query parameters are available:
 * `from` - Offset in the returned list. Defaults to `0`.
 * `limit` - Maximum amount of rooms to return. Defaults to `100`.
 * `order_by` - The method in which to sort the returned list of rooms. Valid values are:
-  - `alphabetical` - Same as `name`. This is deprecated.
+  - `alphabetical` - Rooms are ordered alphabetically by room name. This is the default.
-  - `size` - Same as `joined_members`. This is deprecated.
+  - `size` - Rooms are ordered by the number of members. Largest to smallest.
-  - `name` - Rooms are ordered alphabetically by room name. This is the default.
-  - `canonical_alias` - Rooms are ordered alphabetically by main alias address of the room.
-  - `joined_members` - Rooms are ordered by the number of members. Largest to smallest.
-  - `joined_local_members` - Rooms are ordered by the number of local members. Largest to smallest.
-  - `version` - Rooms are ordered by room version. Largest to smallest.
-  - `creator` - Rooms are ordered alphabetically by creator of the room.
-  - `encryption` - Rooms are ordered alphabetically by the end-to-end encryption algorithm.
-  - `federatable` - Rooms are ordered by whether the room is federatable.
-  - `public` - Rooms are ordered by visibility in room list.
-  - `join_rules` - Rooms are ordered alphabetically by join rules of the room.
-  - `guest_access` - Rooms are ordered alphabetically by guest access option of the room.
-  - `history_visibility` - Rooms are ordered alphabetically by visibility of history of the room.
-  - `state_events` - Rooms are ordered by number of state events. Largest to smallest.
 * `dir` - Direction of room order. Either `f` for forwards or `b` for backwards. Setting
           this value to `b` will reverse the above sort order. Defaults to `f`.
 * `search_term` - Filter rooms by their room name. Search term can be contained in any
@@ -53,16 +26,6 @@ The following fields are possible in the JSON response body:
     - `name` - The name of the room.
     - `canonical_alias` - The canonical (main) alias address of the room.
     - `joined_members` - How many users are currently in the room.
-    - `joined_local_members` - How many local users are currently in the room.
-    - `version` - The version of the room as a string.
-    - `creator` - The `user_id` of the room creator.
-    - `encryption` - Algorithm of end-to-end encryption of messages. Is `null` if encryption is not active.
-    - `federatable` - Whether users on other servers can join this room.
-    - `public` - Whether the room is visible in room directory.
-    - `join_rules` - The type of rules used for users wishing to join this room. One of: ["public", "knock", "invite", "private"].
-    - `guest_access` - Whether guests can join the room. One of: ["can_join", "forbidden"].
-    - `history_visibility` - Who can see the room history. One of: ["invited", "joined", "shared", "world_readable"].
-    - `state_events` - Total number of state_events of a room. Complexity of the room.
 * `offset` - The current pagination offset in rooms. This parameter should be
              used instead of `next_token` for room offset as `next_token` is
              not intended to be parsed.
@@ -90,41 +53,21 @@ GET /_synapse/admin/v1/rooms
 
 Response:
 
-```jsonc
+```
 {
   "rooms": [
     {
       "room_id": "!OGEhHVWSdvArJzumhm:matrix.org",
       "name": "Matrix HQ",
       "canonical_alias": "#matrix:matrix.org",
-      "joined_members": 8326,
+      "joined_members": 8326
-      "joined_local_members": 2,
-      "version": "1",
-      "creator": "@foo:matrix.org",
-      "encryption": null,
-      "federatable": true,
-      "public": true,
-      "join_rules": "invite",
-      "guest_access": null,
-      "history_visibility": "shared",
-      "state_events": 93534
     },
     ... (8 hidden items) ...
     {
       "room_id": "!xYvNcQPhnkrdUmYczI:matrix.org",
       "name": "This Week In Matrix (TWIM)",
       "canonical_alias": "#twim:matrix.org",
-      "joined_members": 314,
+      "joined_members": 314
-      "joined_local_members": 20,
-      "version": "4",
-      "creator": "@foo:matrix.org",
-      "encryption": "m.megolm.v1.aes-sha2",
-      "federatable": true,
-      "public": false,
-      "join_rules": "invite",
-      "guest_access": null,
-      "history_visibility": "shared",
-      "state_events": 8345
     }
   ],
   "offset": 0,
@@ -142,24 +85,14 @@ GET /_synapse/admin/v1/rooms?search_term=TWIM
 
 Response:
 
-```json
+```
 {
   "rooms": [
     {
       "room_id": "!xYvNcQPhnkrdUmYczI:matrix.org",
       "name": "This Week In Matrix (TWIM)",
       "canonical_alias": "#twim:matrix.org",
-      "joined_members": 314,
+      "joined_members": 314
-      "joined_local_members": 20,
-      "version": "4",
-      "creator": "@foo:matrix.org",
-      "encryption": "m.megolm.v1.aes-sha2",
-      "federatable": true,
-      "public": false,
-      "join_rules": "invite",
-      "guest_access": null,
-      "history_visibility": "shared",
-      "state_events": 8
     }
   ],
   "offset": 0,
@@ -177,41 +110,21 @@ GET /_synapse/admin/v1/rooms?order_by=size
 
 Response:
 
-```jsonc
+```
 {
   "rooms": [
     {
       "room_id": "!OGEhHVWSdvArJzumhm:matrix.org",
       "name": "Matrix HQ",
       "canonical_alias": "#matrix:matrix.org",
-      "joined_members": 8326,
+      "joined_members": 8326
-      "joined_local_members": 2,
-      "version": "1",
-      "creator": "@foo:matrix.org",
-      "encryption": null,
-      "federatable": true,
-      "public": true,
-      "join_rules": "invite",
-      "guest_access": null,
-      "history_visibility": "shared",
-      "state_events": 93534
     },
     ... (98 hidden items) ...
     {
       "room_id": "!xYvNcQPhnkrdUmYczI:matrix.org",
       "name": "This Week In Matrix (TWIM)",
       "canonical_alias": "#twim:matrix.org",
-      "joined_members": 314,
+      "joined_members": 314
-      "joined_local_members": 20,
-      "version": "4",
-      "creator": "@foo:matrix.org",
-      "encryption": "m.megolm.v1.aes-sha2",
-      "federatable": true,
-      "public": false,
-      "join_rules": "invite",
-      "guest_access": null,
-      "history_visibility": "shared",
-      "state_events": 8345
     }
   ],
   "offset": 0,
@@ -233,41 +146,21 @@ GET /_synapse/admin/v1/rooms?order_by=size&from=100
 
 Response:
 
-```jsonc
+```
 {
   "rooms": [
     {
       "room_id": "!mscvqgqpHYjBGDxNym:matrix.org",
       "name": "Music Theory",
       "canonical_alias": "#musictheory:matrix.org",
-      "joined_members": 127,
+      "joined_members": 127
-      "joined_local_members": 2,
-      "version": "1",
-      "creator": "@foo:matrix.org",
-      "encryption": null,
-      "federatable": true,
-      "public": true,
-      "join_rules": "invite",
-      "guest_access": null,
-      "history_visibility": "shared",
-      "state_events": 93534
     },
     ... (48 hidden items) ...
     {
       "room_id": "!twcBhHVdZlQWuuxBhN:termina.org.uk",
       "name": "weechat-matrix",
       "canonical_alias": "#weechat-matrix:termina.org.uk",
-      "joined_members": 137,
+      "joined_members": 137
-      "joined_local_members": 20,
-      "version": "4",
-      "creator": "@foo:termina.org.uk",
-      "encryption": null,
-      "federatable": true,
-      "public": true,
-      "join_rules": "invite",
-      "guest_access": null,
-      "history_visibility": "shared",
-      "state_events": 8345
     }
   ],
   "offset": 100,
@@ -278,438 +171,3 @@ Response:
 
 Once the `next_token` parameter is no longer present, we know we've reached the
 end of the list.
-
-# Room Details API
-
-The Room Details admin API allows server admins to get all details of a room.
-
-The following fields are possible in the JSON response body:
-
-* `room_id` - The ID of the room.
-* `name` - The name of the room.
-* `topic` - The topic of the room.
-* `avatar` - The `mxc` URI to the avatar of the room.
-* `canonical_alias` - The canonical (main) alias address of the room.
-* `joined_members` - How many users are currently in the room.
-* `joined_local_members` - How many local users are currently in the room.
-* `joined_local_devices` - How many local devices are currently in the room.
-* `version` - The version of the room as a string.
-* `creator` - The `user_id` of the room creator.
-* `encryption` - Algorithm of end-to-end encryption of messages. Is `null` if encryption is not active.
-* `federatable` - Whether users on other servers can join this room.
-* `public` - Whether the room is visible in room directory.
-* `join_rules` - The type of rules used for users wishing to join this room. One of: ["public", "knock", "invite", "private"].
-* `guest_access` - Whether guests can join the room. One of: ["can_join", "forbidden"].
-* `history_visibility` - Who can see the room history. One of: ["invited", "joined", "shared", "world_readable"].
-* `state_events` - Total number of state_events of a room. Complexity of the room.
-
-## Usage
-
-A standard request:
-
-```
-GET /_synapse/admin/v1/rooms/<room_id>
-
-{}
-```
-
-Response:
-
-```json
-{
-  "room_id": "!mscvqgqpHYjBGDxNym:matrix.org",
-  "name": "Music Theory",
-  "avatar": "mxc://matrix.org/AQDaVFlbkQoErdOgqWRgiGSV",
-  "topic": "Theory, Composition, Notation, Analysis",
-  "canonical_alias": "#musictheory:matrix.org",
-  "joined_members": 127,
-  "joined_local_members": 2,
-  "joined_local_devices": 2,
-  "version": "1",
-  "creator": "@foo:matrix.org",
-  "encryption": null,
-  "federatable": true,
-  "public": true,
-  "join_rules": "invite",
-  "guest_access": null,
-  "history_visibility": "shared",
-  "state_events": 93534
-}
-```
-
-# Room Members API
-
-The Room Members admin API allows server admins to get a list of all members of a room.
-
-The response includes the following fields:
-
-* `members` - A list of all the members that are present in the room, represented by their ids.
-* `total` - Total number of members in the room.
-
-## Usage
-
-A standard request:
-
-```
-GET /_synapse/admin/v1/rooms/<room_id>/members
-
-{}
-```
-
-Response:
-
-```json
-{
-  "members": [
-    "@foo:matrix.org",
-    "@bar:matrix.org",
-    "@foobar:matrix.org"
-  ],
-  "total": 3
-}
-```
-
-# Room State API
-
-The Room State admin API allows server admins to get a list of all state events in a room.
-
-The response includes the following fields:
-
-* `state` - The current state of the room at the time of request.
-
-## Usage
-
-A standard request:
-
-```
-GET /_synapse/admin/v1/rooms/<room_id>/state
-
-{}
-```
-
-Response:
-
-```json
-{
-  "state": [
-    {"type": "m.room.create", "state_key": "", "etc": true},
-    {"type": "m.room.power_levels", "state_key": "", "etc": true},
-    {"type": "m.room.name", "state_key": "", "etc": true}
-  ]
-}
-```
-
-# Delete Room API
-
-The Delete Room admin API allows server admins to remove rooms from server
-and block these rooms.
-
-Shuts down a room. Moves all local users and room aliases automatically to a
-new room if `new_room_user_id` is set. Otherwise local users only
-leave the room without any information.
-
-The new room will be created with the user specified by the `new_room_user_id` parameter
-as room administrator and will contain a message explaining what happened. Users invited
-to the new room will have power level `-10` by default, and thus be unable to speak.
-
-If `block` is `True` it prevents new joins to the old room.
-
-This API will remove all trace of the old room from your database after removing
-all local users. If `purge` is `true` (the default), all traces of the old room will
-be removed from your database after removing all local users. If you do not want
-this to happen, set `purge` to `false`.
-Depending on the amount of history being purged a call to the API may take
-several minutes or longer.
-
-The local server will only have the power to move local user and room aliases to
-the new room. Users on other servers will be unaffected.
-
-The API is:
-
-```
-POST /_synapse/admin/v1/rooms/<room_id>/delete
-```
-
-with a body of:
-```json
-{
-    "new_room_user_id": "@someuser:example.com",
-    "room_name": "Content Violation Notification",
-    "message": "Bad Room has been shutdown due to content violations on this server. Please review our Terms of Service.",
-    "block": true,
-    "purge": true
-}
-```
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see [README.rst](README.rst).
-
-A response body like the following is returned:
-
-```json
-{
-    "kicked_users": [
-        "@foobar:example.com"
-    ],
-    "failed_to_kick_users": [],
-    "local_aliases": [
-        "#badroom:example.com",
-        "#evilsaloon:example.com"
-    ],
-    "new_room_id": "!newroomid:example.com"
-}
-```
-
-## Parameters
-
-The following parameters should be set in the URL:
-
-* `room_id` - The ID of the room.
-
-The following JSON body parameters are available:
-
-* `new_room_user_id` - Optional. If set, a new room will be created with this user ID
-      as the creator and admin, and all users in the old room will be moved into that
-      room. If not set, no new room will be created and the users will just be removed
-      from the old room. The user ID must be on the local server, but does not necessarily
-      have to belong to a registered user.
-* `room_name` - Optional. A string representing the name of the room that new users will be
-                invited to. Defaults to `Content Violation Notification`
-* `message` - Optional. A string containing the first message that will be sent as
-              `new_room_user_id` in the new room. Ideally this will clearly convey why the
-               original room was shut down. Defaults to `Sharing illegal content on this server
-               is not permitted and rooms in violation will be blocked.`
-* `block` - Optional. If set to `true`, this room will be added to a blocking list, preventing
-            future attempts to join the room. Defaults to `false`.
-* `purge` - Optional. If set to `true`, it will remove all traces of the room from your database.
-            Defaults to `true`.
-* `force_purge` - Optional, and ignored unless `purge` is `true`. If set to `true`, it
-  will force a purge to go ahead even if there are local users still in the room. Do not
-  use this unless a regular `purge` operation fails, as it could leave those users'
-  clients in a confused state.
-
-The JSON body must not be empty. The body must be at least `{}`.
-
-## Response
-
-The following fields are returned in the JSON response body:
-
-* `kicked_users` - An array of users (`user_id`) that were kicked.
-* `failed_to_kick_users` - An array of users (`user_id`) that that were not kicked.
-* `local_aliases` - An array of strings representing the local aliases that were migrated from
-                    the old room to the new.
-* `new_room_id` - A string representing the room ID of the new room.
-
-
-## Undoing room shutdowns
-
-*Note*: This guide may be outdated by the time you read it. By nature of room shutdowns being performed at the database level,
-the structure can and does change without notice.
-
-First, it's important to understand that a room shutdown is very destructive. Undoing a shutdown is not as simple as pretending it
-never happened - work has to be done to move forward instead of resetting the past. In fact, in some cases it might not be possible
-to recover at all:
-
-* If the room was invite-only, your users will need to be re-invited.
-* If the room no longer has any members at all, it'll be impossible to rejoin.
-* The first user to rejoin will have to do so via an alias on a different server.
-
-With all that being said, if you still want to try and recover the room:
-
-1. For safety reasons, shut down Synapse.
-2. In the database, run `DELETE FROM blocked_rooms WHERE room_id = '!example:example.org';`
-   * For caution: it's recommended to run this in a transaction: `BEGIN; DELETE ...;`, verify you got 1 result, then `COMMIT;`.
-   * The room ID is the same one supplied to the shutdown room API, not the Content Violation room.
-3. Restart Synapse.
-
-You will have to manually handle, if you so choose, the following:
-
-* Aliases that would have been redirected to the Content Violation room.
-* Users that would have been booted from the room (and will have been force-joined to the Content Violation room).
-* Removal of the Content Violation room if desired.
-
-
-# Make Room Admin API
-
-Grants another user the highest power available to a local user who is in the room.
-If the user is not in the room, and it is not publicly joinable, then invite the user.
-
-By default the server admin (the caller) is granted power, but another user can
-optionally be specified, e.g.:
-
-```
-    POST /_synapse/admin/v1/rooms/<room_id_or_alias>/make_room_admin
-    {
-        "user_id": "@foo:example.com"
-    }
-```
-
-# Forward Extremities Admin API
-
-Enables querying and deleting forward extremities from rooms. When a lot of forward
-extremities accumulate in a room, performance can become degraded. For details, see 
-[#1760](https://github.com/matrix-org/synapse/issues/1760).
-
-## Check for forward extremities
-
-To check the status of forward extremities for a room:
-
-```
-    GET /_synapse/admin/v1/rooms/<room_id_or_alias>/forward_extremities
-```
-
-A response as follows will be returned:
-
-```json
-{
-  "count": 1,
-  "results": [
-    {
-      "event_id": "$M5SP266vsnxctfwFgFLNceaCo3ujhRtg_NiiHabcdefgh",
-      "state_group": 439,
-      "depth": 123,
-      "received_ts": 1611263016761
-    }
-  ]
-}    
-```
-
-## Deleting forward extremities
-
-**WARNING**: Please ensure you know what you're doing and have read 
-the related issue [#1760](https://github.com/matrix-org/synapse/issues/1760).
-Under no situations should this API be executed as an automated maintenance task!
-
-If a room has lots of forward extremities, the extra can be
-deleted as follows:
-
-```
-    DELETE /_synapse/admin/v1/rooms/<room_id_or_alias>/forward_extremities
-```
-
-A response as follows will be returned, indicating the amount of forward extremities
-that were deleted.
-
-```json
-{
-  "deleted": 1
-}
-```
-
-# Event Context API
-
-This API lets a client find the context of an event. This is designed primarily to investigate abuse reports.
-
-```
-GET /_synapse/admin/v1/rooms/<room_id>/context/<event_id>
-```
-
-This API mimmicks [GET /_matrix/client/r0/rooms/{roomId}/context/{eventId}](https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-rooms-roomid-context-eventid). Please refer to the link for all details on parameters and reseponse.
-
-Example response:
-
-```json
-{
-  "end": "t29-57_2_0_2",
-  "events_after": [
-    {
-      "content": {
-        "body": "This is an example text message",
-        "msgtype": "m.text",
-        "format": "org.matrix.custom.html",
-        "formatted_body": "<b>This is an example text message</b>"
-      },
-      "type": "m.room.message",
-      "event_id": "$143273582443PhrSn:example.org",
-      "room_id": "!636q39766251:example.com",
-      "sender": "@example:example.org",
-      "origin_server_ts": 1432735824653,
-      "unsigned": {
-        "age": 1234
-      }
-    }
-  ],
-  "event": {
-    "content": {
-      "body": "filename.jpg",
-      "info": {
-        "h": 398,
-        "w": 394,
-        "mimetype": "image/jpeg",
-        "size": 31037
-      },
-      "url": "mxc://example.org/JWEIFJgwEIhweiWJE",
-      "msgtype": "m.image"
-    },
-    "type": "m.room.message",
-    "event_id": "$f3h4d129462ha:example.com",
-    "room_id": "!636q39766251:example.com",
-    "sender": "@example:example.org",
-    "origin_server_ts": 1432735824653,
-    "unsigned": {
-      "age": 1234
-    }
-  },
-  "events_before": [
-    {
-      "content": {
-        "body": "something-important.doc",
-        "filename": "something-important.doc",
-        "info": {
-          "mimetype": "application/msword",
-          "size": 46144
-        },
-        "msgtype": "m.file",
-        "url": "mxc://example.org/FHyPlCeYUSFFxlgbQYZmoEoe"
-      },
-      "type": "m.room.message",
-      "event_id": "$143273582443PhrSn:example.org",
-      "room_id": "!636q39766251:example.com",
-      "sender": "@example:example.org",
-      "origin_server_ts": 1432735824653,
-      "unsigned": {
-        "age": 1234
-      }
-    }
-  ],
-  "start": "t27-54_2_0_2",
-  "state": [
-    {
-      "content": {
-        "creator": "@example:example.org",
-        "room_version": "1",
-        "m.federate": true,
-        "predecessor": {
-          "event_id": "$something:example.org",
-          "room_id": "!oldroom:example.org"
-        }
-      },
-      "type": "m.room.create",
-      "event_id": "$143273582443PhrSn:example.org",
-      "room_id": "!636q39766251:example.com",
-      "sender": "@example:example.org",
-      "origin_server_ts": 1432735824653,
-      "unsigned": {
-        "age": 1234
-      },
-      "state_key": ""
-    },
-    {
-      "content": {
-        "membership": "join",
-        "avatar_url": "mxc://example.org/SEsfnsuifSDFSSEF",
-        "displayname": "Alice Margatroid"
-      },
-      "type": "m.room.member",
-      "event_id": "$143273582443PhrSn:example.org",
-      "room_id": "!636q39766251:example.com",
-      "sender": "@example:example.org",
-      "origin_server_ts": 1432735824653,
-      "unsigned": {
-        "age": 1234
-      },
-      "state_key": "@alice:example.org"
-    }
-  ]
-}
-```

docs/admin_api/shutdown_room.md

@@ -1,7 +1,4 @@
-# Deprecated: Shutdown room API
+# Shutdown room API
-
-**The old Shutdown room API is deprecated and will be removed in a future release.
-See the new [Delete Room API](rooms.md#delete-room-api) for more details.**
 
 Shuts down a room, preventing new joins and moves local users and room aliases automatically
 to a new room. The new room will be created with the user specified by the
@@ -34,7 +31,7 @@ You will need to authenticate with an access token for an admin user.
 * `message` - Optional. A string containing the first message that will be sent as
               `new_room_user_id` in the new room. Ideally this will clearly convey why the
                original room was shut down.
-
+              
 If not specified, the default value of `room_name` is "Content Violation
 Notification". The default value of `message` is "Sharing illegal content on
 othis server is not permitted and rooms in violation will be blocked."
@@ -73,30 +70,3 @@ Response:
     "new_room_id": "!newroomid:example.com",
 },
 ```
-
-## Undoing room shutdowns
-
-*Note*: This guide may be outdated by the time you read it. By nature of room shutdowns being performed at the database level,
-the structure can and does change without notice.
-
-First, it's important to understand that a room shutdown is very destructive. Undoing a shutdown is not as simple as pretending it
-never happened - work has to be done to move forward instead of resetting the past. In fact, in some cases it might not be possible
-to recover at all:
-
-* If the room was invite-only, your users will need to be re-invited.
-* If the room no longer has any members at all, it'll be impossible to rejoin.
-* The first user to rejoin will have to do so via an alias on a different server.
-
-With all that being said, if you still want to try and recover the room:
-
-1. For safety reasons, shut down Synapse.
-2. In the database, run `DELETE FROM blocked_rooms WHERE room_id = '!example:example.org';`
-   * For caution: it's recommended to run this in a transaction: `BEGIN; DELETE ...;`, verify you got 1 result, then `COMMIT;`.
-   * The room ID is the same one supplied to the shutdown room API, not the Content Violation room.
-3. Restart Synapse.
-
-You will have to manually handle, if you so choose, the following:
-
-* Aliases that would have been redirected to the Content Violation room.
-* Users that would have been booted from the room (and will have been force-joined to the Content Violation room).
-* Removal of the Content Violation room if desired.

docs/admin_api/statistics.md

@@ -1,83 +0,0 @@
-# Users' media usage statistics
-
-Returns information about all local media usage of users. Gives the
-possibility to filter them by time and user.
-
-The API is:
-
-```
-GET /_synapse/admin/v1/statistics/users/media
-```
-
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [README.rst](README.rst).
-
-A response body like the following is returned:
-
-```json
-{
-  "users": [
-    {
-      "displayname": "foo_user_0",
-      "media_count": 2,
-      "media_length": 134,
-      "user_id": "@foo_user_0:test"
-    },
-    {
-      "displayname": "foo_user_1",
-      "media_count": 2,
-      "media_length": 134,
-      "user_id": "@foo_user_1:test"
-    }
-  ],
-  "next_token": 3,
-  "total": 10
-}
-```
-
-To paginate, check for `next_token` and if present, call the endpoint
-again with `from` set to the value of `next_token`. This will return a new page.
-
-If the endpoint does not return a `next_token` then there are no more
-reports to paginate through.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-* `limit`: string representing a positive integer - Is optional but is
-  used for pagination, denoting the maximum number of items to return
-  in this call. Defaults to `100`.
-* `from`: string representing a positive integer - Is optional but used for pagination,
-  denoting the offset in the returned results. This should be treated as an opaque value
-  and not explicitly set to anything other than the return value of `next_token` from a
-  previous call. Defaults to `0`.
-* `order_by` - string - The method in which to sort the returned list of users. Valid values are:
-  - `user_id` - Users are ordered alphabetically by `user_id`. This is the default.
-  - `displayname` - Users are ordered alphabetically by `displayname`.
-  - `media_length` - Users are ordered by the total size of uploaded media in bytes.
-    Smallest to largest.
-  - `media_count` - Users are ordered by number of uploaded media. Smallest to largest.
-* `from_ts` - string representing a positive integer - Considers only
-  files created at this timestamp or later. Unix timestamp in ms.
-* `until_ts` - string representing a positive integer - Considers only
-  files created at this timestamp or earlier. Unix timestamp in ms.
-* `search_term` - string - Filter users by their user ID localpart **or** displayname.
-  The search term can be found in any part of the string.
-  Defaults to no filtering.
-* `dir` - string - Direction of order. Either `f` for forwards or `b` for backwards.
-  Setting this value to `b` will reverse the above sort order. Defaults to `f`.
-
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-* `users` - An array of objects, each containing information
-  about the user and their local media. Objects contain the following fields:
-  - `displayname` - string - Displayname of this user.
-  - `media_count` - integer - Number of uploaded media by this user.
-  - `media_length` - integer - Size of uploaded media in bytes by this user.
-  - `user_id` - string - Fully-qualified user ID (ex. `@user:server.com`).
-* `next_token` - integer - Opaque value used for pagination. See above.
-* `total` - integer - Total number of users after filtering.

docs/admin_api/user_admin_api.rst

@@ -1,53 +1,9 @@
-.. contents::
-
-Query User Account
-==================
-
-This API returns information about a specific user account.
-
-The api is::
-
-    GET /_synapse/admin/v2/users/<user_id>
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-It returns a JSON body like the following:
-
-.. code:: json
-
-    {
-        "displayname": "User",
-        "threepids": [
-            {
-                "medium": "email",
-                "address": "<user_mail_1>"
-            },
-            {
-                "medium": "email",
-                "address": "<user_mail_2>"
-            }
-        ],
-        "avatar_url": "<avatar_url>",
-        "admin": 0,
-        "deactivated": 0,
-        "shadow_banned": 0,
-        "password_hash": "$2b$12$p9B4GkqYdRTPGD",
-        "creation_ts": 1560432506,
-        "appservice_id": null,
-        "consent_server_notice_sent": null,
-        "consent_version": null
-    }
-
-URL parameters:
-
-- ``user_id``: fully-qualified user id: for example, ``@user:server.com``.
-
 Create or modify Account
 ========================
 
 This API allows an administrator to create or modify a user account with a
-specific ``user_id``.
+specific ``user_id``. Be aware that ``user_id`` is fully qualified: for example,
+``@user:server.com``.
 
 This api is::
 
@@ -75,38 +31,16 @@ with a body of:
         "deactivated": false
     }
 
-To use it, you will need to authenticate by providing an ``access_token`` for a
+including an ``access_token`` of a server admin.
-server admin: see `README.rst <README.rst>`_.
-
-URL parameters:
-
-- ``user_id``: fully-qualified user id: for example, ``@user:server.com``.
-
-Body parameters:
-
-- ``password``, optional. If provided, the user's password is updated and all
-  devices are logged out.
-
-- ``displayname``, optional, defaults to the value of ``user_id``.
-
-- ``threepids``, optional, allows setting the third-party IDs (email, msisdn)
-  belonging to a user.
-
-- ``avatar_url``, optional, must be a
-  `MXC URI <https://matrix.org/docs/spec/client_server/r0.6.0#matrix-content-mxc-uris>`_.
-
-- ``admin``, optional, defaults to ``false``.
-
-- ``deactivated``, optional. If unspecified, deactivation state will be left
-  unchanged on existing accounts and set to ``false`` for new accounts.
-  A user cannot be erased by deactivating with this API. For details on deactivating users see
-  `Deactivate Account <#deactivate-account>`_.
 
+The parameter ``displayname`` is optional and defaults to ``user_id``.
+The parameter ``threepids`` is optional.
+The parameter ``avatar_url`` is optional.
+The parameter ``admin`` is optional and defaults to 'false'.
+The parameter ``deactivated`` is optional and defaults to 'false'.
+The parameter ``password`` is optional. If provided the user's password is updated and all devices are logged out.
 If the user already exists then optional parameters default to the current value.
 
-In order to re-activate an account ``deactivated`` must be set to ``false``. If
-users do not login via single-sign-on, a new ``password`` must be provided.
-
 List Accounts
 =============
 
@@ -116,30 +50,17 @@ The api is::
 
     GET /_synapse/admin/v2/users?from=0&limit=10&guests=false
 
-To use it, you will need to authenticate by providing an ``access_token`` for a
+including an ``access_token`` of a server admin.
-server admin: see `README.rst <README.rst>`_.
+The parameters ``from`` and ``limit`` are required only for pagination.
-
+By default, a ``limit`` of 100 is used.
-The parameter ``from`` is optional but used for pagination, denoting the
+The parameter ``user_id`` can be used to select only users with user ids that
-offset in the returned results. This should be treated as an opaque value and
+contain this value.
-not explicitly set to anything other than the return value of ``next_token``
+The parameter ``guests=false`` can be used to exclude guest users,
-from a previous call.
+default is to include guest users.
-
+The parameter ``deactivated=true`` can be used to include deactivated users,
-The parameter ``limit`` is optional but is used for pagination, denoting the
+default is to exclude deactivated users.
-maximum number of items to return in this call. Defaults to ``100``.
+If the endpoint does not return a ``next_token`` then there are no more users left.
-
+It returns a JSON body like the following:
-The parameter ``user_id`` is optional and filters to only return users with user IDs
-that contain this value. This parameter is ignored when using the ``name`` parameter.
-
-The parameter ``name`` is optional and filters to only return users with user ID localparts
-**or** displaynames that contain this value.
-
-The parameter ``guests`` is optional and if ``false`` will **exclude** guest users.
-Defaults to ``true`` to include guest users.
-
-The parameter ``deactivated`` is optional and if ``true`` will **include** deactivated users.
-Defaults to ``false`` to exclude deactivated users.
-
-A JSON body is returned with the following shape:
 
 .. code:: json
 
@@ -147,52 +68,35 @@ A JSON body is returned with the following shape:
         "users": [
             {
                 "name": "<user_id1>",
+                "password_hash": "<password_hash1>",
                 "is_guest": 0,
                 "admin": 0,
                 "user_type": null,
-                "deactivated": 0,
+                "deactivated": 0
-                "shadow_banned": 0,
-                "displayname": "<User One>",
-                "avatar_url": null
             }, {
                 "name": "<user_id2>",
+                "password_hash": "<password_hash2>",
                 "is_guest": 0,
                 "admin": 1,
                 "user_type": null,
-                "deactivated": 0,
+                "deactivated": 0
-                "shadow_banned": 0,
-                "displayname": "<User Two>",
-                "avatar_url": "<avatar_url>"
             }
         ],
-        "next_token": "100",
+        "next_token": "100"
-        "total": 200
     }
 
-To paginate, check for ``next_token`` and if present, call the endpoint again
-with ``from`` set to the value of ``next_token``. This will return a new page.
 
-If the endpoint does not return a ``next_token`` then there are no more users
+Query Account
-to paginate through.
+=============
 
-Query current sessions for a user
+This API returns information about a specific user account.
-=================================
-
-This API returns information about the active sessions for a specific user.
 
 The api is::
 
-    GET /_synapse/admin/v1/whois/<user_id>
+    GET /_synapse/admin/v1/whois/<user_id> (deprecated)
+    GET /_synapse/admin/v2/users/<user_id>
 
-and::
+including an ``access_token`` of a server admin.
-
-    GET /_matrix/client/r0/admin/whois/<userId>
-
-See also: `Client Server API Whois
-<https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-admin-whois-userid>`_
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
 
 It returns a JSON body like the following:
 
@@ -229,11 +133,9 @@ Deactivate Account
 
 This API deactivates an account. It removes active access tokens, resets the
 password, and deletes third-party IDs (to prevent the user requesting a
-password reset).
+password reset). It can also mark the user as GDPR-erased (stopping their data
-
+from distributed further, and deleting it entirely if there are no other
-It can also mark the user as GDPR-erased. This means messages sent by the
+references to it).
-user will still be visible by anyone that was in the room when these messages
-were sent, but hidden from users joining the room afterwards.
 
 The api is::
 
@@ -247,31 +149,11 @@ with a body of:
         "erase": true
     }
 
-To use it, you will need to authenticate by providing an ``access_token`` for a
+including an ``access_token`` of a server admin.
-server admin: see `README.rst <README.rst>`_.
 
-The erase parameter is optional and defaults to ``false``.
+The erase parameter is optional and defaults to 'false'.
 An empty body may be passed for backwards compatibility.
 
-The following actions are performed when deactivating an user:
-
-- Try to unpind 3PIDs from the identity server
-- Remove all 3PIDs from the homeserver
-- Delete all devices and E2EE keys
-- Delete all access tokens
-- Delete the password hash
-- Removal from all rooms the user is a member of
-- Remove the user from the user directory
-- Reject all pending invites
-- Remove all account validity information related to the user
-
-The following additional actions are performed during deactivation if ``erase``
-is set to ``true``:
-
-- Remove the user's display name
-- Remove the user's avatar URL
-- Mark the user as erased
-
 
 Reset password
 ==============
@@ -288,11 +170,10 @@ with a body of:
 
    {
        "new_password": "<secret>",
-       "logout_devices": true
+       "logout_devices": true,
    }
 
-To use it, you will need to authenticate by providing an ``access_token`` for a
+including an ``access_token`` of a server admin.
-server admin: see `README.rst <README.rst>`_.
 
 The parameter ``new_password`` is required.
 The parameter ``logout_devices`` is optional and defaults to ``true``.
@@ -305,8 +186,7 @@ The api is::
 
     GET /_synapse/admin/v1/users/<user_id>/admin
 
-To use it, you will need to authenticate by providing an ``access_token`` for a
+including an ``access_token`` of a server admin.
-server admin: see `README.rst <README.rst>`_.
 
 A response body like the following is returned:
 
@@ -334,492 +214,4 @@ with a body of:
         "admin": true
     }
 
-To use it, you will need to authenticate by providing an ``access_token`` for a
+including an ``access_token`` of a server admin.
-server admin: see `README.rst <README.rst>`_.
-
-
-List room memberships of an user
-================================
-Gets a list of all ``room_id`` that a specific ``user_id`` is member.
-
-The API is::
-
-  GET /_synapse/admin/v1/users/<user_id>/joined_rooms
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-        "joined_rooms": [
-            "!DuGcnbhHGaSZQoNQR:matrix.org",
-            "!ZtSaPCawyWtxfWiIy:matrix.org"
-        ],
-        "total": 2
-    }
-
-The server returns the list of rooms of which the user and the server
-are member. If the user is local, all the rooms of which the user is
-member are returned.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``joined_rooms`` - An array of ``room_id``.
-- ``total`` - Number of rooms.
-
-
-List media of a user
-====================
-Gets a list of all local media that a specific ``user_id`` has created.
-By default, the response is ordered by descending creation date and ascending media ID.
-The newest media is on top. You can change the order with parameters
-``order_by`` and ``dir``.
-
-The API is::
-
-  GET /_synapse/admin/v1/users/<user_id>/media
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-      "media": [
-        {
-          "created_ts": 100400,
-          "last_access_ts": null,
-          "media_id": "qXhyRzulkwLsNHTbpHreuEgo",
-          "media_length": 67,
-          "media_type": "image/png",
-          "quarantined_by": null,
-          "safe_from_quarantine": false,
-          "upload_name": "test1.png"
-        },
-        {
-          "created_ts": 200400,
-          "last_access_ts": null,
-          "media_id": "FHfiSnzoINDatrXHQIXBtahw",
-          "media_length": 67,
-          "media_type": "image/png",
-          "quarantined_by": null,
-          "safe_from_quarantine": false,
-          "upload_name": "test2.png"
-        }
-      ],
-      "next_token": 3,
-      "total": 2
-    }
-
-To paginate, check for ``next_token`` and if present, call the endpoint again
-with ``from`` set to the value of ``next_token``. This will return a new page.
-
-If the endpoint does not return a ``next_token`` then there are no more
-reports to paginate through.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - string - fully qualified: for example, ``@user:server.com``.
-- ``limit``: string representing a positive integer - Is optional but is used for pagination,
-  denoting the maximum number of items to return in this call. Defaults to ``100``.
-- ``from``: string representing a positive integer - Is optional but used for pagination,
-  denoting the offset in the returned results. This should be treated as an opaque value and
-  not explicitly set to anything other than the return value of ``next_token`` from a previous call.
-  Defaults to ``0``.
-- ``order_by`` - The method by which to sort the returned list of media.
-  If the ordered field has duplicates, the second order is always by ascending ``media_id``,
-  which guarantees a stable ordering. Valid values are:
-
-  - ``media_id`` - Media are ordered alphabetically by ``media_id``.
-  - ``upload_name`` - Media are ordered alphabetically by name the media was uploaded with.
-  - ``created_ts`` - Media are ordered by when the content was uploaded in ms.
-    Smallest to largest. This is the default.
-  - ``last_access_ts`` - Media are ordered by when the content was last accessed in ms.
-    Smallest to largest.
-  - ``media_length`` - Media are ordered by length of the media in bytes.
-    Smallest to largest.
-  - ``media_type`` - Media are ordered alphabetically by MIME-type.
-  - ``quarantined_by`` - Media are ordered alphabetically by the user ID that
-    initiated the quarantine request for this media.
-  - ``safe_from_quarantine`` - Media are ordered by the status if this media is safe
-    from quarantining.
-
-- ``dir`` - Direction of media order. Either ``f`` for forwards or ``b`` for backwards.
-  Setting this value to ``b`` will reverse the above sort order. Defaults to ``f``.
-
-If neither ``order_by`` nor ``dir`` is set, the default order is newest media on top
-(corresponds to ``order_by`` = ``created_ts`` and ``dir`` = ``b``).
-
-Caution. The database only has indexes on the columns ``media_id``,
-``user_id`` and ``created_ts``. This means that if a different sort order is used
-(``upload_name``, ``last_access_ts``, ``media_length``, ``media_type``,
-``quarantined_by`` or ``safe_from_quarantine``), this can cause a large load on the
-database, especially for large environments.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``media`` - An array of objects, each containing information about a media.
-  Media objects contain the following fields:
-
-  - ``created_ts`` - integer - Timestamp when the content was uploaded in ms.
-  - ``last_access_ts`` - integer - Timestamp when the content was last accessed in ms.
-  - ``media_id`` - string - The id used to refer to the media.
-  - ``media_length`` - integer - Length of the media in bytes.
-  - ``media_type`` - string - The MIME-type of the media.
-  - ``quarantined_by`` - string - The user ID that initiated the quarantine request
-    for this media.
-
-  - ``safe_from_quarantine`` - bool - Status if this media is safe from quarantining.
-  - ``upload_name`` - string - The name the media was uploaded with.
-
-- ``next_token``: integer - Indication for pagination. See above.
-- ``total`` - integer - Total number of media.
-
-Login as a user
-===============
-
-Get an access token that can be used to authenticate as that user. Useful for
-when admins wish to do actions on behalf of a user.
-
-The API is::
-
-    POST /_synapse/admin/v1/users/<user_id>/login
-    {}
-
-An optional ``valid_until_ms`` field can be specified in the request body as an
-integer timestamp that specifies when the token should expire. By default tokens
-do not expire.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-        "access_token": "<opaque_access_token_string>"
-    }
-
-
-This API does *not* generate a new device for the user, and so will not appear
-their ``/devices`` list, and in general the target user should not be able to
-tell they have been logged in as.
-
-To expire the token call the standard ``/logout`` API with the token.
-
-Note: The token will expire if the *admin* user calls ``/logout/all`` from any
-of their devices, but the token will *not* expire if the target user does the
-same.
-
-
-User devices
-============
-
-List all devices
-----------------
-Gets information about all devices for a specific ``user_id``.
-
-The API is::
-
-  GET /_synapse/admin/v2/users/<user_id>/devices
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-      "devices": [
-        {
-          "device_id": "QBUAZIFURK",
-          "display_name": "android",
-          "last_seen_ip": "1.2.3.4",
-          "last_seen_ts": 1474491775024,
-          "user_id": "<user_id>"
-        },
-        {
-          "device_id": "AUIECTSRND",
-          "display_name": "ios",
-          "last_seen_ip": "1.2.3.5",
-          "last_seen_ts": 1474491775025,
-          "user_id": "<user_id>"
-        }
-      ],
-      "total": 2
-    }
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``devices`` - An array of objects, each containing information about a device.
-  Device objects contain the following fields:
-
-  - ``device_id`` - Identifier of device.
-  - ``display_name`` - Display name set by the user for this device.
-    Absent if no name has been set.
-  - ``last_seen_ip`` - The IP address where this device was last seen.
-    (May be a few minutes out of date, for efficiency reasons).
-  - ``last_seen_ts`` - The timestamp (in milliseconds since the unix epoch) when this
-    devices was last seen. (May be a few minutes out of date, for efficiency reasons).
-  - ``user_id`` - Owner of  device.
-
-- ``total`` - Total number of user's devices.
-
-Delete multiple devices
-------------------
-Deletes the given devices for a specific ``user_id``, and invalidates
-any access token associated with them.
-
-The API is::
-
-    POST /_synapse/admin/v2/users/<user_id>/delete_devices
-
-    {
-      "devices": [
-        "QBUAZIFURK",
-        "AUIECTSRND"
-      ],
-    }
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-An empty JSON dict is returned.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-
-The following fields are required in the JSON request body:
-
-- ``devices`` - The list of device IDs to delete.
-
-Show a device
----------------
-Gets information on a single device, by ``device_id`` for a specific ``user_id``.
-
-The API is::
-
-    GET /_synapse/admin/v2/users/<user_id>/devices/<device_id>
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-      "device_id": "<device_id>",
-      "display_name": "android",
-      "last_seen_ip": "1.2.3.4",
-      "last_seen_ts": 1474491775024,
-      "user_id": "<user_id>"
-    }
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-- ``device_id`` - The device to retrieve.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``device_id`` - Identifier of device.
-- ``display_name`` - Display name set by the user for this device.
-  Absent if no name has been set.
-- ``last_seen_ip`` - The IP address where this device was last seen.
-  (May be a few minutes out of date, for efficiency reasons).
-- ``last_seen_ts`` - The timestamp (in milliseconds since the unix epoch) when this
-  devices was last seen. (May be a few minutes out of date, for efficiency reasons).
-- ``user_id`` - Owner of  device.
-
-Update a device
----------------
-Updates the metadata on the given ``device_id`` for a specific ``user_id``.
-
-The API is::
-
-    PUT /_synapse/admin/v2/users/<user_id>/devices/<device_id>
-
-    {
-      "display_name": "My other phone"
-    }
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-An empty JSON dict is returned.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-- ``device_id`` - The device to update.
-
-The following fields are required in the JSON request body:
-
-- ``display_name`` - The new display name for this device. If not given,
-  the display name is unchanged.
-
-Delete a device
----------------
-Deletes the given ``device_id`` for a specific ``user_id``,
-and invalidates any access token associated with it.
-
-The API is::
-
-    DELETE /_synapse/admin/v2/users/<user_id>/devices/<device_id>
-
-    {}
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-An empty JSON dict is returned.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-- ``device_id`` - The device to delete.
-
-List all pushers
-================
-Gets information about all pushers for a specific ``user_id``.
-
-The API is::
-
-  GET /_synapse/admin/v1/users/<user_id>/pushers
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-      "pushers": [
-        {
-          "app_display_name":"HTTP Push Notifications",
-          "app_id":"m.http",
-          "data": {
-            "url":"example.com"
-          },
-          "device_display_name":"pushy push",
-          "kind":"http",
-          "lang":"None",
-          "profile_tag":"",
-          "pushkey":"a@example.com"
-        }
-      ],
-      "total": 1
-    }
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``pushers`` - An array containing the current pushers for the user
-
-  - ``app_display_name`` - string - A string that will allow the user to identify
-    what application owns this pusher.
-
-  - ``app_id`` - string - This is a reverse-DNS style identifier for the application.
-    Max length, 64 chars.
-
-  - ``data`` - A dictionary of information for the pusher implementation itself.
-
-    - ``url`` - string - Required if ``kind`` is ``http``. The URL to use to send
-      notifications to.
-
-    - ``format`` - string - The format to use when sending notifications to the
-      Push Gateway.
-
-  - ``device_display_name`` - string -  A string that will allow the user to identify
-    what device owns this pusher.
-
-  - ``profile_tag`` - string - This string determines which set of device specific rules
-    this pusher executes.
-
-  - ``kind`` - string -  The kind of pusher. "http" is a pusher that sends HTTP pokes.
-  - ``lang`` - string - The preferred language for receiving notifications
-    (e.g. 'en' or 'en-US')
-
-  - ``profile_tag`` - string - This string determines which set of device specific rules
-    this pusher executes.
-
-  - ``pushkey`` - string - This is a unique identifier for this pusher.
-    Max length, 512 bytes.
-
-- ``total`` - integer - Number of pushers.
-
-See also `Client-Server API Spec <https://matrix.org/docs/spec/client_server/latest#get-matrix-client-r0-pushers>`_
-
-Shadow-banning users
-====================
-
-Shadow-banning is a useful tool for moderating malicious or egregiously abusive users.
-A shadow-banned users receives successful responses to their client-server API requests,
-but the events are not propagated into rooms. This can be an effective tool as it
-(hopefully) takes longer for the user to realise they are being moderated before
-pivoting to another account.
-
-Shadow-banning a user should be used as a tool of last resort and may lead to confusing
-or broken behaviour for the client. A shadow-banned user will not receive any
-notification and it is generally more appropriate to ban or kick abusive users.
-A shadow-banned user will be unable to contact anyone on the server.
-
-The API is::
-
-  POST /_synapse/admin/v1/users/<user_id>/shadow_ban
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-An empty JSON dict is returned.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - The fully qualified MXID: for example, ``@user:server.com``. The user must
-  be local.

docs/application_services.md

@@ -23,13 +23,9 @@ namespaces:
   users:  # List of users we're interested in
     - exclusive: <bool>
       regex: <regex>
-      group_id: <group>
     - ...
   aliases: []  # List of aliases we're interested in
   rooms: [] # List of room ids we're interested in
 ```
 
-`exclusive`: If enabled, only this application service is allowed to register users in its namespace(s).
-`group_id`: All users of this application service are dynamically joined to this group. This is useful for e.g user organisation or flairs.
-
 See the [spec](https://matrix.org/docs/spec/application_service/unstable.html) for further details on how application services work.

docs/auth_chain_diff.dot

@@ -1,32 +0,0 @@
-digraph auth {
-    nodesep=0.5;
-    rankdir="RL";
-
-    C [label="Create (1,1)"];
-
-    BJ [label="Bob's Join (2,1)", color=red];
-    BJ2 [label="Bob's Join (2,2)", color=red];
-    BJ2 -> BJ [color=red, dir=none];
-
-    subgraph cluster_foo {
-        A1 [label="Alice's invite (4,1)", color=blue];
-        A2 [label="Alice's Join (4,2)", color=blue];
-        A3 [label="Alice's Join (4,3)", color=blue];
-        A3 -> A2 -> A1 [color=blue, dir=none];
-        color=none;
-    }
-
-    PL1 [label="Power Level (3,1)", color=darkgreen];
-    PL2 [label="Power Level (3,2)", color=darkgreen];
-    PL2 -> PL1 [color=darkgreen, dir=none];
-
-    {rank = same; C; BJ; PL1; A1;}
-
-    A1 -> C [color=grey];
-    A1 -> BJ [color=grey];
-    PL1 -> C [color=grey];
-    BJ2 -> PL1 [penwidth=2];
-
-    A3 -> PL2 [penwidth=2];
-    A1 -> PL1 -> BJ -> C [penwidth=2];
-}

docs/auth_chain_difference_algorithm.md

@@ -1,108 +0,0 @@
-# Auth Chain Difference Algorithm
-
-The auth chain difference algorithm is used by V2 state resolution, where a
-naive implementation can be a significant source of CPU and DB usage.
-
-### Definitions
-
-A *state set* is a set of state events; e.g. the input of a state resolution
-algorithm is a collection of state sets.
-
-The *auth chain* of a set of events are all the events' auth events and *their*
-auth events, recursively (i.e. the events reachable by walking the graph induced
-by an event's auth events links).
-
-The *auth chain difference* of a collection of state sets is the union minus the
-intersection of the sets of auth chains corresponding to the state sets, i.e an
-event is in the auth chain difference if it is reachable by walking the auth
-event graph from at least one of the state sets but not from *all* of the state
-sets.
-
-## Breadth First Walk Algorithm
-
-A way of calculating the auth chain difference without calculating the full auth
-chains for each state set is to do a parallel breadth first walk (ordered by
-depth) of each state set's auth chain. By tracking which events are reachable
-from each state set we can finish early if every pending event is reachable from
-every state set.
-
-This can work well for state sets that have a small auth chain difference, but
-can be very inefficient for larger differences. However, this algorithm is still
-used if we don't have a chain cover index for the room (e.g. because we're in
-the process of indexing it).
-
-## Chain Cover Index
-
-Synapse computes auth chain differences by pre-computing a "chain cover" index
-for the auth chain in a room, allowing efficient reachability queries like "is
-event A in the auth chain of event B". This is done by assigning every event a
-*chain ID* and *sequence number* (e.g. `(5,3)`), and having a map of *links*
-between chains (e.g. `(5,3) -> (2,4)`) such that A is reachable by B (i.e. `A`
-is in the auth chain of `B`) if and only if either:
-
-1. A and B have the same chain ID and `A`'s sequence number is less than `B`'s
-   sequence number; or
-2. there is a link `L` between `B`'s chain ID and `A`'s chain ID such that
-   `L.start_seq_no` <= `B.seq_no` and `A.seq_no` <= `L.end_seq_no`.
-
-There are actually two potential implementations, one where we store links from
-each chain to every other reachable chain (the transitive closure of the links
-graph), and one where we remove redundant links (the transitive reduction of the
-links graph) e.g. if we have chains `C3 -> C2 -> C1` then the link `C3 -> C1`
-would not be stored. Synapse uses the former implementations so that it doesn't
-need to recurse to test reachability between chains.
-
-### Example
-
-An example auth graph would look like the following, where chains have been
-formed based on type/state_key and are denoted by colour and are labelled with
-`(chain ID, sequence number)`. Links are denoted by the arrows (links in grey
-are those that would be remove in the second implementation described above).
-
-![Example](auth_chain_diff.dot.png)
-
-Note that we don't include all links between events and their auth events, as
-most of those links would be redundant. For example, all events point to the
-create event, but each chain only needs the one link from it's base to the
-create event.
-
-## Using the Index
-
-This index can be used to calculate the auth chain difference of the state sets
-by looking at the chain ID and sequence numbers reachable from each state set:
-
-1. For every state set lookup the chain ID/sequence numbers of each state event
-2. Use the index to find all chains and the maximum sequence number reachable
-   from each state set.
-3. The auth chain difference is then all events in each chain that have sequence
-   numbers between the maximum sequence number reachable from *any* state set and
-   the minimum reachable by *all* state sets (if any).
-
-Note that steps 2 is effectively calculating the auth chain for each state set
-(in terms of chain IDs and sequence numbers), and step 3 is calculating the
-difference between the union and intersection of the auth chains.
-
-### Worked Example
-
-For example, given the above graph, we can calculate the difference between
-state sets consisting of:
-
-1. `S1`: Alice's invite `(4,1)` and Bob's second join `(2,2)`; and
-2. `S2`: Alice's second join `(4,3)` and Bob's first join `(2,1)`.
-
-Using the index we see that the following auth chains are reachable from each
-state set:
-
-1. `S1`: `(1,1)`, `(2,2)`, `(3,1)` & `(4,1)`
-2. `S2`: `(1,1)`, `(2,1)`, `(3,2)` & `(4,3)`
-
-And so, for each the ranges that are in the auth chain difference:
-1. Chain 1: None, (since everything can reach the create event).
-2. Chain 2: The range `(1, 2]` (i.e. just `2`), as `1` is reachable by all state
-   sets and the maximum reachable is `2` (corresponding to Bob's second join).
-3. Chain 3: Similarly the range `(1, 2]` (corresponding to the second power
-   level).
-4. Chain 4: The range `(1, 3]` (corresponding to both of Alice's joins).
-
-So the final result is: Bob's second join `(2,2)`, the second power level
-`(3,2)` and both of Alice's joins `(4,2)` & `(4,3)`.

docs/code_style.md

@@ -8,16 +8,16 @@ errors in code.
 
 The necessary tools are detailed below.
 
-First install them with:
-
-    pip install -e ".[lint,mypy]"
-
 -   **black**
 
     The Synapse codebase uses [black](https://pypi.org/project/black/)
     as an opinionated code formatter, ensuring all comitted code is
     properly formatted.
 
+    First install `black` with:
+
+        pip install --upgrade black
+
     Have `black` auto-format your code (it shouldn't change any
     functionality) with:
 
@@ -28,6 +28,10 @@ First install them with:
     `flake8` is a code checking tool. We require code to pass `flake8`
     before being merged into the codebase.
 
+    Install `flake8` with:
+
+        pip install --upgrade flake8 flake8-comprehensions
+
     Check all application and test code with:
 
         flake8 synapse tests
@@ -37,6 +41,10 @@ First install them with:
     `isort` ensures imports are nicely formatted, and can suggest and
     auto-fix issues such as double-importing.
 
+    Install `isort` with:
+
+        pip install --upgrade isort
+
     Auto-fix imports with:
 
         isort -rc synapse tests
@@ -56,6 +64,8 @@ save as it takes a while and is very resource intensive.
     -   Use underscores for functions and variables.
 -   **Docstrings**: should follow the [google code
     style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings).
+    This is so that we can generate documentation with
+    [sphinx](http://sphinxcontrib-napoleon.readthedocs.org/en/latest/).
     See the
     [examples](http://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html)
     in the sphinx documentation.

docs/dev/cas.md

@@ -1,64 +0,0 @@
-# How to test CAS as a developer without a server
-
-The [django-mama-cas](https://github.com/jbittel/django-mama-cas) project is an
-easy to run CAS implementation built on top of Django.
-
-## Prerequisites
-
-1. Create a new virtualenv: `python3 -m venv <your virtualenv>`
-2. Activate your virtualenv: `source /path/to/your/virtualenv/bin/activate`
-3. Install Django and django-mama-cas:
-   ```
-   python -m pip install "django<3" "django-mama-cas==2.4.0"
-   ```
-4. Create a Django project in the current directory:
-   ```
-   django-admin startproject cas_test .
-   ```
-5. Follow the [install directions](https://django-mama-cas.readthedocs.io/en/latest/installation.html#configuring) for django-mama-cas
-6. Setup the SQLite database: `python manage.py migrate`
-7. Create a user:
-   ```
-   python manage.py createsuperuser
-   ```
-   1. Use whatever you want as the username and password.
-   2. Leave the other fields blank.
-8. Use the built-in Django test server to serve the CAS endpoints on port 8000:
-   ```
-   python manage.py runserver
-   ```
-
-You should now have a Django project configured to serve CAS authentication with
-a single user created.
-
-## Configure Synapse (and Element) to use CAS
-
-1. Modify your `homeserver.yaml` to enable CAS and point it to your locally
-   running Django test server:
-   ```yaml
-   cas_config:
-     enabled: true
-     server_url: "http://localhost:8000"
-     service_url: "http://localhost:8081"
-     #displayname_attribute: name
-     #required_attributes:
-     #    name: value
-   ```
-2. Restart Synapse.
-
-Note that the above configuration assumes the homeserver is running on port 8081
-and that the CAS server is on port 8000, both on localhost.
-
-## Testing the configuration
-
-Then in Element:
-
-1. Visit the login page with a Element pointing at your homeserver.
-2. Click the Single Sign-On button.
-3. Login using the credentials created with `createsuperuser`.
-4. You should be logged in.
-
-If you want to repeat this process you'll need to manually logout first:
-
-1. http://localhost:8000/admin/
-2. Click "logout" in the top right.

docs/dev/git.md

@@ -1,148 +0,0 @@
-Some notes on how we use git
-============================
-
-On keeping the commit history clean
------------------------------------
-
-In an ideal world, our git commit history would be a linear progression of
-commits each of which contains a single change building on what came
-before. Here, by way of an arbitrary example, is the top of `git log --graph
-b2dba0607`:
-
-<img src="git/clean.png" alt="clean git graph" width="500px">
-
-Note how the commit comment explains clearly what is changing and why. Also
-note the *absence* of merge commits, as well as the absence of commits called
-things like (to pick a few culprits):
-[“pep8”](https://github.com/matrix-org/synapse/commit/84691da6c), [“fix broken
-test”](https://github.com/matrix-org/synapse/commit/474810d9d),
-[“oops”](https://github.com/matrix-org/synapse/commit/c9d72e457),
-[“typo”](https://github.com/matrix-org/synapse/commit/836358823), or [“Who's
-the president?”](https://github.com/matrix-org/synapse/commit/707374d5d).
-
-There are a number of reasons why keeping a clean commit history is a good
-thing:
-
- * From time to time, after a change lands, it turns out to be necessary to
-   revert it, or to backport it to a release branch. Those operations are
-   *much* easier when the change is contained in a single commit.
-
- * Similarly, it's much easier to answer questions like “is the fix for
-   `/publicRooms` on the release branch?” if that change consists of a single
-   commit.
-
- * Likewise: “what has changed on this branch in the last week?” is much
-   clearer without merges and “pep8” commits everywhere.
-
- * Sometimes we need to figure out where a bug got introduced, or some
-   behaviour changed. One way of doing that is with `git bisect`: pick an
-   arbitrary commit between the known good point and the known bad point, and
-   see how the code behaves. However, that strategy fails if the commit you
-   chose is the middle of someone's epic branch in which they broke the world
-   before putting it back together again.
-
-One counterargument is that it is sometimes useful to see how a PR evolved as
-it went through review cycles. This is true, but that information is always
-available via the GitHub UI (or via the little-known [refs/pull
-namespace](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/checking-out-pull-requests-locally)).
-
-
-Of course, in reality, things are more complicated than that. We have release
-branches as well as `develop` and `master`, and we deliberately merge changes
-between them. Bugs often slip through and have to be fixed later. That's all
-fine: this not a cast-iron rule which must be obeyed, but an ideal to aim
-towards.
-
-Merges, squashes, rebases: wtf?
--------------------------------
-
-Ok, so that's what we'd like to achieve. How do we achieve it?
-
-The TL;DR is: when you come to merge a pull request, you *probably* want to
-“squash and merge”:
-
-![squash and merge](git/squash.png).
-
-(This applies whether you are merging your own PR, or that of another
-contributor.)
-
-“Squash and merge”<sup id="a1">[1](#f1)</sup> takes all of the changes in the
-PR, and bundles them into a single commit. GitHub gives you the opportunity to
-edit the commit message before you confirm, and normally you should do so,
-because the default will be useless (again: `* woops typo` is not a useful
-thing to keep in the historical record).
-
-The main problem with this approach comes when you have a series of pull
-requests which build on top of one another: as soon as you squash-merge the
-first PR, you'll end up with a stack of conflicts to resolve in all of the
-others. In general, it's best to avoid this situation in the first place by
-trying not to have multiple related PRs in flight at the same time. Still,
-sometimes that's not possible and doing a regular merge is the lesser evil.
-
-Another occasion in which a regular merge makes more sense is a PR where you've
-deliberately created a series of commits each of which makes sense in its own
-right. For example: [a PR which gradually propagates a refactoring operation
-through the codebase](https://github.com/matrix-org/synapse/pull/6837), or [a
-PR which is the culmination of several other
-PRs](https://github.com/matrix-org/synapse/pull/5987). In this case the ability
-to figure out when a particular change/bug was introduced could be very useful.
-
-Ultimately: **this is not a hard-and-fast-rule**. If in doubt, ask yourself “do
-each of the commits I am about to merge make sense in their own right”, but
-remember that we're just doing our best to balance “keeping the commit history
-clean” with other factors.
-
-Git branching model
--------------------
-
-A [lot](https://nvie.com/posts/a-successful-git-branching-model/)
-[of](http://scottchacon.com/2011/08/31/github-flow.html)
-[words](https://www.endoflineblog.com/gitflow-considered-harmful) have been
-written in the past about git branching models (no really, [a
-lot](https://martinfowler.com/articles/branching-patterns.html)). I tend to
-think the whole thing is overblown. Fundamentally, it's not that
-complicated. Here's how we do it.
-
-Let's start with a picture:
-
-![branching model](git/branches.jpg)
-
-It looks complicated, but it's really not. There's one basic rule: *anyone* is
-free to merge from *any* more-stable branch to *any* less-stable branch at
-*any* time<sup id="a2">[2](#f2)</sup>. (The principle behind this is that if a
-change is good enough for the more-stable branch, then it's also good enough go
-put in a less-stable branch.)
-
-Meanwhile, merging (or squashing, as per the above) from a less-stable to a
-more-stable branch is a deliberate action in which you want to publish a change
-or a set of changes to (some subset of) the world: for example, this happens
-when a PR is landed, or as part of our release process.
-
-So, what counts as a more- or less-stable branch? A little reflection will show
-that our active branches are ordered thus, from more-stable to less-stable:
-
- * `master` (tracks our last release).
- * `release-vX.Y.Z` (the branch where we prepare the next release)<sup
-   id="a3">[3](#f3)</sup>.
- * PR branches which are targeting the release.
- * `develop` (our "mainline" branch containing our bleeding-edge).
- * regular PR branches.
-
-The corollary is: if you have a bugfix that needs to land in both
-`release-vX.Y.Z` *and* `develop`, then you should base your PR on
-`release-vX.Y.Z`, get it merged there, and then merge from `release-vX.Y.Z` to
-`develop`. (If a fix lands in `develop` and we later need it in a
-release-branch, we can of course cherry-pick it, but landing it in the release
-branch first helps reduce the chance of annoying conflicts.)
-
----
-
-<b id="f1">[1]</b>: “Squash and merge” is GitHub's term for this
-operation. Given that there is no merge involved, I'm not convinced it's the
-most intuitive name. [^](#a1)
-
-<b id="f2">[2]</b>: Well, anyone with commit access.[^](#a2)
-
-<b id="f3">[3]</b>: Very, very occasionally (I think this has happened once in
-the history of Synapse), we've had two releases in flight at once. Obviously,
-`release-v1.2.3` is more-stable than `release-v1.3.0`. [^](#a3)

docs/dev/saml.md

@@ -18,13 +18,9 @@ To make Synapse (and therefore Riot) use it:
        metadata:
          local: ["samling.xml"]   
    ```
-5. Ensure that your `homeserver.yaml` has a setting for `public_baseurl`:
+5. Run `apt-get install xmlsec1` and `pip install --upgrade --force 'pysaml2>=4.5.0'` to ensure
-   ```yaml
-   public_baseurl: http://localhost:8080/
-   ```
-6. Run `apt-get install xmlsec1` and `pip install --upgrade --force 'pysaml2>=4.5.0'` to ensure
    the dependencies are installed and ready to go.
-7. Restart Synapse.
+6. Restart Synapse.
 
 Then in Riot:
 

docs/federate.md

@@ -47,18 +47,6 @@ you invite them to. This can be caused by an incorrectly-configured reverse
 proxy: see [reverse_proxy.md](<reverse_proxy.md>) for instructions on how to correctly
 configure a reverse proxy.
 
-### Known issues
-
-**HTTP `308 Permanent Redirect` redirects are not followed**: Due to missing features
-in the HTTP library used by Synapse, 308 redirects are currently not followed by
-federating servers, which can cause `M_UNKNOWN` or `401 Unauthorized` errors. This
-may affect users who are redirecting apex-to-www (e.g. `example.com` -> `www.example.com`),
-and especially users of the Kubernetes *Nginx Ingress* module, which uses 308 redirect
-codes by default. For those Kubernetes users, [this Stackoverflow post](https://stackoverflow.com/a/52617528/5096871) 
-might be helpful. For other users, switching to a `301 Moved Permanently` code may be
-an option. 308 redirect codes will be supported properly in a future
-release of Synapse.
-
 ## Running a demo federation of Synapses
 
 If you want to get up and running quickly with a trio of homeservers in a

docs/jwt.md

@@ -1,97 +0,0 @@
-# JWT Login Type
-
-Synapse comes with a non-standard login type to support
-[JSON Web Tokens](https://en.wikipedia.org/wiki/JSON_Web_Token). In general the
-documentation for
-[the login endpoint](https://matrix.org/docs/spec/client_server/r0.6.1#login)
-is still valid (and the mechanism works similarly to the
-[token based login](https://matrix.org/docs/spec/client_server/r0.6.1#token-based)).
-
-To log in using a JSON Web Token, clients should submit a `/login` request as
-follows:
-
-```json
-{
-  "type": "org.matrix.login.jwt",
-  "token": "<jwt>"
-}
-```
-
-Note that the login type of `m.login.jwt` is supported, but is deprecated. This
-will be removed in a future version of Synapse.
-
-The `token` field should include the JSON web token with the following claims:
-
-* The `sub` (subject) claim is required and should encode the local part of the
-  user ID.
-* The expiration time (`exp`), not before time (`nbf`), and issued at (`iat`)
-  claims are optional, but validated if present.
-* The issuer (`iss`) claim is optional, but required and validated if configured.
-* The audience (`aud`) claim is optional, but required and validated if configured.
-  Providing the audience claim when not configured will cause validation to fail.
-
-In the case that the token is not valid, the homeserver must respond with
-`403 Forbidden` and an error code of `M_FORBIDDEN`.
-
-As with other login types, there are additional fields (e.g. `device_id` and
-`initial_device_display_name`) which can be included in the above request.
-
-## Preparing Synapse
-
-The JSON Web Token integration in Synapse uses the
-[`PyJWT`](https://pypi.org/project/pyjwt/) library, which must be installed
-as follows:
-
- * The relevant libraries are included in the Docker images and Debian packages
-   provided by `matrix.org` so no further action is needed.
-
- * If you installed Synapse into a virtualenv, run `/path/to/env/bin/pip
-   install synapse[pyjwt]` to install the necessary dependencies.
-
- * For other installation mechanisms, see the documentation provided by the
-   maintainer.
-
-To enable the JSON web token integration, you should then add an `jwt_config` section
-to your configuration file (or uncomment the `enabled: true` line in the
-existing section). See [sample_config.yaml](./sample_config.yaml) for some
-sample settings.
-
-## How to test JWT as a developer
-
-Although JSON Web Tokens are typically generated from an external server, the
-examples below use [PyJWT](https://pyjwt.readthedocs.io/en/latest/) directly.
-
-1.  Configure Synapse with JWT logins, note that this example uses a pre-shared
-    secret and an algorithm of HS256:
-
-    ```yaml
-    jwt_config:
-        enabled: true
-        secret: "my-secret-token"
-        algorithm: "HS256"
-    ```
-2.  Generate a JSON web token:
-
-    ```bash
-    $ pyjwt --key=my-secret-token --alg=HS256 encode sub=test-user
-    eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ0ZXN0LXVzZXIifQ.Ag71GT8v01UO3w80aqRPTeuVPBIBZkYhNTJJ-_-zQIc
-    ```
-3.  Query for the login types and ensure `org.matrix.login.jwt` is there:
-
-    ```bash
-    curl http://localhost:8080/_matrix/client/r0/login
-    ```
-4.  Login used the generated JSON web token from above:
-
-    ```bash
-    $ curl http://localhost:8082/_matrix/client/r0/login -X POST \
-        --data '{"type":"org.matrix.login.jwt","token":"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ0ZXN0LXVzZXIifQ.Ag71GT8v01UO3w80aqRPTeuVPBIBZkYhNTJJ-_-zQIc"}'
-    {
-        "access_token": "<access token>",
-        "device_id": "ACBDEFGHI",
-        "home_server": "localhost:8080",
-        "user_id": "@test-user:localhost:8480"
-    }
-    ```
-
-You should now be able to use the returned access token to query the client API.

docs/log_contexts.md

@@ -29,13 +29,14 @@ from synapse.logging import context         # omitted from future snippets
 def handle_request(request_id):
     request_context = context.LoggingContext()
 
-    calling_context = context.set_current_context(request_context)
+    calling_context = context.LoggingContext.current_context()
+    context.LoggingContext.set_current_context(request_context)
     try:
         request_context.request = request_id
         do_request_handling()
         logger.debug("finished")
     finally:
-        context.set_current_context(calling_context)
+        context.LoggingContext.set_current_context(calling_context)
 
 def do_request_handling():
     logger.debug("phew")  # this will be logged against request_id

docs/manhole.md

@@ -5,45 +5,8 @@ The "manhole" allows server administrators to access a Python shell on a running
 Synapse installation. This is a very powerful mechanism for administration and
 debugging.
 
-**_Security Warning_**
-
-Note that this will give administrative access to synapse to **all users** with
-shell access to the server. It should therefore **not** be enabled in
-environments where untrusted users have shell access.
-
-***
-
 To enable it, first uncomment the `manhole` listener configuration in
-`homeserver.yaml`. The configuration is slightly different if you're using docker.
+`homeserver.yaml`:
-
-#### Docker config
-
-If you are using Docker, set `bind_addresses` to `['0.0.0.0']` as shown:
-
-```yaml
-listeners:
-  - port: 9000
-    bind_addresses: ['0.0.0.0']
-    type: manhole
-```
-
-When using `docker run` to start the server, you will then need to change the command to the following to include the
-`manhole` port forwarding. The `-p 127.0.0.1:9000:9000` below is important: it 
-ensures that access to the `manhole` is only possible for local users.
-
-```bash
-docker run -d --name synapse \
-    --mount type=volume,src=synapse-data,dst=/data \
-    -p 8008:8008 \
-    -p 127.0.0.1:9000:9000 \
-    matrixdotorg/synapse:latest
-```
-
-#### Native config
-
-If you are not using docker, set `bind_addresses` to `['::1', '127.0.0.1']` as shown.
-The `bind_addresses` in the example below is important: it ensures that access to the
-`manhole` is only possible for local users).
 
 ```yaml
 listeners:
@@ -52,7 +15,12 @@ listeners:
     type: manhole
 ```
 
-#### Accessing synapse manhole
+(`bind_addresses` in the above is important: it ensures that access to the
+manhole is only possible for local users).
+
+Note that this will give administrative access to synapse to **all users** with
+shell access to the server. It should therefore **not** be enabled in
+environments where untrusted users have shell access.
 
 Then restart synapse, and point an ssh client at port 9000 on localhost, using
 the username `matrix`:
@@ -67,12 +35,9 @@ This gives a Python REPL in which `hs` gives access to the
 `synapse.server.HomeServer` object - which in turn gives access to many other
 parts of the process.
 
-Note that any call which returns a coroutine will need to be wrapped in `ensureDeferred`.
-
 As a simple example, retrieving an event from the database:
 
-```pycon
+```
->>> from twisted.internet import defer
+>>> hs.get_datastore().get_event('$1416420717069yeQaw:matrix.org')
->>> defer.ensureDeferred(hs.get_datastore().get_event('$1416420717069yeQaw:matrix.org'))
 <Deferred at 0x7ff253fc6998 current result: <FrozenEvent event_id='$1416420717069yeQaw:matrix.org', type='m.room.create', state_key=''>>
 ```

docs/message_retention_policies.md

@@ -136,34 +136,24 @@ the server's database.
 
 ### Lifetime limits
 
-Server admins can set limits on the values of `max_lifetime` to use when
+**Note: this feature is mainly useful within a closed federation or on
-purging old events in a room. These limits can be defined as such in the
+servers that don't federate, because there currently is no way to
-`retention` section of the configuration file:
+enforce these limits in an open federation.**
+
+Server admins can restrict the values their local users are allowed to
+use for both `min_lifetime` and `max_lifetime`. These limits can be
+defined as such in the `retention` section of the configuration file:
 
 ```yaml
   allowed_lifetime_min: 1d
   allowed_lifetime_max: 1y
 ```
 
-The limits are considered when running purge jobs. If necessary, the
+Here, `allowed_lifetime_min` is the lowest value a local user can set
-effective value of `max_lifetime` will be brought between
+for both `min_lifetime` and `max_lifetime`, and `allowed_lifetime_max`
-`allowed_lifetime_min` and `allowed_lifetime_max` (inclusive).
+is the highest value. Both parameters are optional (e.g. setting
-This means that, if the value of `max_lifetime` defined in the room's state
+`allowed_lifetime_min` but not `allowed_lifetime_max` only enforces a
-is lower than `allowed_lifetime_min`, the value of `allowed_lifetime_min`
+minimum and no maximum).
-will be used instead. Likewise, if the value of `max_lifetime` is higher
-than `allowed_lifetime_max`, the value of `allowed_lifetime_max` will be
-used instead.
-
-In the example above, we ensure Synapse never deletes events that are less
-than one day old, and that it always deletes events that are over a year
-old.
-
-If a default policy is set, and its `max_lifetime` value is lower than
-`allowed_lifetime_min` or higher than `allowed_lifetime_max`, the same
-process applies.
-
-Both parameters are optional; if one is omitted Synapse won't use it to
-adjust the effective value of `max_lifetime`.
 
 Like other settings in this section, these parameters can be expressed
 either as a duration or as a number of milliseconds.

docs/metrics-howto.md

@@ -13,12 +13,10 @@
     can be enabled by adding the \"metrics\" resource to the existing
     listener as such:
 
-    ```yaml
+        resources:
-      resources:
+          - names:
-        - names:
+            - client
-          - client
+            - metrics
-          - metrics
-    ```
 
     This provides a simple way of adding metrics to your Synapse
     installation, and serves under `/_synapse/metrics`. If you do not
@@ -29,17 +27,15 @@
     different thread to Synapse. This can make it more resilient to
     heavy load meaning metrics cannot be retrieved, and can be exposed
     to just internal networks easier. The served metrics are available
-    over HTTP only, and will be available at `/_synapse/metrics`.
+    over HTTP only, and will be available at `/`.
 
     Add a new listener to homeserver.yaml:
 
-    ```yaml
+        listeners:
-      listeners:
+          - type: metrics
-        - type: metrics
+            port: 9000
-          port: 9000
+            bind_addresses:
-          bind_addresses:
+              - '0.0.0.0'
-            - '0.0.0.0'
-    ```
 
     For both options, you will need to ensure that `enable_metrics` is
     set to `True`.
@@ -51,13 +47,10 @@
     It needs to set the `metrics_path` to a non-default value (under
     `scrape_configs`):
 
-    ```yaml
+        - job_name: "synapse"
-      - job_name: "synapse"
+          metrics_path: "/_synapse/metrics"
-        scrape_interval: 15s
+          static_configs:
-        metrics_path: "/_synapse/metrics"
+            - targets: ["my.server.here:port"]
-        static_configs:
-          - targets: ["my.server.here:port"]
-    ```
 
     where `my.server.here` is the IP address of Synapse, and `port` is
     the listener port configured with the `metrics` resource.
@@ -67,66 +60,6 @@
 
 1.  Restart Prometheus.
 
-1.  Consider using the [grafana dashboard](https://github.com/matrix-org/synapse/tree/master/contrib/grafana/)
-    and required [recording rules](https://github.com/matrix-org/synapse/tree/master/contrib/prometheus/) 
-
-## Monitoring workers
-
-To monitor a Synapse installation using
-[workers](https://github.com/matrix-org/synapse/blob/master/docs/workers.md),
-every worker needs to be monitored independently, in addition to
-the main homeserver process. This is because workers don't send
-their metrics to the main homeserver process, but expose them
-directly (if they are configured to do so).
-
-To allow collecting metrics from a worker, you need to add a
-`metrics` listener to its configuration, by adding the following
-under `worker_listeners`:
-
-```yaml
-  - type: metrics
-    bind_address: ''
-    port: 9101
-```
-
-The `bind_address` and `port` parameters should be set so that
-the resulting listener can be reached by prometheus, and they
-don't clash with an existing worker.
-With this example, the worker's metrics would then be available
-on `http://127.0.0.1:9101`.
-
-Example Prometheus target for Synapse with workers:
-
-```yaml
-  - job_name: "synapse"
-    scrape_interval: 15s
-    metrics_path: "/_synapse/metrics"
-    static_configs:
-      - targets: ["my.server.here:port"]
-        labels:
-          instance: "my.server"
-          job: "master"
-          index: 1
-      - targets: ["my.workerserver.here:port"]
-        labels:
-          instance: "my.server"
-          job: "generic_worker"
-          index: 1
-      - targets: ["my.workerserver.here:port"]
-        labels:
-          instance: "my.server"
-          job: "generic_worker"
-          index: 2
-      - targets: ["my.workerserver.here:port"]
-        labels:
-          instance: "my.server"
-          job: "media_repository"
-          index: 1
-```
-
-Labels (`instance`, `job`, `index`) can be defined as anything.
-The labels are used to group graphs in grafana.
-
 ## Renaming of metrics & deprecation of old names in 1.2
 
 Synapse 1.2 updates the Prometheus metrics to match the naming

docs/openid.md

@@ -1,448 +0,0 @@
-# Configuring Synapse to authenticate against an OpenID Connect provider
-
-Synapse can be configured to use an OpenID Connect Provider (OP) for
-authentication, instead of its own local password database.
-
-Any OP should work with Synapse, as long as it supports the authorization code
-flow. There are a few options for that:
-
- - start a local OP. Synapse has been tested with [Hydra][hydra] and
-   [Dex][dex-idp].  Note that for an OP to work, it should be served under a
-   secure (HTTPS) origin.  A certificate signed with a self-signed, locally
-   trusted CA should work. In that case, start Synapse with a `SSL_CERT_FILE`
-   environment variable set to the path of the CA.
-
- - set up a SaaS OP, like [Google][google-idp], [Auth0][auth0] or
-   [Okta][okta]. Synapse has been tested with Auth0 and Google.
-
-It may also be possible to use other OAuth2 providers which provide the
-[authorization code grant type](https://tools.ietf.org/html/rfc6749#section-4.1),
-such as [Github][github-idp].
-
-[google-idp]: https://developers.google.com/identity/protocols/oauth2/openid-connect
-[auth0]: https://auth0.com/
-[okta]: https://www.okta.com/
-[dex-idp]: https://github.com/dexidp/dex
-[keycloak-idp]: https://www.keycloak.org/docs/latest/server_admin/#sso-protocols
-[hydra]: https://www.ory.sh/docs/hydra/
-[github-idp]: https://developer.github.com/apps/building-oauth-apps/authorizing-oauth-apps
-
-## Preparing Synapse
-
-The OpenID integration in Synapse uses the
-[`authlib`](https://pypi.org/project/Authlib/) library, which must be installed
-as follows:
-
- * The relevant libraries are included in the Docker images and Debian packages
-   provided by `matrix.org` so no further action is needed.
-
- * If you installed Synapse into a virtualenv, run `/path/to/env/bin/pip
-   install matrix-synapse[oidc]` to install the necessary dependencies.
-
- * For other installation mechanisms, see the documentation provided by the
-   maintainer.
-
-To enable the OpenID integration, you should then add a section to the `oidc_providers`
-setting in your configuration file (or uncomment one of the existing examples).
-See [sample_config.yaml](./sample_config.yaml) for some sample settings, as well as
-the text below for example configurations for specific providers.
-
-## Sample configs
-
-Here are a few configs for providers that should work with Synapse.
-
-### Microsoft Azure Active Directory
-Azure AD can act as an OpenID Connect Provider. Register a new application under
-*App registrations* in the Azure AD management console. The RedirectURI for your
-application should point to your matrix server:
-`[synapse public baseurl]/_synapse/client/oidc/callback`
-
-Go to *Certificates & secrets* and register a new client secret. Make note of your
-Directory (tenant) ID as it will be used in the Azure links.
-Edit your Synapse config file and change the `oidc_config` section:
-
-```yaml
-oidc_providers:
-  - idp_id: microsoft
-    idp_name: Microsoft
-    issuer: "https://login.microsoftonline.com/<tenant id>/v2.0"
-    client_id: "<client id>"
-    client_secret: "<client secret>"
-    scopes: ["openid", "profile"]
-    authorization_endpoint: "https://login.microsoftonline.com/<tenant id>/oauth2/v2.0/authorize"
-    token_endpoint: "https://login.microsoftonline.com/<tenant id>/oauth2/v2.0/token"
-    userinfo_endpoint: "https://graph.microsoft.com/oidc/userinfo"
-
-    user_mapping_provider:
-      config:
-        localpart_template: "{{ user.preferred_username.split('@')[0] }}"
-        display_name_template: "{{ user.name }}"
-```
-
-### [Dex][dex-idp]
-
-[Dex][dex-idp] is a simple, open-source, certified OpenID Connect Provider.
-Although it is designed to help building a full-blown provider with an
-external database, it can be configured with static passwords in a config file.
-
-Follow the [Getting Started guide](https://dexidp.io/docs/getting-started/)
-to install Dex.
-
-Edit `examples/config-dev.yaml` config file from the Dex repo to add a client:
-
-```yaml
-staticClients:
-- id: synapse
-  secret: secret
-  redirectURIs:
-  - '[synapse public baseurl]/_synapse/client/oidc/callback'
-  name: 'Synapse'
-```
-
-Run with `dex serve examples/config-dev.yaml`.
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: dex
-    idp_name: "My Dex server"
-    skip_verification: true # This is needed as Dex is served on an insecure endpoint
-    issuer: "http://127.0.0.1:5556/dex"
-    client_id: "synapse"
-    client_secret: "secret"
-    scopes: ["openid", "profile"]
-    user_mapping_provider:
-      config:
-        localpart_template: "{{ user.name }}"
-        display_name_template: "{{ user.name|capitalize }}"
-```
-### [Keycloak][keycloak-idp]
-
-[Keycloak][keycloak-idp] is an opensource IdP maintained by Red Hat.
-
-Follow the [Getting Started Guide](https://www.keycloak.org/getting-started) to install Keycloak and set up a realm.
-
-1. Click `Clients` in the sidebar and click `Create`
-
-2. Fill in the fields as below:
-
-| Field | Value |
-|-----------|-----------|
-| Client ID | `synapse` |
-| Client Protocol | `openid-connect` |
-
-3. Click `Save`
-4. Fill in the fields as below:
-
-| Field | Value |
-|-----------|-----------|
-| Client ID | `synapse` |
-| Enabled | `On` |
-| Client Protocol | `openid-connect` |
-| Access Type | `confidential` |
-| Valid Redirect URIs | `[synapse public baseurl]/_synapse/client/oidc/callback` |
-
-5. Click `Save`
-6. On the Credentials tab, update the fields:
-
-| Field | Value |
-|-------|-------|
-| Client Authenticator | `Client ID and Secret` |
-
-7. Click `Regenerate Secret`
-8. Copy Secret
-
-```yaml
-oidc_providers:
-  - idp_id: keycloak
-    idp_name: "My KeyCloak server"
-    issuer: "https://127.0.0.1:8443/auth/realms/{realm_name}"
-    client_id: "synapse"
-    client_secret: "copy secret generated from above"
-    scopes: ["openid", "profile"]
-    user_mapping_provider:
-      config:
-        localpart_template: "{{ user.preferred_username }}"
-        display_name_template: "{{ user.name }}"
-```
-### [Auth0][auth0]
-
-1. Create a regular web application for Synapse
-2. Set the Allowed Callback URLs to `[synapse public baseurl]/_synapse/client/oidc/callback`
-3. Add a rule to add the `preferred_username` claim.
-   <details>
-    <summary>Code sample</summary>
-
-    ```js
-    function addPersistenceAttribute(user, context, callback) {
-      user.user_metadata = user.user_metadata || {};
-      user.user_metadata.preferred_username = user.user_metadata.preferred_username || user.user_id;
-      context.idToken.preferred_username = user.user_metadata.preferred_username;
-
-      auth0.users.updateUserMetadata(user.user_id, user.user_metadata)
-        .then(function(){
-            callback(null, user, context);
-        })
-        .catch(function(err){
-            callback(err);
-        });
-    }
-    ```
-  </details>
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: auth0
-    idp_name: Auth0
-    issuer: "https://your-tier.eu.auth0.com/" # TO BE FILLED
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    scopes: ["openid", "profile"]
-    user_mapping_provider:
-      config:
-        localpart_template: "{{ user.preferred_username }}"
-        display_name_template: "{{ user.name }}"
-```
-
-### GitHub
-
-GitHub is a bit special as it is not an OpenID Connect compliant provider, but
-just a regular OAuth2 provider.
-
-The [`/user` API endpoint](https://developer.github.com/v3/users/#get-the-authenticated-user)
-can be used to retrieve information on the authenticated user. As the Synapse
-login mechanism needs an attribute to uniquely identify users, and that endpoint
-does not return a `sub` property, an alternative `subject_claim` has to be set.
-
-1. Create a new OAuth application: https://github.com/settings/applications/new.
-2. Set the callback URL to `[synapse public baseurl]/_synapse/client/oidc/callback`.
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: github
-    idp_name: Github
-    idp_brand: "github"  # optional: styling hint for clients
-    discover: false
-    issuer: "https://github.com/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    authorization_endpoint: "https://github.com/login/oauth/authorize"
-    token_endpoint: "https://github.com/login/oauth/access_token"
-    userinfo_endpoint: "https://api.github.com/user"
-    scopes: ["read:user"]
-    user_mapping_provider:
-      config:
-        subject_claim: "id"
-        localpart_template: "{{ user.login }}"
-        display_name_template: "{{ user.name }}"
-```
-
-### [Google][google-idp]
-
-1. Set up a project in the Google API Console (see
-   https://developers.google.com/identity/protocols/oauth2/openid-connect#appsetup).
-2. add an "OAuth Client ID" for a Web Application under "Credentials".
-3. Copy the Client ID and Client Secret, and add the following to your synapse config:
-   ```yaml
-   oidc_providers:
-     - idp_id: google
-       idp_name: Google
-       idp_brand: "google"  # optional: styling hint for clients
-       issuer: "https://accounts.google.com/"
-       client_id: "your-client-id" # TO BE FILLED
-       client_secret: "your-client-secret" # TO BE FILLED
-       scopes: ["openid", "profile"]
-       user_mapping_provider:
-         config:
-           localpart_template: "{{ user.given_name|lower }}"
-           display_name_template: "{{ user.name }}"
-   ```
-4. Back in the Google console, add this Authorized redirect URI: `[synapse
-   public baseurl]/_synapse/client/oidc/callback`.
-
-### Twitch
-
-1. Setup a developer account on [Twitch](https://dev.twitch.tv/)
-2. Obtain the OAuth 2.0 credentials by [creating an app](https://dev.twitch.tv/console/apps/)
-3. Add this OAuth Redirect URL: `[synapse public baseurl]/_synapse/client/oidc/callback`
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: twitch
-    idp_name: Twitch
-    issuer: "https://id.twitch.tv/oauth2/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    client_auth_method: "client_secret_post"
-    user_mapping_provider:
-      config:
-        localpart_template: "{{ user.preferred_username }}"
-        display_name_template: "{{ user.name }}"
-```
-
-### GitLab
-
-1. Create a [new application](https://gitlab.com/profile/applications).
-2. Add the `read_user` and `openid` scopes.
-3. Add this Callback URL: `[synapse public baseurl]/_synapse/client/oidc/callback`
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: gitlab
-    idp_name: Gitlab
-    idp_brand: "gitlab"  # optional: styling hint for clients
-    issuer: "https://gitlab.com/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    client_auth_method: "client_secret_post"
-    scopes: ["openid", "read_user"]
-    user_profile_method: "userinfo_endpoint"
-    user_mapping_provider:
-      config:
-        localpart_template: '{{ user.nickname }}'
-        display_name_template: '{{ user.name }}'
-```
-
-### Facebook
-
-Like Github, Facebook provide a custom OAuth2 API rather than an OIDC-compliant
-one so requires a little more configuration.
-
-0. You will need a Facebook developer account. You can register for one
-   [here](https://developers.facebook.com/async/registration/).
-1. On the [apps](https://developers.facebook.com/apps/) page of the developer
-   console, "Create App", and choose "Build Connected Experiences".
-2. Once the app is created, add "Facebook Login" and choose "Web". You don't
-   need to go through the whole form here.
-3. In the left-hand menu, open "Products"/"Facebook Login"/"Settings".
-   * Add `[synapse public baseurl]/_synapse/client/oidc/callback` as an OAuth Redirect
-     URL.
-4. In the left-hand menu, open "Settings/Basic". Here you can copy the "App ID"
-   and "App Secret" for use below.
-
-Synapse config:
-
-```yaml
-  - idp_id: facebook
-    idp_name: Facebook
-    idp_brand: "facebook"  # optional: styling hint for clients
-    discover: false
-    issuer: "https://facebook.com"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    scopes: ["openid", "email"]
-    authorization_endpoint: https://facebook.com/dialog/oauth
-    token_endpoint: https://graph.facebook.com/v9.0/oauth/access_token
-    user_profile_method: "userinfo_endpoint"
-    userinfo_endpoint: "https://graph.facebook.com/v9.0/me?fields=id,name,email,picture"
-    user_mapping_provider:
-      config:
-        subject_claim: "id"
-        display_name_template: "{{ user.name }}"
-```
-
-Relevant documents:
- * https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow
- * Using Facebook's Graph API: https://developers.facebook.com/docs/graph-api/using-graph-api/
- * Reference to the User endpoint: https://developers.facebook.com/docs/graph-api/reference/user
-
-### Gitea
-
-Gitea is, like Github, not an OpenID provider, but just an OAuth2 provider.
-
-The [`/user` API endpoint](https://try.gitea.io/api/swagger#/user/userGetCurrent)
-can be used to retrieve information on the authenticated user. As the Synapse
-login mechanism needs an attribute to uniquely identify users, and that endpoint
-does not return a `sub` property, an alternative `subject_claim` has to be set.
-
-1. Create a new application.
-2. Add this Callback URL: `[synapse public baseurl]/_synapse/client/oidc/callback`
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: gitea
-    idp_name: Gitea
-    discover: false
-    issuer: "https://your-gitea.com/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    client_auth_method: client_secret_post
-    scopes: [] # Gitea doesn't support Scopes
-    authorization_endpoint: "https://your-gitea.com/login/oauth/authorize"
-    token_endpoint: "https://your-gitea.com/login/oauth/access_token"
-    userinfo_endpoint: "https://your-gitea.com/api/v1/user"
-    user_mapping_provider:
-      config:
-        subject_claim: "id"
-        localpart_template: "{{ user.login }}"
-        display_name_template: "{{ user.full_name }}"
-```
-
-### XWiki
-
-Install [OpenID Connect Provider](https://extensions.xwiki.org/xwiki/bin/view/Extension/OpenID%20Connect/OpenID%20Connect%20Provider/) extension in your [XWiki](https://www.xwiki.org) instance.
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: xwiki
-    idp_name: "XWiki"
-    issuer: "https://myxwikihost/xwiki/oidc/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_auth_method: none
-    scopes: ["openid", "profile"]
-    user_profile_method: "userinfo_endpoint"
-    user_mapping_provider:
-      config:
-        localpart_template: "{{ user.preferred_username }}"
-        display_name_template: "{{ user.name }}"
-```
-
-## Apple
-
-Configuring "Sign in with Apple" (SiWA) requires an Apple Developer account.
-
-You will need to create a new "Services ID" for SiWA, and create and download a
-private key with "SiWA" enabled.
-
-As well as the private key file, you will need:
- * Client ID: the "identifier" you gave the "Services ID"
- * Team ID: a 10-character ID associated with your developer account.
- * Key ID: the 10-character identifier for the key.
-
-https://help.apple.com/developer-account/?lang=en#/dev77c875b7e has more
-documentation on setting up SiWA.
-
-The synapse config will look like this:
-
-```yaml
-  - idp_id: apple
-    idp_name: Apple
-    issuer: "https://appleid.apple.com"
-    client_id: "your-client-id" # Set to the "identifier" for your "ServicesID"
-    client_auth_method: "client_secret_post"
-    client_secret_jwt_key:
-      key_file: "/path/to/AuthKey_KEYIDCODE.p8"  # point to your key file
-      jwt_header:
-        alg: ES256
-        kid: "KEYIDCODE"   # Set to the 10-char Key ID
-      jwt_payload:
-        iss: TEAMIDCODE    # Set to the 10-char Team ID
-    scopes: ["name", "email", "openid"]
-    authorization_endpoint: https://appleid.apple.com/auth/authorize?response_mode=form_post
-    user_mapping_provider:
-      config:
-        email_template: "{{ user.email }}"
-```

docs/password_auth_providers.md

@@ -9,115 +9,108 @@ into Synapse, and provides a number of methods by which it can integrate
 with the authentication system.
 
 This document serves as a reference for those looking to implement their
-own password auth providers. Additionally, here is a list of known
+own password auth providers.
-password auth provider module implementations:
-
-* [matrix-synapse-ldap3](https://github.com/matrix-org/matrix-synapse-ldap3/)
-* [matrix-synapse-shared-secret-auth](https://github.com/devture/matrix-synapse-shared-secret-auth)
-* [matrix-synapse-rest-password-provider](https://github.com/ma1uta/matrix-synapse-rest-password-provider)
 
 ## Required methods
 
 Password auth provider classes must provide the following methods:
 
-* `parse_config(config)`
+*class* `SomeProvider.parse_config`(*config*)
-  This method is passed the `config` object for this module from the
-  homeserver configuration file.
 
-  It should perform any appropriate sanity checks on the provided
+> This method is passed the `config` object for this module from the
-  configuration, and return an object which is then passed into
+> homeserver configuration file.
-  `__init__`.
+>
+> It should perform any appropriate sanity checks on the provided
+> configuration, and return an object which is then passed into
+> `__init__`.
 
-  This method should have the `@staticmethod` decoration.
+*class* `SomeProvider`(*config*, *account_handler*)
 
-* `__init__(self, config, account_handler)`
+> The constructor is passed the config object returned by
-
+> `parse_config`, and a `synapse.module_api.ModuleApi` object which
-  The constructor is passed the config object returned by
+> allows the password provider to check if accounts exist and/or create
-  `parse_config`, and a `synapse.module_api.ModuleApi` object which
+> new ones.
-  allows the password provider to check if accounts exist and/or create
-  new ones.
 
 ## Optional methods
 
-Password auth provider classes may optionally provide the following methods:
+Password auth provider classes may optionally provide the following
+methods.
 
-* `get_db_schema_files(self)`
+*class* `SomeProvider.get_db_schema_files`()
 
-  This method, if implemented, should return an Iterable of
+> This method, if implemented, should return an Iterable of
-  `(name, stream)` pairs of database schema files. Each file is applied
+> `(name, stream)` pairs of database schema files. Each file is applied
-  in turn at initialisation, and a record is then made in the database
+> in turn at initialisation, and a record is then made in the database
-  so that it is not re-applied on the next start.
+> so that it is not re-applied on the next start.
 
-* `get_supported_login_types(self)`
+`someprovider.get_supported_login_types`()
 
-  This method, if implemented, should return a `dict` mapping from a
+> This method, if implemented, should return a `dict` mapping from a
-  login type identifier (such as `m.login.password`) to an iterable
+> login type identifier (such as `m.login.password`) to an iterable
-  giving the fields which must be provided by the user in the submission
+> giving the fields which must be provided by the user in the submission
-  to [the `/login` API](https://matrix.org/docs/spec/client_server/latest#post-matrix-client-r0-login).
+> to the `/login` api. These fields are passed in the `login_dict`
-  These fields are passed in the `login_dict` dictionary to `check_auth`.
+> dictionary to `check_auth`.
+>
+> For example, if a password auth provider wants to implement a custom
+> login type of `com.example.custom_login`, where the client is expected
+> to pass the fields `secret1` and `secret2`, the provider should
+> implement this method and return the following dict:
+>
+>     {"com.example.custom_login": ("secret1", "secret2")}
 
-  For example, if a password auth provider wants to implement a custom
+`someprovider.check_auth`(*username*, *login_type*, *login_dict*)
-  login type of `com.example.custom_login`, where the client is expected
-  to pass the fields `secret1` and `secret2`, the provider should
-  implement this method and return the following dict:
 
-  ```python
+> This method is the one that does the real work. If implemented, it
-  {"com.example.custom_login": ("secret1", "secret2")}
+> will be called for each login attempt where the login type matches one
-  ```
+> of the keys returned by `get_supported_login_types`.
+>
+> It is passed the (possibly UNqualified) `user` provided by the client,
+> the login type, and a dictionary of login secrets passed by the
+> client.
+>
+> The method should return a Twisted `Deferred` object, which resolves
+> to the canonical `@localpart:domain` user id if authentication is
+> successful, and `None` if not.
+>
+> Alternatively, the `Deferred` can resolve to a `(str, func)` tuple, in
+> which case the second field is a callback which will be called with
+> the result from the `/login` call (including `access_token`,
+> `device_id`, etc.)
 
-* `check_auth(self, username, login_type, login_dict)`
+`someprovider.check_3pid_auth`(*medium*, *address*, *password*)
 
-  This method does the real work. If implemented, it
+> This method, if implemented, is called when a user attempts to
-  will be called for each login attempt where the login type matches one
+> register or log in with a third party identifier, such as email. It is
-  of the keys returned by `get_supported_login_types`.
+> passed the medium (ex. "email"), an address (ex.
+> "<jdoe@example.com>") and the user's password.
+>
+> The method should return a Twisted `Deferred` object, which resolves
+> to a `str` containing the user's (canonical) User ID if
+> authentication was successful, and `None` if not.
+>
+> As with `check_auth`, the `Deferred` may alternatively resolve to a
+> `(user_id, callback)` tuple.
 
-  It is passed the (possibly unqualified) `user` field provided by the client,
+`someprovider.check_password`(*user_id*, *password*)
-  the login type, and a dictionary of login secrets passed by the
-  client.
 
-  The method should return an `Awaitable` object, which resolves
+> This method provides a simpler interface than
-  to the canonical `@localpart:domain` user ID if authentication is
+> `get_supported_login_types` and `check_auth` for password auth
-  successful, and `None` if not.
+> providers that just want to provide a mechanism for validating
+> `m.login.password` logins.
+>
+> Iif implemented, it will be called to check logins with an
+> `m.login.password` login type. It is passed a qualified
+> `@localpart:domain` user id, and the password provided by the user.
+>
+> The method should return a Twisted `Deferred` object, which resolves
+> to `True` if authentication is successful, and `False` if not.
 
-  Alternatively, the `Awaitable` can resolve to a `(str, func)` tuple, in
+`someprovider.on_logged_out`(*user_id*, *device_id*, *access_token*)
-  which case the second field is a callback which will be called with
-  the result from the `/login` call (including `access_token`,
-  `device_id`, etc.)
 
-* `check_3pid_auth(self, medium, address, password)`
+> This method, if implemented, is called when a user logs out. It is
-
+> passed the qualified user ID, the ID of the deactivated device (if
-  This method, if implemented, is called when a user attempts to
+> any: access tokens are occasionally created without an associated
-  register or log in with a third party identifier, such as email. It is
+> device ID), and the (now deactivated) access token.
-  passed the medium (ex. "email"), an address (ex.
+>
-  "<jdoe@example.com>") and the user's password.
+> It may return a Twisted `Deferred` object; the logout request will
-
+> wait for the deferred to complete but the result is ignored.
-  The method should return an `Awaitable` object, which resolves
-  to a `str` containing the user's (canonical) User id if
-  authentication was successful, and `None` if not.
-
-  As with `check_auth`, the `Awaitable` may alternatively resolve to a
-  `(user_id, callback)` tuple.
-
-* `check_password(self, user_id, password)`
-
-  This method provides a simpler interface than
-  `get_supported_login_types` and `check_auth` for password auth
-  providers that just want to provide a mechanism for validating
-  `m.login.password` logins.
-
-  If implemented, it will be called to check logins with an
-  `m.login.password` login type. It is passed a qualified
-  `@localpart:domain` user id, and the password provided by the user.
-
-  The method should return an `Awaitable` object, which resolves
-  to `True` if authentication is successful, and `False` if not.
-
-* `on_logged_out(self, user_id, device_id, access_token)`
-
-  This method, if implemented, is called when a user logs out. It is
-  passed the qualified user ID, the ID of the deactivated device (if
-  any: access tokens are occasionally created without an associated
-  device ID), and the (now deactivated) access token.
-
-  It may return an `Awaitable` object; the logout request will
-  wait for the `Awaitable` to complete, but the result is ignored.

docs/postgres.md

@@ -18,7 +18,7 @@ connect to a postgres database.
     virtualenv](../INSTALL.md#installing-from-source), you can install
     the library with:
 
-        ~/synapse/env/bin/pip install "matrix-synapse[postgres]"
+        ~/synapse/env/bin/pip install matrix-synapse[postgres]
 
     (substituting the path to your virtualenv for `~/synapse/env`, if
     you used a different path). You will require the postgres
@@ -61,33 +61,7 @@ Note that the PostgreSQL database *must* have the correct encoding set
 
 You may need to enable password authentication so `synapse_user` can
 connect to the database. See
-<https://www.postgresql.org/docs/current/auth-pg-hba-conf.html>.
+<https://www.postgresql.org/docs/11/auth-pg-hba-conf.html>.
-
-If you get an error along the lines of `FATAL:  Ident authentication failed for
-user "synapse_user"`, you may need to use an authentication method other than
-`ident`:
-
-* If the `synapse_user` user has a password, add the password to the `database:`
-  section of `homeserver.yaml`. Then add the following to `pg_hba.conf`:
-
-  ```
-  host    synapse     synapse_user    ::1/128     md5  # or `scram-sha-256` instead of `md5` if you use that
-  ```
-
-* If the `synapse_user` user does not have a password, then a password doesn't
-  have to be added to `homeserver.yaml`. But the following does need to be added
-  to `pg_hba.conf`:
-
-  ```
-  host    synapse     synapse_user    ::1/128     trust
-  ```
-
-Note that line order matters in `pg_hba.conf`, so make sure that if you do add a
-new line, it is inserted before:
-
-```
-host    all         all             ::1/128     ident
-```
 
 ### Fixing incorrect `COLLATE` or `CTYPE`
 
@@ -98,7 +72,8 @@ underneath the database, or if a different version of the locale is used on any
 replicas.
 
 The safest way to fix the issue is to take a dump and recreate the database with
-the correct `COLLATE` and `CTYPE` parameters (as shown above). It is also possible to change the
+the correct `COLLATE` and `CTYPE` parameters (as per
+[docs/postgres.md](docs/postgres.md)). It is also possible to change the
 parameters on a live database and run a `REINDEX` on the entire database,
 however extreme care must be taken to avoid database corruption.
 
@@ -106,17 +81,6 @@ Note that the above may fail with an error about duplicate rows if corruption
 has already occurred, and such duplicate rows will need to be manually removed.
 
 
-## Fixing inconsistent sequences error
-
-Synapse uses Postgres sequences to generate IDs for various tables. A sequence
-and associated table can get out of sync if, for example, Synapse has been
-downgraded and then upgraded again.
-
-To fix the issue shut down Synapse (including any and all workers) and run the
-SQL command included in the error message. Once done Synapse should start
-successfully.
-
-
 ## Tuning Postgres
 
 The default settings should be fine for most deployments. For larger
@@ -141,41 +105,19 @@ of free memory the database host has available.
 When you are ready to start using PostgreSQL, edit the `database`
 section in your config file to match the following lines:
 
-```yaml
+    database:
-database:
+        name: psycopg2
-  name: psycopg2
+        args:
-  args:
+            user: <user>
-    user: <user>
+            password: <pass>
-    password: <pass>
+            database: <db>
-    database: <db>
+            host: <host>
-    host: <host>
+            cp_min: 5
-    cp_min: 5
+            cp_max: 10
-    cp_max: 10
-```
 
 All key, values in `args` are passed to the `psycopg2.connect(..)`
 function, except keys beginning with `cp_`, which are consumed by the
-twisted adbapi connection pool. See the [libpq
+twisted adbapi connection pool.
-documentation](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS)
-for a list of options which can be passed.
-
-You should consider tuning the `args.keepalives_*` options if there is any danger of
-the connection between your homeserver and database dropping, otherwise Synapse
-may block for an extended period while it waits for a response from the
-database server. Example values might be:
-
-```yaml
-# seconds of inactivity after which TCP should send a keepalive message to the server
-keepalives_idle: 10
-
-# the number of seconds after which a TCP keepalive message that is not
-# acknowledged by the server should be retransmitted
-keepalives_interval: 10
-
-# the number of TCP keepalives that can be lost before the client's connection
-# to the server is considered dead
-keepalives_count: 3
-```
 
 ## Porting from SQLite
 
@@ -199,9 +141,6 @@ to do step 2.
 
 It is safe to at any time kill the port script and restart it.
 
-Note that the database may take up significantly more (25% - 100% more)
-space on disk after porting to Postgres.
-
 ### Using the port script
 
 Firstly, shut down the currently running synapse server and copy its

docs/reverse_proxy.md

@@ -3,211 +3,115 @@
 It is recommended to put a reverse proxy such as
 [nginx](https://nginx.org/en/docs/http/ngx_http_proxy_module.html),
 [Apache](https://httpd.apache.org/docs/current/mod/mod_proxy_http.html),
-[Caddy](https://caddyserver.com/docs/quick-starts/reverse-proxy),
+[Caddy](https://caddyserver.com/docs/proxy) or
-[HAProxy](https://www.haproxy.org/) or
+[HAProxy](https://www.haproxy.org/) in front of Synapse. One advantage
-[relayd](https://man.openbsd.org/relayd.8) in front of Synapse. One advantage
 of doing so is that it means that you can expose the default https port
 (443) to Matrix clients without needing to run Synapse with root
 privileges.
 
-You should configure your reverse proxy to forward requests to `/_matrix` or
+> **NOTE**: Your reverse proxy must not `canonicalise` or `normalise`
-`/_synapse/client` to Synapse, and have it set the `X-Forwarded-For` and
+the requested URI in any way (for example, by decoding `%xx` escapes).
-`X-Forwarded-Proto` request headers.
+Beware that Apache *will* canonicalise URIs unless you specifify
+`nocanon`.
 
-You should remember that Matrix clients and other Matrix servers do not
+When setting up a reverse proxy, remember that Matrix clients and other
-necessarily need to connect to your server via the same server name or
+Matrix servers do not necessarily need to connect to your server via the
-port. Indeed, clients will use port 443 by default, whereas servers default to
+same server name or port. Indeed, clients will use port 443 by default,
-port 8448. Where these are different, we refer to the 'client port' and the
+whereas servers default to port 8448. Where these are different, we
-'federation port'. See [the Matrix
+refer to the 'client port' and the \'federation port\'. See [the Matrix
 specification](https://matrix.org/docs/spec/server_server/latest#resolving-server-names)
 for more details of the algorithm used for federation connections, and
 [delegate.md](<delegate.md>) for instructions on setting up delegation.
 
-**NOTE**: Your reverse proxy must not `canonicalise` or `normalise`
-the requested URI in any way (for example, by decoding `%xx` escapes).
-Beware that Apache *will* canonicalise URIs unless you specify
-`nocanon`.
-
 Let's assume that we expect clients to connect to our server at
 `https://matrix.example.com`, and other servers to connect at
 `https://example.com:8448`.  The following sections detail the configuration of
 the reverse proxy and the homeserver.
 
-## Reverse-proxy configuration examples
+## Webserver configuration examples
 
-**NOTE**: You only need one of these.
+> **NOTE**: You only need one of these.
 
 ### nginx
 
-```
+        server {
-server {
+            listen 443 ssl;
-    listen 443 ssl http2;
+            listen [::]:443 ssl;
-    listen [::]:443 ssl http2;
+            server_name matrix.example.com;
 
-    # For the federation port
+            location /_matrix {
-    listen 8448 ssl http2 default_server;
+                proxy_pass http://localhost:8008;
-    listen [::]:8448 ssl http2 default_server;
+                proxy_set_header X-Forwarded-For $remote_addr;
+            }
+        }
 
-    server_name matrix.example.com;
+        server {
+            listen 8448 ssl default_server;
+            listen [::]:8448 ssl default_server;
+            server_name example.com;
 
-    location ~* ^(\/_matrix|\/_synapse\/client) {
+            location / {
-        proxy_pass http://localhost:8008;
+                proxy_pass http://localhost:8008;
-        proxy_set_header X-Forwarded-For $remote_addr;
+                proxy_set_header X-Forwarded-For $remote_addr;
-        proxy_set_header X-Forwarded-Proto $scheme;
+            }
-        proxy_set_header Host $host;
+        }
 
-        # Nginx by default only allows file uploads up to 1M in size
+> **NOTE**: Do not add a `/` after the port in `proxy_pass`, otherwise nginx will
-        # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
-        client_max_body_size 50M;
-    }
-}
-```
-
-**NOTE**: Do not add a path after the port in `proxy_pass`, otherwise nginx will
 canonicalise/normalise the URI.
 
-### Caddy 1
+### Caddy
 
-```
+        matrix.example.com {
-matrix.example.com {
+          proxy /_matrix http://localhost:8008 {
-  proxy /_matrix http://localhost:8008 {
+            transparent
-    transparent
+          }
-  }
+        }
 
-  proxy /_synapse/client http://localhost:8008 {
+        example.com:8448 {
-    transparent
+          proxy / http://localhost:8008 {
-  }
+            transparent
-}
+          }
-
+        }
-example.com:8448 {
-  proxy / http://localhost:8008 {
-    transparent
-  }
-}
-```
-
-### Caddy 2
-
-```
-matrix.example.com {
-  reverse_proxy /_matrix/* http://localhost:8008
-  reverse_proxy /_synapse/client/* http://localhost:8008
-}
-
-example.com:8448 {
-  reverse_proxy http://localhost:8008
-}
-```
 
 ### Apache
 
-```
+        <VirtualHost *:443>
-<VirtualHost *:443>
+            SSLEngine on
-    SSLEngine on
+            ServerName matrix.example.com;
-    ServerName matrix.example.com;
 
-    RequestHeader set "X-Forwarded-Proto" expr=%{REQUEST_SCHEME}
+            AllowEncodedSlashes NoDecode
-    AllowEncodedSlashes NoDecode
+            ProxyPass /_matrix http://127.0.0.1:8008/_matrix nocanon
-    ProxyPass /_matrix http://127.0.0.1:8008/_matrix nocanon
+            ProxyPassReverse /_matrix http://127.0.0.1:8008/_matrix
-    ProxyPassReverse /_matrix http://127.0.0.1:8008/_matrix
+        </VirtualHost>
-    ProxyPass /_synapse/client http://127.0.0.1:8008/_synapse/client nocanon
-    ProxyPassReverse /_synapse/client http://127.0.0.1:8008/_synapse/client
-</VirtualHost>
 
-<VirtualHost *:8448>
+        <VirtualHost *:8448>
-    SSLEngine on
+            SSLEngine on
-    ServerName example.com;
+            ServerName example.com;
 
-    RequestHeader set "X-Forwarded-Proto" expr=%{REQUEST_SCHEME}
+            AllowEncodedSlashes NoDecode
-    AllowEncodedSlashes NoDecode
+            ProxyPass /_matrix http://127.0.0.1:8008/_matrix nocanon
-    ProxyPass /_matrix http://127.0.0.1:8008/_matrix nocanon
+            ProxyPassReverse /_matrix http://127.0.0.1:8008/_matrix
-    ProxyPassReverse /_matrix http://127.0.0.1:8008/_matrix
+        </VirtualHost>
-</VirtualHost>
-```
 
-**NOTE**: ensure the  `nocanon` options are included.
+> **NOTE**: ensure the  `nocanon` options are included.
-
-**NOTE 2**: It appears that Synapse is currently incompatible with the ModSecurity module for Apache (`mod_security2`). If you need it enabled for other services on your web server, you can disable it for Synapse's two VirtualHosts by including the following lines before each of the two `</VirtualHost>` above:
-
-```
-<IfModule security2_module>
-    SecRuleEngine off
-</IfModule>
-```
 
 ### HAProxy
 
-```
+        frontend https
-frontend https
+          bind :::443 v4v6 ssl crt /etc/ssl/haproxy/ strict-sni alpn h2,http/1.1
-  bind :::443 v4v6 ssl crt /etc/ssl/haproxy/ strict-sni alpn h2,http/1.1
-  http-request set-header X-Forwarded-Proto https if { ssl_fc }
-  http-request set-header X-Forwarded-Proto http if !{ ssl_fc }
-  http-request set-header X-Forwarded-For %[src]
 
-  # Matrix client traffic
+          # Matrix client traffic
-  acl matrix-host hdr(host) -i matrix.example.com
+          acl matrix-host hdr(host) -i matrix.example.com
-  acl matrix-path path_beg /_matrix
+          acl matrix-path path_beg /_matrix
-  acl matrix-path path_beg /_synapse/client
 
-  use_backend matrix if matrix-host matrix-path
+          use_backend matrix if matrix-host matrix-path
 
-frontend matrix-federation
+        frontend matrix-federation
-  bind :::8448 v4v6 ssl crt /etc/ssl/haproxy/synapse.pem alpn h2,http/1.1
+          bind :::8448 v4v6 ssl crt /etc/ssl/haproxy/synapse.pem alpn h2,http/1.1
-  http-request set-header X-Forwarded-Proto https if { ssl_fc }
+          default_backend matrix
-  http-request set-header X-Forwarded-Proto http if !{ ssl_fc }
-  http-request set-header X-Forwarded-For %[src]
 
-  default_backend matrix
+        backend matrix
-
+          server matrix 127.0.0.1:8008
-backend matrix
-  server matrix 127.0.0.1:8008
-```
-
-### Relayd
-
-```
-table <webserver>    { 127.0.0.1 }
-table <matrixserver> { 127.0.0.1 }
-
-http protocol "https" {
-    tls { no tlsv1.0, ciphers "HIGH" }
-    tls keypair "example.com"
-    match header set "X-Forwarded-For"   value "$REMOTE_ADDR"
-    match header set "X-Forwarded-Proto" value "https"
-
-    # set CORS header for .well-known/matrix/server, .well-known/matrix/client
-    # httpd does not support setting headers, so do it here
-    match request path "/.well-known/matrix/*" tag "matrix-cors"
-    match response tagged "matrix-cors" header set "Access-Control-Allow-Origin" value "*"
-
-    pass quick path "/_matrix/*"         forward to <matrixserver>
-    pass quick path "/_synapse/client/*" forward to <matrixserver>
-
-    # pass on non-matrix traffic to webserver
-    pass                                 forward to <webserver>
-}
-
-relay "https_traffic" {
-    listen on egress port 443 tls
-    protocol "https"
-    forward to <matrixserver> port 8008 check tcp
-    forward to <webserver>    port 8080 check tcp
-}
-
-http protocol "matrix" {
-    tls { no tlsv1.0, ciphers "HIGH" }
-    tls keypair "example.com"
-    block
-    pass quick path "/_matrix/*"         forward to <matrixserver>
-    pass quick path "/_synapse/client/*" forward to <matrixserver>
-}
-
-relay "matrix_federation" {
-    listen on egress port 8448 tls
-    protocol "matrix"
-    forward to <matrixserver> port 8008 check tcp
-}
-```
 
 ## Homeserver Configuration
 
@@ -218,17 +122,3 @@ client IP addresses are recorded correctly.
 Having done so, you can then use `https://matrix.example.com` (instead
 of `https://matrix.example.com:8448`) as the "Custom server" when
 connecting to Synapse from a client.
-
-
-## Health check endpoint
-
-Synapse exposes a health check endpoint for use by reverse proxies.
-Each configured HTTP listener has a `/health` endpoint which always returns
-200 OK (and doesn't get logged).
-
-## Synapse administration endpoints
-
-Endpoints for administering your Synapse instance are placed under
-`/_synapse/admin`. These require authentication through an access token of an
-admin user. However as access to these endpoints grants the caller a lot of power,
-we do not recommend exposing them to the public internet without good reason.

docs/saml_mapping_providers.md

@@ -0,0 +1,77 @@
+# SAML Mapping Providers
+
+A SAML mapping provider is a Python class (loaded via a Python module) that
+works out how to map attributes of a SAML response object to Matrix-specific
+user attributes. Details such as user ID localpart, displayname, and even avatar
+URLs are all things that can be mapped from talking to a SSO service.
+
+As an example, a SSO service may return the email address
+"john.smith@example.com" for a user, whereas Synapse will need to figure out how
+to turn that into a displayname when creating a Matrix user for this individual.
+It may choose `John Smith`, or `Smith, John [Example.com]` or any number of
+variations. As each Synapse configuration may want something different, this is
+where SAML mapping providers come into play.
+
+## Enabling Providers
+
+External mapping providers are provided to Synapse in the form of an external
+Python module. Retrieve this module from [PyPi](https://pypi.org) or elsewhere,
+then tell Synapse where to look for the handler class by editing the
+`saml2_config.user_mapping_provider.module` config option.
+
+`saml2_config.user_mapping_provider.config` allows you to provide custom
+configuration options to the module. Check with the module's documentation for
+what options it provides (if any). The options listed by default are for the
+user mapping provider built in to Synapse. If using a custom module, you should
+comment these options out and use those specified by the module instead.
+
+## Building a Custom Mapping Provider
+
+A custom mapping provider must specify the following methods:
+
+* `__init__(self, parsed_config)`
+   - Arguments:
+     - `parsed_config` - A configuration object that is the return value of the
+       `parse_config` method. You should set any configuration options needed by
+       the module here.
+* `saml_response_to_user_attributes(self, saml_response, failures)`
+    - Arguments:
+      - `saml_response` - A `saml2.response.AuthnResponse` object to extract user
+                          information from.
+      - `failures` - An `int` that represents the amount of times the returned
+                     mxid localpart mapping has failed.  This should be used
+                     to create a deduplicated mxid localpart which should be
+                     returned instead. For example, if this method returns
+                     `john.doe` as the value of `mxid_localpart` in the returned
+                     dict, and that is already taken on the homeserver, this
+                     method will be called again with the same parameters but
+                     with failures=1. The method should then return a different
+                     `mxid_localpart` value, such as `john.doe1`.
+    - This method must return a dictionary, which will then be used by Synapse
+      to build a new user. The following keys are allowed:
+       * `mxid_localpart` - Required. The mxid localpart of the new user.
+       * `displayname` - The displayname of the new user. If not provided, will default to
+                         the value of `mxid_localpart`.
+* `parse_config(config)`
+    - This method should have the `@staticmethod` decoration.
+    - Arguments:
+        - `config` - A `dict` representing the parsed content of the
+          `saml2_config.user_mapping_provider.config` homeserver config option.
+           Runs on homeserver startup. Providers should extract any option values
+           they need here.
+    - Whatever is returned will be passed back to the user mapping provider module's
+      `__init__` method during construction.
+* `get_saml_attributes(config)`
+    - This method should have the `@staticmethod` decoration.
+    - Arguments:
+        - `config` - A object resulting from a call to `parse_config`.
+    - Returns a tuple of two sets. The first set equates to the saml auth
+      response attributes that are required for the module to function, whereas
+      the second set consists of those attributes which can be used if available,
+      but are not necessary.
+
+## Synapse's Default Provider
+
+Synapse has a built-in SAML mapping provider if a custom provider isn't
+specified in the config. It is located at
+[`synapse.handlers.saml_handler.DefaultSamlMappingProvider`](../synapse/handlers/saml_handler.py).

docs/sample_log_config.yaml

@@ -3,11 +3,7 @@
 # This is a YAML file containing a standard Python logging configuration
 # dictionary. See [1] for details on the valid settings.
 #
-# Synapse also supports structured logging for machine readable logs which can
-# be ingested by ELK stacks. See [2] for details.
-#
 # [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
-# [2]: https://github.com/matrix-org/synapse/blob/master/docs/structured_logging.md
 
 version: 1
 
@@ -15,33 +11,24 @@ formatters:
     precise:
         format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
 
+filters:
+    context:
+        (): synapse.logging.context.LoggingContextFilter
+        request: ""
+
 handlers:
     file:
-        class: logging.handlers.TimedRotatingFileHandler
+        class: logging.handlers.RotatingFileHandler
         formatter: precise
         filename: /var/log/matrix-synapse/homeserver.log
-        when: midnight
+        maxBytes: 104857600
-        backupCount: 3  # Does not include the current log file.
+        backupCount: 10
+        filters: [context]
         encoding: utf8
-
-    # Default to buffering writes to log file for efficiency. This means that
-    # will be a delay for INFO/DEBUG logs to get written, but WARNING/ERROR
-    # logs will still be flushed immediately.
-    buffer:
-        class: logging.handlers.MemoryHandler
-        target: file
-        # The capacity is the number of log lines that are buffered before
-        # being written to disk. Increasing this will lead to better
-        # performance, at the expensive of it taking longer for log lines to
-        # be written to disk.
-        capacity: 10
-        flushLevel: 30  # Flush for WARNING logs as well
-
-    # A handler that writes logs to stderr. Unused by default, but can be used
-    # instead of "buffer" and "file" in the logger handlers.
     console:
         class: logging.StreamHandler
         formatter: precise
+        filters: [context]
 
 loggers:
     synapse.storage.SQL:
@@ -49,23 +36,8 @@ loggers:
         # information such as access tokens.
         level: INFO
 
-    twisted:
-        # We send the twisted logging directly to the file handler,
-        # to work around https://github.com/matrix-org/synapse/issues/3471
-        # when using "buffer" logger. Use "console" to log to stderr instead.
-        handlers: [file]
-        propagate: false
-
 root:
     level: INFO
-
+    handlers: [file, console]
-    # Write logs to the `buffer` handler, which will buffer them together in memory,
-    # then write them to a file.
-    #
-    # Replace "buffer" with "console" to log to stderr instead. (Note that you'll
-    # also need to update the configuration for the `twisted` logger above, in
-    # this case.)
-    #
-    handlers: [buffer]
 
 disable_existing_loggers: false

docs/spam_checker.md

@@ -11,10 +11,9 @@ able to be imported by the running Synapse.
 The Python class is instantiated with two objects:
 
 * Any configuration (see below).
-* An instance of `synapse.module_api.ModuleApi`.
+* An instance of `synapse.spam_checker_api.SpamCheckerApi`.
 
 It then implements methods which return a boolean to alter behavior in Synapse.
-All the methods must be defined.
 
 There's a generic method for checking every event (`check_event_for_spam`), as
 well as some specific methods:
@@ -23,63 +22,41 @@ well as some specific methods:
 * `user_may_create_room`
 * `user_may_create_room_alias`
 * `user_may_publish_room`
-* `check_username_for_spam`
-* `check_registration_for_spam`
-* `check_media_file_for_spam`
 
-The details of each of these methods (as well as their inputs and outputs)
+The details of the each of these methods (as well as their inputs and outputs)
 are documented in the `synapse.events.spamcheck.SpamChecker` class.
 
-The `ModuleApi` class provides a way for the custom spam checker class to
+The `SpamCheckerApi` class provides a way for the custom spam checker class to
-call back into the homeserver internals.
+call back into the homeserver internals. It currently implements the following
+methods:
 
-Additionally, a `parse_config` method is mandatory and receives the plugin config
+* `get_state_events_in_room`
-dictionary. After parsing, It must return an object which will be
-passed to `__init__` later.
 
 ### Example
 
 ```python
-from synapse.spam_checker_api import RegistrationBehaviour
-
 class ExampleSpamChecker:
     def __init__(self, config, api):
         self.config = config
         self.api = api
 
-    @staticmethod
+    def check_event_for_spam(self, foo):
-    def parse_config(config):
-        return config
-        
-    async def check_event_for_spam(self, foo):
         return False  # allow all events
 
-    async def user_may_invite(self, inviter_userid, invitee_userid, room_id):
+    def user_may_invite(self, inviter_userid, invitee_userid, room_id):
         return True  # allow all invites
 
-    async def user_may_create_room(self, userid):
+    def user_may_create_room(self, userid):
         return True  # allow all room creations
 
-    async def user_may_create_room_alias(self, userid, room_alias):
+    def user_may_create_room_alias(self, userid, room_alias):
         return True  # allow all room aliases
 
-    async def user_may_publish_room(self, userid, room_id):
+    def user_may_publish_room(self, userid, room_id):
         return True  # allow publishing of all rooms
 
-    async def check_username_for_spam(self, user_profile):
+    def check_username_for_spam(self, user_profile):
         return False  # allow all usernames
-
-    async def check_registration_for_spam(
-        self,
-        email_threepid,
-        username,
-        request_info,
-        auth_provider_id,
-    ):
-        return RegistrationBehaviour.ALLOW  # allow all registrations
-
-    async def check_media_file_for_spam(self, file_wrapper, file_info):
-        return False  # allow all media
 ```
 
 ## Configuration
@@ -87,12 +64,10 @@ class ExampleSpamChecker:
 Modify the `spam_checker` section of your `homeserver.yaml` in the following
 manner:
 
-Create a list entry with the keys `module` and `config`.
+`module` should point to the fully qualified Python class that implements your
+custom logic, e.g. `my_module.ExampleSpamChecker`.
 
-* `module` should point to the fully qualified Python class that implements your
+`config` is a dictionary that gets passed to the spam checker class.
-  custom logic, e.g. `my_module.ExampleSpamChecker`.
-
-* `config` is a dictionary that gets passed to the spam checker class.
 
 ### Example
 
@@ -100,15 +75,12 @@ This section might look like:
 
 ```yaml
 spam_checker:
-  - module: my_module.ExampleSpamChecker
+  module: my_module.ExampleSpamChecker
-    config:
+  config:
-      # Enable or disable a specific option in ExampleSpamChecker.
+    # Enable or disable a specific option in ExampleSpamChecker.
-      my_custom_option: true
+    my_custom_option: true
 ```
 
-More spam checkers can be added in tandem by appending more items to the list. An
-action is blocked when at least one of the configured spam checkers flags it.
-
 ## Examples
 
 The [Mjolnir](https://github.com/matrix-org/mjolnir) project is a full fledged

docs/sphinx/README.rst

@@ -0,0 +1 @@
+TODO: how (if at all) is this actually maintained?

docs/sphinx/conf.py

@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+#
+# Synapse documentation build configuration file, created by
+# sphinx-quickstart on Tue Jun 10 17:31:02 2014.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys
+import os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath(".."))
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.coverage",
+    "sphinx.ext.ifconfig",
+    "sphinxcontrib.napoleon",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# The suffix of source filenames.
+source_suffix = ".rst"
+
+# The encoding of source files.
+# source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = "index"
+
+# General information about the project.
+project = "Synapse"
+copyright = (
+    "Copyright 2014-2017 OpenMarket Ltd, 2017 Vector Creations Ltd, 2017 New Vector Ltd"
+)
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = "1.0"
+# The full version, including alpha/beta/rc tags.
+release = "1.0"
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+# language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+# today = ''
+# Else, today_fmt is used as the format for a strftime call.
+# today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ["_build"]
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+# default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+# add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+# add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+# show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = "sphinx"
+
+# A list of ignored prefixes for module index sorting.
+# modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+# keep_warnings = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = "default"
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+# html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+# html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+# html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+# html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+# html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+# html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+# html_extra_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+# html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+# html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+# html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+# html_additional_pages = {}
+
+# If false, no module index is generated.
+# html_domain_indices = True
+
+# If false, no index is generated.
+# html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+# html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+# html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+# html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+# html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+# html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+# html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = "Synapsedoc"
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [("index", "Synapse.tex", "Synapse Documentation", "TNG", "manual")]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+# latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+# latex_use_parts = False
+
+# If true, show page references after internal links.
+# latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+# latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+# latex_appendices = []
+
+# If false, no module index is generated.
+# latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [("index", "synapse", "Synapse Documentation", ["TNG"], 1)]
+
+# If true, show URL addresses after external links.
+# man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (
+        "index",
+        "Synapse",
+        "Synapse Documentation",
+        "TNG",
+        "Synapse",
+        "One line description of project.",
+        "Miscellaneous",
+    )
+]
+
+# Documents to append as an appendix to all manuals.
+# texinfo_appendices = []
+
+# If false, no module index is generated.
+# texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+# texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+# texinfo_no_detailmenu = False
+
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {"http://docs.python.org/": None}
+
+napoleon_include_special_with_doc = True
+napoleon_use_ivar = True

docs/sphinx/index.rst

@@ -0,0 +1,20 @@
+.. Synapse documentation master file, created by
+   sphinx-quickstart on Tue Jun 10 17:31:02 2014.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to Synapse's documentation!
+===================================
+
+Contents:
+
+.. toctree::
+   synapse
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+

docs/sphinx/modules.rst

@@ -0,0 +1,7 @@
+synapse
+=======
+
+.. toctree::
+   :maxdepth: 4
+
+   synapse

docs/sphinx/synapse.api.auth.rst

@@ -0,0 +1,7 @@
+synapse.api.auth module
+=======================
+
+.. automodule:: synapse.api.auth
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/sphinx/synapse.api.constants.rst

@@ -0,0 +1,7 @@
+synapse.api.constants module
+============================
+
+.. automodule:: synapse.api.constants
+    :members:
+    :undoc-members:
+    :show-inheritance: