Basic release script

Fix error handling around when completing an AS transaction (#8693 )
Merge branch 'master' into develop
2020-10-30 19:28:53 +00:00 · 2020-10-30 16:50:48 +00:00 · 2020-10-30 16:27:02 +00:00 · 2020-10-30 15:33:33 +00:00 · 2020-10-30 15:27:58 +00:00 · 2020-10-30 15:25:44 +00:00
356 changed files with 10073 additions and 4382 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,22 +1,36 @@
-version: 2
+version: 2.1
 jobs:
  dockerhubuploadrelease:
-    machine: true
+    docker:
+      - image: docker:git
    steps:
      - checkout
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} .
+      - setup_remote_docker
+      - docker_prepare
      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
+      - docker_build:
+          tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
+          platforms: linux/amd64
+      - docker_build:
+          tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
+          platforms: linux/amd64,linux/arm/v7,linux/arm64
+
  dockerhubuploadlatest:
-    machine: true
+    docker:
+      - image: docker:git
    steps:
      - checkout
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest .
+      - setup_remote_docker
+      - docker_prepare
      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      - run: docker push matrixdotorg/synapse:latest
+      - docker_build:
+          tag: -t matrixdotorg/synapse:latest
+          platforms: linux/amd64
+      - docker_build:
+          tag: -t matrixdotorg/synapse:latest
+          platforms: linux/amd64,linux/arm/v7,linux/arm64

 workflows:
-  version: 2
  build:
    jobs:
      - dockerhubuploadrelease:
@@ -29,3 +43,33 @@ workflows:
          filters:
            branches:
              only: master
+
+commands:
+  docker_prepare:
+    description: Downloads the buildx cli plugin and enables multiarch images
+    parameters:
+      buildx_version:
+        type: string
+        default: "v0.4.1"
+    steps:
+      - run: apk add --no-cache curl
+      - run: mkdir -vp ~/.docker/cli-plugins/ ~/dockercache
+      - run: curl --silent -L "https://github.com/docker/buildx/releases/download/<< parameters.buildx_version >>/buildx-<< parameters.buildx_version >>.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
+      - run: chmod a+x ~/.docker/cli-plugins/docker-buildx
+      # install qemu links in /proc/sys/fs/binfmt_misc on the docker instance running the circleci job
+      - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+      # create a context named `builder` for the builds
+      - run: docker context create builder
+      # create a buildx builder using the new context, and set it as the default
+      - run: docker buildx create builder --use
+
+  docker_build:
+    description: Builds and pushed images to dockerhub using buildx
+    parameters:
+      platforms:
+        type: string
+        default: linux/amd64
+      tag:
+        type: string
+    steps:
+      - run: docker buildx build -f docker/Dockerfile --push --platform << parameters.platforms >> --label gitsha1=${CIRCLE_SHA1} << parameters.tag >> --progress=plain .
--- a/.gitignore
+++ b/.gitignore
@@ -21,6 +21,7 @@ _trial_temp*/
 /.python-version
 /*.signing.key
 /env/
+/.venv*/
 /homeserver*.yaml
 /logs
 /media_store/
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,171 @@
+Synapse 1.22.1 (2020-10-30)
+===========================
+
+Bugfixes
+--------
+
+- Fix a bug where an appservice may not be forwarded events for a room it was recently invited to. Broke in v1.22.0. ([\#8676](https://github.com/matrix-org/synapse/issues/8676))
+- Fix `Object of type frozendict is not JSON serializable` exceptions when using third-party event rules. Broke in v1.22.0. ([\#8678](https://github.com/matrix-org/synapse/issues/8678))
+
+
+Synapse 1.22.0 (2020-10-27)
+===========================
+
+No significant changes.
+
+
+Synapse 1.22.0rc2 (2020-10-26)
+==============================
+
+Bugfixes
+--------
+
+- Fix bugs where ephemeral events were not sent to appservices. Broke in v1.22.0rc1. ([\#8648](https://github.com/matrix-org/synapse/issues/8648), [\#8656](https://github.com/matrix-org/synapse/issues/8656))
+- Fix `user_daily_visits` table to not have duplicate rows per user/device due to multiple user agents. Broke in v1.22.0rc1. ([\#8654](https://github.com/matrix-org/synapse/issues/8654))
+
+Synapse 1.22.0rc1 (2020-10-22)
+==============================
+
+Features
+--------
+
+- Add a configuration option for always using the "userinfo endpoint" for OpenID Connect. This fixes support for some identity providers, e.g. GitLab. Contributed by Benjamin Koch. ([\#7658](https://github.com/matrix-org/synapse/issues/7658))
+- Add ability for `ThirdPartyEventRules` modules to query and manipulate whether a room is in the public rooms directory. ([\#8292](https://github.com/matrix-org/synapse/issues/8292), [\#8467](https://github.com/matrix-org/synapse/issues/8467))
+- Add support for olm fallback keys ([MSC2732](https://github.com/matrix-org/matrix-doc/pull/2732)). ([\#8312](https://github.com/matrix-org/synapse/issues/8312), [\#8501](https://github.com/matrix-org/synapse/issues/8501))
+- Add support for running background tasks in a separate worker process. ([\#8369](https://github.com/matrix-org/synapse/issues/8369), [\#8458](https://github.com/matrix-org/synapse/issues/8458), [\#8489](https://github.com/matrix-org/synapse/issues/8489), [\#8513](https://github.com/matrix-org/synapse/issues/8513), [\#8544](https://github.com/matrix-org/synapse/issues/8544), [\#8599](https://github.com/matrix-org/synapse/issues/8599))
+- Add support for device dehydration ([MSC2697](https://github.com/matrix-org/matrix-doc/pull/2697)). ([\#8380](https://github.com/matrix-org/synapse/issues/8380))
+- Add support for [MSC2409](https://github.com/matrix-org/matrix-doc/pull/2409), which allows sending typing, read receipts, and presence events to appservices. ([\#8437](https://github.com/matrix-org/synapse/issues/8437), [\#8590](https://github.com/matrix-org/synapse/issues/8590))
+- Change default room version to "6", per [MSC2788](https://github.com/matrix-org/matrix-doc/pull/2788). ([\#8461](https://github.com/matrix-org/synapse/issues/8461))
+- Add the ability to send non-membership events into a room via the `ModuleApi`. ([\#8479](https://github.com/matrix-org/synapse/issues/8479))
+- Increase default upload size limit from 10M to 50M. Contributed by @Akkowicz. ([\#8502](https://github.com/matrix-org/synapse/issues/8502))
+- Add support for modifying event content in `ThirdPartyRules` modules. ([\#8535](https://github.com/matrix-org/synapse/issues/8535), [\#8564](https://github.com/matrix-org/synapse/issues/8564))
+
+
+Bugfixes
+--------
+
+- Fix a longstanding bug where invalid ignored users in account data could break clients. ([\#8454](https://github.com/matrix-org/synapse/issues/8454))
+- Fix a bug where backfilling a room with an event that was missing the `redacts` field would break. ([\#8457](https://github.com/matrix-org/synapse/issues/8457))
+- Don't attempt to respond to some requests if the client has already disconnected. ([\#8465](https://github.com/matrix-org/synapse/issues/8465))
+- Fix message duplication if something goes wrong after persisting the event. ([\#8476](https://github.com/matrix-org/synapse/issues/8476))
+- Fix incremental sync returning an incorrect `prev_batch` token in timeline section, which when used to paginate returned events that were included in the incremental sync. Broken since v0.16.0. ([\#8486](https://github.com/matrix-org/synapse/issues/8486))
+- Expose the `uk.half-shot.msc2778.login.application_service` to clients from the login API. This feature was added in v1.21.0, but was not exposed as a potential login flow. ([\#8504](https://github.com/matrix-org/synapse/issues/8504))
+- Fix error code for `/profile/{userId}/displayname` to be `M_BAD_JSON`. ([\#8517](https://github.com/matrix-org/synapse/issues/8517))
+- Fix a bug introduced in v1.7.0 that could cause Synapse to insert values from non-state `m.room.retention` events into the `room_retention` database table. ([\#8527](https://github.com/matrix-org/synapse/issues/8527))
+- Fix not sending events over federation when using sharded event writers. ([\#8536](https://github.com/matrix-org/synapse/issues/8536))
+- Fix a long standing bug where email notifications for encrypted messages were blank. ([\#8545](https://github.com/matrix-org/synapse/issues/8545))
+- Fix increase in the number of `There was no active span...` errors logged when using OpenTracing. ([\#8567](https://github.com/matrix-org/synapse/issues/8567))
+- Fix a bug that prevented errors encountered during execution of the `synapse_port_db` from being correctly printed. ([\#8585](https://github.com/matrix-org/synapse/issues/8585))
+- Fix appservice transactions to only include a maximum of 100 persistent and 100 ephemeral events. ([\#8606](https://github.com/matrix-org/synapse/issues/8606))
+
+
+Updates to the Docker image
+---------------------------
+
+- Added multi-arch support (arm64,arm/v7) for the docker images. Contributed by @maquis196. ([\#7921](https://github.com/matrix-org/synapse/issues/7921))
+- Add support for passing commandline args to the synapse process. Contributed by @samuel-p. ([\#8390](https://github.com/matrix-org/synapse/issues/8390))
+
+
+Improved Documentation
+----------------------
+
+- Update the directions for using the manhole with coroutines. ([\#8462](https://github.com/matrix-org/synapse/issues/8462))
+- Improve readme by adding new shield.io badges. ([\#8493](https://github.com/matrix-org/synapse/issues/8493))
+- Added note about docker in manhole.md regarding which ip address to bind to. Contributed by @Maquis196. ([\#8526](https://github.com/matrix-org/synapse/issues/8526))
+- Document the new behaviour of the `allowed_lifetime_min` and `allowed_lifetime_max` settings in the room retention configuration. ([\#8529](https://github.com/matrix-org/synapse/issues/8529))
+
+
+Deprecations and Removals
+-------------------------
+
+- Drop unused `device_max_stream_id` table. ([\#8589](https://github.com/matrix-org/synapse/issues/8589))
+
+
+Internal Changes
+----------------
+
+- Check for unreachable code with mypy. ([\#8432](https://github.com/matrix-org/synapse/issues/8432))
+- Add unit test for event persister sharding. ([\#8433](https://github.com/matrix-org/synapse/issues/8433))
+- Allow events to be sent to clients sooner when using sharded event persisters. ([\#8439](https://github.com/matrix-org/synapse/issues/8439), [\#8488](https://github.com/matrix-org/synapse/issues/8488), [\#8496](https://github.com/matrix-org/synapse/issues/8496), [\#8499](https://github.com/matrix-org/synapse/issues/8499))
+- Configure `public_baseurl` when using demo scripts. ([\#8443](https://github.com/matrix-org/synapse/issues/8443))
+- Add SQL logging on queries that happen during startup. ([\#8448](https://github.com/matrix-org/synapse/issues/8448))
+- Speed up unit tests when using PostgreSQL. ([\#8450](https://github.com/matrix-org/synapse/issues/8450))
+- Remove redundant database loads of stream_ordering for events we already have. ([\#8452](https://github.com/matrix-org/synapse/issues/8452))
+- Reduce inconsistencies between codepaths for membership and non-membership events. ([\#8463](https://github.com/matrix-org/synapse/issues/8463))
+- Combine `SpamCheckerApi` with the more generic `ModuleApi`. ([\#8464](https://github.com/matrix-org/synapse/issues/8464))
+- Additional testing for `ThirdPartyEventRules`. ([\#8468](https://github.com/matrix-org/synapse/issues/8468))
+- Add `-d` option to `./scripts-dev/lint.sh` to lint files that have changed since the last git commit. ([\#8472](https://github.com/matrix-org/synapse/issues/8472))
+- Unblacklist some sytests. ([\#8474](https://github.com/matrix-org/synapse/issues/8474))
+- Include the log level in the phone home stats. ([\#8477](https://github.com/matrix-org/synapse/issues/8477))
+- Remove outdated sphinx documentation, scripts and configuration. ([\#8480](https://github.com/matrix-org/synapse/issues/8480))
+- Clarify error message when plugin config parsers raise an error. ([\#8492](https://github.com/matrix-org/synapse/issues/8492))
+- Remove the deprecated `Handlers` object. ([\#8494](https://github.com/matrix-org/synapse/issues/8494))
+- Fix a threadsafety bug in unit tests. ([\#8497](https://github.com/matrix-org/synapse/issues/8497))
+- Add user agent to user_daily_visits table. ([\#8503](https://github.com/matrix-org/synapse/issues/8503))
+- Add type hints to various parts of the code base. ([\#8407](https://github.com/matrix-org/synapse/issues/8407), [\#8505](https://github.com/matrix-org/synapse/issues/8505), [\#8507](https://github.com/matrix-org/synapse/issues/8507), [\#8547](https://github.com/matrix-org/synapse/issues/8547), [\#8562](https://github.com/matrix-org/synapse/issues/8562), [\#8609](https://github.com/matrix-org/synapse/issues/8609))
+- Remove unused code from the test framework. ([\#8514](https://github.com/matrix-org/synapse/issues/8514))
+- Apply some internal fixes to the `HomeServer` class to make its code more idiomatic and statically-verifiable. ([\#8515](https://github.com/matrix-org/synapse/issues/8515))
+- Factor out common code between `RoomMemberHandler._locally_reject_invite` and `EventCreationHandler.create_event`. ([\#8537](https://github.com/matrix-org/synapse/issues/8537))
+- Improve database performance by executing more queries without starting transactions. ([\#8542](https://github.com/matrix-org/synapse/issues/8542))
+- Rename `Cache` to `DeferredCache`, to better reflect its purpose. ([\#8548](https://github.com/matrix-org/synapse/issues/8548))
+- Move metric registration code down into `LruCache`. ([\#8561](https://github.com/matrix-org/synapse/issues/8561), [\#8591](https://github.com/matrix-org/synapse/issues/8591))
+- Replace `DeferredCache` with the lighter-weight `LruCache` where possible. ([\#8563](https://github.com/matrix-org/synapse/issues/8563))
+- Add virtualenv-generated folders to `.gitignore`. ([\#8566](https://github.com/matrix-org/synapse/issues/8566))
+- Add `get_immediate` method to `DeferredCache`. ([\#8568](https://github.com/matrix-org/synapse/issues/8568))
+- Fix mypy not properly checking across the codebase, additionally, fix a typing assertion error in `handlers/auth.py`. ([\#8569](https://github.com/matrix-org/synapse/issues/8569))
+- Fix `synmark` benchmark runner. ([\#8571](https://github.com/matrix-org/synapse/issues/8571))
+- Modify `DeferredCache.get()` to return `Deferred`s instead of `ObservableDeferred`s. ([\#8572](https://github.com/matrix-org/synapse/issues/8572))
+- Adjust a protocol-type definition to fit `sqlite3` assertions. ([\#8577](https://github.com/matrix-org/synapse/issues/8577))
+- Support macOS on the `synmark` benchmark runner. ([\#8578](https://github.com/matrix-org/synapse/issues/8578))
+- Update `mypy` static type checker to 0.790. ([\#8583](https://github.com/matrix-org/synapse/issues/8583), [\#8600](https://github.com/matrix-org/synapse/issues/8600))
+- Re-organize the structured logging code to separate the TCP transport handling from the JSON formatting. ([\#8587](https://github.com/matrix-org/synapse/issues/8587))
+- Remove extraneous unittest logging decorators from unit tests. ([\#8592](https://github.com/matrix-org/synapse/issues/8592))
+- Minor optimisations in caching code. ([\#8593](https://github.com/matrix-org/synapse/issues/8593), [\#8594](https://github.com/matrix-org/synapse/issues/8594))
+
+
+Synapse 1.21.2 (2020-10-15)
+===========================
+
+Debian packages and Docker images have been rebuilt using the latest versions of dependency libraries, including authlib 0.15.1. Please see bugfixes below.
+
+Security advisory
+-----------------
+
+* HTML pages served via Synapse were vulnerable to cross-site scripting (XSS)
+  attacks. All server administrators are encouraged to upgrade.
+  ([\#8444](https://github.com/matrix-org/synapse/pull/8444))
+  ([CVE-2020-26891](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26891))
+
+  This fix was originally included in v1.21.0 but was missing a security advisory.
+
+  This was reported by [Denis Kasak](https://github.com/dkasak).
+
+Bugfixes
+--------
+
+- Fix rare bug where sending an event would fail due to a racey assertion. ([\#8530](https://github.com/matrix-org/synapse/issues/8530))
+- An updated version of the authlib dependency is included in the Docker and Debian images to fix an issue using OpenID Connect. See [\#8534](https://github.com/matrix-org/synapse/issues/8534) for details.
+
+
+Synapse 1.21.1 (2020-10-13)
+===========================
+
+This release fixes a regression in v1.21.0 that prevented debian packages from being built.
+It is otherwise identical to v1.21.0.
+
+Synapse 1.21.0 (2020-10-12)
+===========================
+
+No significant changes since v1.21.0rc3.
+
+As [noted in
+v1.20.0](https://github.com/matrix-org/synapse/blob/release-v1.21.0/CHANGES.md#synapse-1200-2020-09-22),
+a future release will drop support for accessing Synapse's
+[Admin API](https://github.com/matrix-org/synapse/tree/master/docs/admin_api) under the
+`/_matrix/client/*` endpoint prefixes. At that point, the Admin API will only
+be accessible under `/_synapse/admin`.
+
+
 Synapse 1.21.0rc3 (2020-10-08)
 ==============================

@@ -154,9 +322,11 @@ API](https://github.com/matrix-org/synapse/tree/master/docs) has been
 accessible under the `/_matrix/client/api/v1/admin`,
 `/_matrix/client/unstable/admin`, `/_matrix/client/r0/admin` and
 `/_synapse/admin` prefixes. In a future release, we will be dropping support
-for accessing Synapse's Admin API using the `/_matrix/client/*` prefixes. This
-makes it easier for homeserver admins to lock down external access to the Admin
-API endpoints.
+for accessing Synapse's Admin API using the `/_matrix/client/*` prefixes.
+
+From that point, the Admin API will only be accessible under `/_synapse/admin`.
+This makes it easier for homeserver admins to lock down external access to the
+Admin API endpoints.

 Synapse 1.20.0rc5 (2020-09-18)
 ==============================
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -46,7 +46,7 @@ locally. You'll need python 3.6 or later, and to install a number of tools:

 ```
 # Install the dependencies
-pip install -e ".[lint]"
+pip install -e ".[lint,mypy]"

 # Run the linter script
 ./scripts-dev/lint.sh
@@ -63,6 +63,10 @@ run-time:
 ./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder
 ```

+You can also provide the `-d` option, which will lint the files that have been
+changed since the last git commit. This will often be significantly faster than
+linting the whole codebase.
+
 Before pushing new changes, ensure they don't produce linting errors. Commit any
 files that were corrected.

--- a/INSTALL.md
+++ b/INSTALL.md
@@ -57,7 +57,7 @@ light workloads.
 System requirements:

 - POSIX-compliant system (tested on Linux & OS X)
- Python 3.5.2 or later, up to Python 3.8.
+- Python 3.5.2 or later, up to Python 3.9.
 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org

 Synapse is written in Python but some of the libraries it uses are written in
--- a/README.rst
+++ b/README.rst
@@ -256,9 +256,9 @@ directory of your choice::
 Synapse has a number of external dependencies, that are easiest
 to install using pip and a virtualenv::

-    virtualenv -p python3 env
-    source env/bin/activate
-    python -m pip install --no-use-pep517 -e ".[all]"
+    python3 -m venv ./env
+    source ./env/bin/activate
+    pip install -e ".[all,test]"

 This will run a process of downloading and installing all the needed
 dependencies into a virtual env.
@@ -270,9 +270,9 @@ check that everything is installed as it should be::

 This should end with a 'PASSED' result::

-    Ran 143 tests in 0.601s
+    Ran 1266 tests in 643.930s

-    PASSED (successes=143)
+    PASSED (skips=15, successes=1251)

 Running the Integration Tests
 =============================
--- a/UPGRADE.rst
+++ b/UPGRADE.rst
@@ -75,6 +75,22 @@ for example:
     wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
     dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb

+Upgrading to v1.23.0
+====================
+
+Structured logging configuration breaking changes
+-------------------------------------------------
+
+This release deprecates use of the ``structured: true`` logging configuration for
+structured logging. If your logging configuration contains ``structured: true``
+then it should be modified based on the `structured logging documentation
+<https://github.com/matrix-org/synapse/blob/master/docs/structured_logging.md>`_.
+
+The ``structured`` and ``drains`` logging options are now deprecated and should
+be replaced by standard logging configuration of ``handlers`` and ``formatters`.
+
+A future will release of Synapse will make using ``structured: true`` an error.
+
 Upgrading to v1.22.0
 ====================

--- a/changelog.d/7658.feature
+++ b/changelog.d/7658.feature
@@ -1 +0,0 @@
-Add a configuration option for always using the "userinfo endpoint" for OpenID Connect. This fixes support for some identity providers, e.g. GitLab. Contributed by Benjamin Koch.
--- a/changelog.d/8292.feature
+++ b/changelog.d/8292.feature
@@ -1 +0,0 @@
-Allow `ThirdPartyEventRules` modules to query and manipulate whether a room is in the public rooms directory.
--- a/changelog.d/8312.feature
+++ b/changelog.d/8312.feature
@@ -1 +0,0 @@
-Add support for olm fallback keys ([MSC2732](https://github.com/matrix-org/matrix-doc/pull/2732)).
--- a/changelog.d/8369.feature
+++ b/changelog.d/8369.feature
@@ -1 +0,0 @@
-Allow running background tasks in a separate worker process.
--- a/changelog.d/8380.feature
+++ b/changelog.d/8380.feature
@@ -1 +0,0 @@
-Add support for device dehydration ([MSC2697](https://github.com/matrix-org/matrix-doc/pull/2697)).
--- a/changelog.d/8407.misc
+++ b/changelog.d/8407.misc
@@ -1 +0,0 @@
-Add typing information to the device handler.
--- a/changelog.d/8432.misc
+++ b/changelog.d/8432.misc
@@ -1 +0,0 @@
-Check for unreachable code with mypy.
--- a/changelog.d/8433.misc
+++ b/changelog.d/8433.misc
@@ -1 +0,0 @@
-Add unit test for event persister sharding.
--- a/changelog.d/8439.misc
+++ b/changelog.d/8439.misc
@@ -1 +0,0 @@
-Allow events to be sent to clients sooner when using sharded event persisters.
--- a/changelog.d/8443.misc
+++ b/changelog.d/8443.misc
@@ -1 +0,0 @@
-Configure `public_baseurl` when using demo scripts.
--- a/changelog.d/8448.misc
+++ b/changelog.d/8448.misc
@@ -1 +0,0 @@
-Add SQL logging on queries that happen during startup.
--- a/changelog.d/8450.misc
+++ b/changelog.d/8450.misc
@@ -1 +0,0 @@
-Speed up unit tests when using PostgreSQL.
--- a/changelog.d/8452.misc
+++ b/changelog.d/8452.misc
@@ -1 +0,0 @@
-Remove redundant databae loads of stream_ordering for events we already have.
--- a/changelog.d/8454.bugfix
+++ b/changelog.d/8454.bugfix
@@ -1 +0,0 @@
-Fix a longstanding bug where invalid ignored users in account data could break clients.
--- a/changelog.d/8455.bugfix
+++ b/changelog.d/8455.bugfix
@@ -0,0 +1 @@
+Fix fetching of E2E cross signing keys over federation when only one of the master key and device signing key is cached already.
--- a/changelog.d/8457.bugfix
+++ b/changelog.d/8457.bugfix
@@ -1 +0,0 @@
-Fix a bug where backfilling a room with an event that was missing the `redacts` field would break.
--- a/changelog.d/8458.feature
+++ b/changelog.d/8458.feature
@@ -1 +0,0 @@
-Allow running background tasks in a separate worker process.
--- a/changelog.d/8461.feature
+++ b/changelog.d/8461.feature
@@ -1 +0,0 @@
-Change default room version to "6", per [MSC2788](https://github.com/matrix-org/matrix-doc/pull/2788).
--- a/changelog.d/8462.doc
+++ b/changelog.d/8462.doc
@@ -1 +0,0 @@
-Update the directions for using the manhole with coroutines.
--- a/changelog.d/8463.misc
+++ b/changelog.d/8463.misc
@@ -1 +0,0 @@
-Reduce inconsistencies between codepaths for membership and non-membership events.
--- a/changelog.d/8464.misc
+++ b/changelog.d/8464.misc
@@ -1 +0,0 @@
-Combine `SpamCheckerApi` with the more generic `ModuleApi`.
--- a/changelog.d/8465.bugfix
+++ b/changelog.d/8465.bugfix
@@ -1 +0,0 @@
-Don't attempt to respond to some requests if the client has already disconnected.
--- a/changelog.d/8467.feature
+++ b/changelog.d/8467.feature
@@ -1 +0,0 @@
-Allow `ThirdPartyEventRules` modules to query and manipulate whether a room is in the public rooms directory.
--- a/changelog.d/8468.misc
+++ b/changelog.d/8468.misc
@@ -1 +0,0 @@
-Additional testing for `ThirdPartyEventRules`.
--- a/changelog.d/8474.misc
+++ b/changelog.d/8474.misc
@@ -1 +0,0 @@
-Unblacklist some sytests.
--- a/changelog.d/8477.misc
+++ b/changelog.d/8477.misc
@@ -1 +0,0 @@
-Include the log level in the phone home stats.
--- a/changelog.d/8480.misc
+++ b/changelog.d/8480.misc
@@ -1 +0,0 @@
-Remove outdated sphinx documentation, scripts and configuration.
--- a/changelog.d/8486.bugfix
+++ b/changelog.d/8486.bugfix
@@ -1 +0,0 @@
-Fix incremental sync returning an incorrect `prev_batch` token in timeline section, which when used to paginate returned events that were included in the incremental sync. Broken since v0.16.0.
--- a/changelog.d/8492.misc
+++ b/changelog.d/8492.misc
@@ -1 +0,0 @@
-Clarify error message when plugin config parsers raise an error.
--- a/changelog.d/8493.doc
+++ b/changelog.d/8493.doc
@@ -1 +0,0 @@
-Improve readme by adding new shield.io badges.
--- a/changelog.d/8519.feature
+++ b/changelog.d/8519.feature
@@ -0,0 +1 @@
+Add an admin api to delete a single file or files were not used for a defined time from server. Contributed by @dklimpel.
--- a/changelog.d/8539.feature
+++ b/changelog.d/8539.feature
@@ -0,0 +1 @@
+Split admin API for reported events (`GET /_synapse/admin/v1/event_reports`) into detail and list endpoints. This is a breaking change to #8217 which was introduced in Synapse v1.21.0. Those who already use this API should check their scripts. Contributed by @dklimpel.
--- a/changelog.d/8559.misc
+++ b/changelog.d/8559.misc
@@ -0,0 +1 @@
+Optimise `/createRoom` with multiple invited users.
--- a/changelog.d/8580.bugfix
+++ b/changelog.d/8580.bugfix
@@ -0,0 +1 @@
+Fix a bug where Synapse would blindly forward bad responses from federation to clients when retrieving profile information.
--- a/changelog.d/8582.doc
+++ b/changelog.d/8582.doc
@@ -0,0 +1 @@
+Instructions for Azure AD in the OpenID Connect documentation. Contributed by peterk.
--- a/changelog.d/8595.misc
+++ b/changelog.d/8595.misc
@@ -0,0 +1 @@
+Implement and use an @lru_cache decorator.
--- a/changelog.d/8607.feature
+++ b/changelog.d/8607.feature
@@ -0,0 +1 @@
+Support generating structured logs via the standard logging configuration.
--- a/changelog.d/8610.feature
+++ b/changelog.d/8610.feature
@@ -0,0 +1 @@
+Add an admin APIs to allow server admins to list users' pushers. Contributed by @dklimpel.
--- a/changelog.d/8614.misc
+++ b/changelog.d/8614.misc
@@ -0,0 +1 @@
+Don't instansiate Requester directly.
--- a/changelog.d/8615.misc
+++ b/changelog.d/8615.misc
@@ -0,0 +1 @@
+Type hints for `RegistrationStore`.
--- a/changelog.d/8616.misc
+++ b/changelog.d/8616.misc
@@ -0,0 +1 @@
+Change schema to support access tokens belonging to one user but granting access to another.
--- a/changelog.d/8620.bugfix
+++ b/changelog.d/8620.bugfix
@@ -0,0 +1 @@
+Fix a bug where the account validity endpoint would silently fail if the user ID did not have an expiration time. It now returns a 400 error.
--- a/changelog.d/8621.misc
+++ b/changelog.d/8621.misc
@@ -0,0 +1 @@
+Remove unused OPTIONS handlers.
--- a/changelog.d/8627.bugfix
+++ b/changelog.d/8627.bugfix
@@ -0,0 +1 @@
+Fix email notifications for invites without local state.
--- a/changelog.d/8628.bugfix
+++ b/changelog.d/8628.bugfix
@@ -0,0 +1 @@
+Fix handling of invalid group IDs to return a 400 rather than log an exception and return a 500.
--- a/changelog.d/8632.bugfix
+++ b/changelog.d/8632.bugfix
@@ -0,0 +1 @@
+Fix handling of User-Agent headers that are invalid UTF-8, which caused user agents of users to not get correctly recorded.
--- a/changelog.d/8633.misc
+++ b/changelog.d/8633.misc
@@ -0,0 +1 @@
+Run `mypy` as part of the lint.sh script.
--- a/changelog.d/8634.misc
+++ b/changelog.d/8634.misc
@@ -0,0 +1 @@
+Correct Synapse's PyPI package name in the OpenID Connect installation instructions.
--- a/changelog.d/8635.doc
+++ b/changelog.d/8635.doc
@@ -0,0 +1 @@
+Improve the sample configuration for single sign-on providers.
--- a/changelog.d/8639.misc
+++ b/changelog.d/8639.misc
@@ -0,0 +1 @@
+Fix typos and spelling errors in the code.
--- a/changelog.d/8640.misc
+++ b/changelog.d/8640.misc
@@ -0,0 +1 @@
+Reduce number of OpenTracing spans started.
--- a/changelog.d/8643.bugfix
+++ b/changelog.d/8643.bugfix
@@ -0,0 +1 @@
+Fix a bug in the `joined_rooms` admin API if the user has never joined any rooms. The bug was introduced, along with the API, in v1.21.0.
--- a/changelog.d/8644.misc
+++ b/changelog.d/8644.misc
@@ -0,0 +1 @@
+Add field `total` to device list in admin API.
--- a/changelog.d/8647.feature
+++ b/changelog.d/8647.feature
@@ -0,0 +1 @@
+Add an admin API `GET /_synapse/admin/v1/users/<user_id>/media` to get information about uploaded media. Contributed by @dklimpel.
--- a/changelog.d/8655.misc
+++ b/changelog.d/8655.misc
@@ -0,0 +1 @@
+Add more type hints to the application services code.
--- a/changelog.d/8657.doc
+++ b/changelog.d/8657.doc
@@ -0,0 +1 @@
+Fix the filepath of Dex's example config and the link to Dex's Getting Started guide in the OpenID Connect docs.
--- a/changelog.d/8664.misc
+++ b/changelog.d/8664.misc
@@ -0,0 +1 @@
+Tell Black to format code for Python 3.5.
--- a/changelog.d/8665.doc
+++ b/changelog.d/8665.doc
@@ -0,0 +1 @@
+Note support for Python 3.9.
--- a/changelog.d/8666.doc
+++ b/changelog.d/8666.doc
@@ -0,0 +1 @@
+Minor updates to docs on running tests.
--- a/changelog.d/8667.doc
+++ b/changelog.d/8667.doc
@@ -0,0 +1 @@
+Interlink prometheus/grafana documentation.
--- a/changelog.d/8668.misc
+++ b/changelog.d/8668.misc
@@ -0,0 +1 @@
+Reduce number of OpenTracing spans started.
--- a/changelog.d/8669.misc
+++ b/changelog.d/8669.misc
@@ -0,0 +1 @@
+Don't pull event from DB when handling replication traffic.
--- a/changelog.d/8670.misc
+++ b/changelog.d/8670.misc
@@ -0,0 +1 @@
+Reduce number of OpenTracing spans started.
--- a/changelog.d/8671.misc
+++ b/changelog.d/8671.misc
@@ -0,0 +1 @@
+Abstract some invite-related code in preparation for landing knocking.
--- a/changelog.d/8679.misc
+++ b/changelog.d/8679.misc
@@ -0,0 +1 @@
+Clarify representation of events in logfiles.
--- a/changelog.d/8680.misc
+++ b/changelog.d/8680.misc
@@ -0,0 +1 @@
+Don't require `hiredis` package to be installed to run unit tests.
--- a/changelog.d/8682.bugfix
+++ b/changelog.d/8682.bugfix
@@ -0,0 +1 @@
+Fix exception during handling multiple concurrent requests for remote media when using multiple media repositories.
--- a/changelog.d/8684.misc
+++ b/changelog.d/8684.misc
@@ -0,0 +1 @@
+Fix typing info on cache call signature to accept `on_invalidate`.
--- a/changelog.d/8685.feature
+++ b/changelog.d/8685.feature
@@ -0,0 +1 @@
+Support generating structured logs via the standard logging configuration.
--- a/changelog.d/8688.misc
+++ b/changelog.d/8688.misc
@@ -0,0 +1 @@
+Abstract some invite-related code in preparation for landing knocking.
--- a/changelog.d/8689.feature
+++ b/changelog.d/8689.feature
@@ -0,0 +1 @@
+Add an admin APIs to allow server admins to list users' pushers. Contributed by @dklimpel.
--- a/changelog.d/8690.misc
+++ b/changelog.d/8690.misc
@@ -0,0 +1 @@
+Fail tests if they do not await coroutines.
--- a/changelog.d/8693.misc
+++ b/changelog.d/8693.misc
@@ -0,0 +1 @@
+Add more type hints to the application services code.
--- a/contrib/grafana/README.md
+++ b/contrib/grafana/README.md
@@ -3,4 +3,4 @@
 0. Set up Prometheus and Grafana. Out of scope for this readme. Useful documentation about using Grafana with Prometheus: http://docs.grafana.org/features/datasources/prometheus/
 1. Have your Prometheus scrape your Synapse. https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md
 2. Import dashboard into Grafana. Download `synapse.json`. Import it to Grafana and select the correct Prometheus datasource. http://docs.grafana.org/reference/export_import/
-3. Set up additional recording rules
+3. Set up required recording rules. https://github.com/matrix-org/synapse/tree/master/contrib/prometheus
--- a/debian/build_virtualenv
+++ b/debian/build_virtualenv
@@ -42,7 +42,7 @@ dh_virtualenv \
    --preinstall="mock" \
    --extra-pip-arg="--no-cache-dir" \
    --extra-pip-arg="--compile" \
-    --extras="all,systemd"
+    --extras="all,systemd,test"

 PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
 VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,38 @@
+matrix-synapse-py3 (1.22.1) stable; urgency=medium
+
+  * New synapse release 1.22.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Fri, 30 Oct 2020 15:25:37 +0000
+
+matrix-synapse-py3 (1.22.0) stable; urgency=medium
+
+  * New synapse release 1.22.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 27 Oct 2020 12:07:12 +0000
+
+matrix-synapse-py3 (1.21.2) stable; urgency=medium
+
+  [ Synapse Packaging team ]
+  * New synapse release 1.21.2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 15 Oct 2020 09:23:27 -0400
+
+matrix-synapse-py3 (1.21.1) stable; urgency=medium
+
+  [ Synapse Packaging team ]
+  * New synapse release 1.21.1.
+
+  [ Andrew Morgan ]
+  * Explicitly install "test" python dependencies.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Oct 2020 10:24:13 +0100
+
+matrix-synapse-py3 (1.21.0) stable; urgency=medium
+
+  * New synapse release 1.21.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Mon, 12 Oct 2020 15:47:44 +0100
+
 matrix-synapse-py3 (1.20.1) stable; urgency=medium

  * New synapse release 1.20.1.
--- a/docker/README.md
+++ b/docker/README.md
@@ -83,7 +83,7 @@ docker logs synapse
 If all is well, you should now be able to connect to http://localhost:8008 and
 see a confirmation message.

-The following environment variables are supported in run mode:
+The following environment variables are supported in `run` mode:

 * `SYNAPSE_CONFIG_DIR`: where additional config files are stored. Defaults to
  `/data`.
@@ -94,6 +94,20 @@ The following environment variables are supported in run mode:
 * `UID`, `GID`: the user and group id to run Synapse as. Defaults to `991`, `991`.
 * `TZ`: the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) the container will run with. Defaults to `UTC`.

+For more complex setups (e.g. for workers) you can also pass your args directly to synapse using `run` mode. For example like this:
+
+```
+docker run -d --name synapse \
+    --mount type=volume,src=synapse-data,dst=/data \
+    -p 8008:8008 \
+    matrixdotorg/synapse:latest run \
+    -m synapse.app.generic_worker \
+    --config-path=/data/homeserver.yaml \
+    --config-path=/data/generic_worker.yaml
+```
+
+If you do not provide `-m`, the value of the `SYNAPSE_WORKER` environment variable is used. If you do not provide at least one `--config-path` or `-c`, the value of the `SYNAPSE_CONFIG_PATH` environment variable is used instead.
+
 ## Generating an (admin) user

 After synapse is running, you may wish to create a user via `register_new_matrix_user`.
--- a/docker/conf/homeserver.yaml
+++ b/docker/conf/homeserver.yaml
@@ -90,7 +90,7 @@ federation_rc_concurrent: 3

 media_store_path: "/data/media"
 uploads_path: "/data/uploads"
-max_upload_size: "{{ SYNAPSE_MAX_UPLOAD_SIZE or "10M" }}"
+max_upload_size: "{{ SYNAPSE_MAX_UPLOAD_SIZE or "50M" }}"
 max_image_pixels: "32M"
 dynamic_thumbnails: false

--- a/docker/start.py
+++ b/docker/start.py
@@ -179,7 +179,7 @@ def run_generate_config(environ, ownership):


 def main(args, environ):
-    mode = args[1] if len(args) > 1 else None
+    mode = args[1] if len(args) > 1 else "run"
    desired_uid = int(environ.get("UID", "991"))
    desired_gid = int(environ.get("GID", "991"))
    synapse_worker = environ.get("SYNAPSE_WORKER", "synapse.app.homeserver")
@@ -205,36 +205,47 @@ def main(args, environ):
            config_dir, config_path, environ, ownership
        )

-    if mode is not None:
+    if mode != "run":
        error("Unknown execution mode '%s'" % (mode,))

-    config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
-    config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
+    args = args[2:]

-    if not os.path.exists(config_path):
-        if "SYNAPSE_SERVER_NAME" in environ:
-            error(
-                """\
+    if "-m" not in args:
+        args = ["-m", synapse_worker] + args
+
+    # if there are no config files passed to synapse, try adding the default file
+    if not any(p.startswith("--config-path") or p.startswith("-c") for p in args):
+        config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
+        config_path = environ.get(
+            "SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml"
+        )
+
+        if not os.path.exists(config_path):
+            if "SYNAPSE_SERVER_NAME" in environ:
+                error(
+                    """\
 Config file '%s' does not exist.

 The synapse docker image no longer supports generating a config file on-the-fly
 based on environment variables. You can migrate to a static config file by
 running with 'migrate_config'. See the README for more details.
 """
+                    % (config_path,)
+                )
+
+            error(
+                "Config file '%s' does not exist. You should either create a new "
+                "config file by running with the `generate` argument (and then edit "
+                "the resulting file before restarting) or specify the path to an "
+                "existing config file with the SYNAPSE_CONFIG_PATH variable."
                % (config_path,)
            )

-        error(
-            "Config file '%s' does not exist. You should either create a new "
-            "config file by running with the `generate` argument (and then edit "
-            "the resulting file before restarting) or specify the path to an "
-            "existing config file with the SYNAPSE_CONFIG_PATH variable."
-            % (config_path,)
-        )
+        args += ["--config-path", config_path]

-    log("Starting synapse with config file " + config_path)
+    log("Starting synapse with args " + " ".join(args))

-    args = ["python", "-m", synapse_worker, "--config-path", config_path]
+    args = ["python"] + args
    if ownership is not None:
        args = ["gosu", ownership] + args
        os.execv("/usr/sbin/gosu", args)
--- a/docs/admin_api/event_reports.rst
+++ b/docs/admin_api/event_reports.rst
@@ -17,67 +17,26 @@ It returns a JSON body like the following:
    {
        "event_reports": [
            {
-                "content": {
-                    "reason": "foo",
-                    "score": -100
-                },
                "event_id": "$bNUFCwGzWca1meCGkjp-zwslF-GfVcXukvRLI1_FaVY",
-                "event_json": {
-                    "auth_events": [
-                        "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M",
-                        "$oggsNXxzPFRE3y53SUNd7nsj69-QzKv03a1RucHu-ws"
-                    ],
-                    "content": {
-                        "body": "matrix.org: This Week in Matrix",
-                        "format": "org.matrix.custom.html",
-                        "formatted_body": "<strong>matrix.org</strong>:<br><a href=\"https://matrix.org/blog/\"><strong>This Week in Matrix</strong></a>",
-                        "msgtype": "m.notice"
-                    },
-                    "depth": 546,
-                    "hashes": {
-                        "sha256": "xK1//xnmvHJIOvbgXlkI8eEqdvoMmihVDJ9J4SNlsAw"
-                    },
-                    "origin": "matrix.org",
-                    "origin_server_ts": 1592291711430,
-                    "prev_events": [
-                        "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M"
-                    ],
-                    "prev_state": [],
-                    "room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org",
-                    "sender": "@foobar:matrix.org",
-                    "signatures": {
-                        "matrix.org": {
-                            "ed25519:a_JaEG": "cs+OUKW/iHx5pEidbWxh0UiNNHwe46Ai9LwNz+Ah16aWDNszVIe2gaAcVZfvNsBhakQTew51tlKmL2kspXk/Dg"
-                        }
-                    },
-                    "type": "m.room.message",
-                    "unsigned": {
-                        "age_ts": 1592291711430,
-                    }
-                },
                "id": 2,
                "reason": "foo",
+                "score": -100,
                "received_ts": 1570897107409,
-                "room_alias": "#alias1:matrix.org",
+                "canonical_alias": "#alias1:matrix.org",
                "room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org",
+                "name": "Matrix HQ",
                "sender": "@foobar:matrix.org",
                "user_id": "@foo:matrix.org"
            },
            {
-                "content": {
-                    "reason": "bar",
-                    "score": -100
-                },
                "event_id": "$3IcdZsDaN_En-S1DF4EMCy3v4gNRKeOJs8W5qTOKj4I",
-                "event_json": {
-                    // hidden items
-                    // see above
-                },
                "id": 3,
                "reason": "bar",
+                "score": -100,
                "received_ts": 1598889612059,
-                "room_alias": "#alias2:matrix.org",
+                "canonical_alias": "#alias2:matrix.org",
                "room_id": "!eGvUQuTCkHGVwNMOjv:matrix.org",
+                "name": "Your room name here",
                "sender": "@foobar:matrix.org",
                "user_id": "@bar:matrix.org"
            }
@@ -113,17 +72,94 @@ The following fields are returned in the JSON response body:
 - ``id``: integer - ID of event report.
 - ``received_ts``: integer - The timestamp (in milliseconds since the unix epoch) when this report was sent.
 - ``room_id``: string - The ID of the room in which the event being reported is located.
+- ``name``: string - The name of the room.
 - ``event_id``: string - The ID of the reported event.
 - ``user_id``: string - This is the user who reported the event and wrote the reason.
 - ``reason``: string - Comment made by the ``user_id`` in this report. May be blank.
- ``content``: object - Content of reported event.
-
-  - ``reason``: string - Comment made by the ``user_id`` in this report. May be blank.
-  - ``score``: integer - Content is reported based upon a negative score, where -100 is "most offensive" and 0 is "inoffensive".
-
+- ``score``: integer - Content is reported based upon a negative score, where -100 is "most offensive" and 0 is "inoffensive".
 - ``sender``: string - This is the ID of the user who sent the original message/event that was reported.
- ``room_alias``: string - The alias of the room. ``null`` if the room does not have a canonical alias set.
- ``event_json``: object - Details of the original event that was reported.
+- ``canonical_alias``: string - The canonical alias of the room. ``null`` if the room does not have a canonical alias set.
 - ``next_token``: integer - Indication for pagination. See above.
 - ``total``: integer - Total number of event reports related to the query (``user_id`` and ``room_id``).

+Show details of a specific event report
+=======================================
+
+This API returns information about a specific event report.
+
+The api is::
+
+    GET /_synapse/admin/v1/event_reports/<report_id>
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+It returns a JSON body like the following:
+
+.. code:: jsonc
+
+    {
+        "event_id": "$bNUFCwGzWca1meCGkjp-zwslF-GfVcXukvRLI1_FaVY",
+        "event_json": {
+            "auth_events": [
+                "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M",
+                "$oggsNXxzPFRE3y53SUNd7nsj69-QzKv03a1RucHu-ws"
+            ],
+            "content": {
+                "body": "matrix.org: This Week in Matrix",
+                "format": "org.matrix.custom.html",
+                "formatted_body": "<strong>matrix.org</strong>:<br><a href=\"https://matrix.org/blog/\"><strong>This Week in Matrix</strong></a>",
+                "msgtype": "m.notice"
+            },
+            "depth": 546,
+            "hashes": {
+                "sha256": "xK1//xnmvHJIOvbgXlkI8eEqdvoMmihVDJ9J4SNlsAw"
+            },
+            "origin": "matrix.org",
+            "origin_server_ts": 1592291711430,
+            "prev_events": [
+                "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M"
+            ],
+            "prev_state": [],
+            "room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org",
+            "sender": "@foobar:matrix.org",
+            "signatures": {
+                "matrix.org": {
+                    "ed25519:a_JaEG": "cs+OUKW/iHx5pEidbWxh0UiNNHwe46Ai9LwNz+Ah16aWDNszVIe2gaAcVZfvNsBhakQTew51tlKmL2kspXk/Dg"
+                }
+            },
+            "type": "m.room.message",
+            "unsigned": {
+                "age_ts": 1592291711430,
+            }
+        },
+        "id": <report_id>,
+        "reason": "foo",
+        "score": -100,
+        "received_ts": 1570897107409,
+        "canonical_alias": "#alias1:matrix.org",
+        "room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org",
+        "name": "Matrix HQ",
+        "sender": "@foobar:matrix.org",
+        "user_id": "@foo:matrix.org"
+    }
+
+**URL parameters:**
+
+- ``report_id``: string - The ID of the event report.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- ``id``: integer - ID of event report.
+- ``received_ts``: integer - The timestamp (in milliseconds since the unix epoch) when this report was sent.
+- ``room_id``: string - The ID of the room in which the event being reported is located.
+- ``name``: string - The name of the room.
+- ``event_id``: string - The ID of the reported event.
+- ``user_id``: string - This is the user who reported the event and wrote the reason.
+- ``reason``: string - Comment made by the ``user_id`` in this report. May be blank.
+- ``score``: integer - Content is reported based upon a negative score, where -100 is "most offensive" and 0 is "inoffensive".
+- ``sender``: string - This is the ID of the user who sent the original message/event that was reported.
+- ``canonical_alias``: string - The canonical alias of the room. ``null`` if the room does not have a canonical alias set.
+- ``event_json``: object - Details of the original event that was reported.
--- a/docs/admin_api/media_admin_api.md
+++ b/docs/admin_api/media_admin_api.md
@@ -100,3 +100,82 @@ Response:
  "num_quarantined": 10  # The number of media items successfully quarantined
 }
 ```
+
+# Delete local media
+This API deletes the *local* media from the disk of your own server.
+This includes any local thumbnails and copies of media downloaded from
+remote homeservers.
+This API will not affect media that has been uploaded to external
+media repositories (e.g https://github.com/turt2live/matrix-media-repo/).
+See also [purge_remote_media.rst](purge_remote_media.rst).
+
+## Delete a specific local media
+Delete a specific `media_id`.
+
+Request:
+
+```
+DELETE /_synapse/admin/v1/media/<server_name>/<media_id>
+
+{}
+```
+
+URL Parameters
+
+* `server_name`: string - The name of your local server (e.g `matrix.org`)
+* `media_id`: string - The ID of the media (e.g `abcdefghijklmnopqrstuvwx`)
+
+Response:
+
+```json
+    {
+       "deleted_media": [
+          "abcdefghijklmnopqrstuvwx"
+       ],
+       "total": 1
+    }
+```
+
+The following fields are returned in the JSON response body:
+
+* `deleted_media`: an array of strings - List of deleted `media_id`
+* `total`: integer - Total number of deleted `media_id`
+
+## Delete local media by date or size
+
+Request:
+
+```
+POST /_synapse/admin/v1/media/<server_name>/delete?before_ts=<before_ts>
+
+{}
+```
+
+URL Parameters
+
+* `server_name`: string - The name of your local server (e.g `matrix.org`).
+* `before_ts`: string representing a positive integer - Unix timestamp in ms.
+Files that were last used before this timestamp will be deleted. It is the timestamp of
+last access and not the timestamp creation. 
+* `size_gt`: Optional - string representing a positive integer - Size of the media in bytes.
+Files that are larger will be deleted. Defaults to `0`.
+* `keep_profiles`: Optional - string representing a boolean - Switch to also delete files
+that are still used in image data (e.g user profile, room avatar).
+If `false` these files will be deleted. Defaults to `true`.
+
+Response:
+
+```json
+    {
+       "deleted_media": [
+          "abcdefghijklmnopqrstuvwx",
+          "abcdefghijklmnopqrstuvwz"
+       ],
+       "total": 2
+    }
+```
+
+The following fields are returned in the JSON response body:
+
+* `deleted_media`: an array of strings - List of deleted `media_id`
+* `total`: integer - Total number of deleted `media_id`
--- a/docs/admin_api/user_admin_api.rst
+++ b/docs/admin_api/user_admin_api.rst
@@ -341,6 +341,89 @@ The following fields are returned in the JSON response body:
 - ``total`` - Number of rooms.


+List media of an user
+================================
+Gets a list of all local media that a specific ``user_id`` has created.
+The response is ordered by creation date descending and media ID descending.
+The newest media is on top.
+
+The API is::
+
+  GET /_synapse/admin/v1/users/<user_id>/media
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+A response body like the following is returned:
+
+.. code:: json
+
+    {
+      "media": [
+        {
+          "created_ts": 100400,
+          "last_access_ts": null,
+          "media_id": "qXhyRzulkwLsNHTbpHreuEgo",
+          "media_length": 67,
+          "media_type": "image/png",
+          "quarantined_by": null,
+          "safe_from_quarantine": false,
+          "upload_name": "test1.png"
+        },
+        {
+          "created_ts": 200400,
+          "last_access_ts": null,
+          "media_id": "FHfiSnzoINDatrXHQIXBtahw",
+          "media_length": 67,
+          "media_type": "image/png",
+          "quarantined_by": null,
+          "safe_from_quarantine": false,
+          "upload_name": "test2.png"
+        }
+      ],
+      "next_token": 3,
+      "total": 2
+    }
+
+To paginate, check for ``next_token`` and if present, call the endpoint again
+with ``from`` set to the value of ``next_token``. This will return a new page.
+
+If the endpoint does not return a ``next_token`` then there are no more
+reports to paginate through.
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - string - fully qualified: for example, ``@user:server.com``.
+- ``limit``: string representing a positive integer - Is optional but is used for pagination,
+  denoting the maximum number of items to return in this call. Defaults to ``100``.
+- ``from``: string representing a positive integer - Is optional but used for pagination,
+  denoting the offset in the returned results. This should be treated as an opaque value and
+  not explicitly set to anything other than the return value of ``next_token`` from a previous call.
+  Defaults to ``0``.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- ``media`` - An array of objects, each containing information about a media.
+  Media objects contain the following fields:
+
+  - ``created_ts`` - integer - Timestamp when the content was uploaded in ms.
+  - ``last_access_ts`` - integer - Timestamp when the content was last accessed in ms.
+  - ``media_id`` - string - The id used to refer to the media.
+  - ``media_length`` - integer - Length of the media in bytes.
+  - ``media_type`` - string - The MIME-type of the media.
+  - ``quarantined_by`` - string - The user ID that initiated the quarantine request
+    for this media.
+
+  - ``safe_from_quarantine`` - bool - Status if this media is safe from quarantining.
+  - ``upload_name`` - string - The name the media was uploaded with.
+
+- ``next_token``: integer - Indication for pagination. See above.
+- ``total`` - integer - Total number of media.
+
 User devices
 ============

@@ -375,7 +458,8 @@ A response body like the following is returned:
          "last_seen_ts": 1474491775025,
          "user_id": "<user_id>"
        }
-      ]
+      ],
+      "total": 2
    }

 **Parameters**
@@ -400,6 +484,8 @@ The following fields are returned in the JSON response body:
    devices was last seen. (May be a few minutes out of date, for efficiency reasons).
  - ``user_id`` - Owner of  device.

+- ``total`` - Total number of user's devices.
+
 Delete multiple devices
 ------------------
 Deletes the given devices for a specific ``user_id``, and invalidates
@@ -525,3 +611,82 @@ The following parameters should be set in the URL:

 - ``user_id`` - fully qualified: for example, ``@user:server.com``.
 - ``device_id`` - The device to delete.
+
+List all pushers
+================
+Gets information about all pushers for a specific ``user_id``.
+
+The API is::
+
+  GET /_synapse/admin/v1/users/<user_id>/pushers
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+A response body like the following is returned:
+
+.. code:: json
+
+    {
+      "pushers": [
+        {
+          "app_display_name":"HTTP Push Notifications",
+          "app_id":"m.http",
+          "data": {
+            "url":"example.com"
+          },
+          "device_display_name":"pushy push",
+          "kind":"http",
+          "lang":"None",
+          "profile_tag":"",
+          "pushkey":"a@example.com"
+        }
+      ],
+      "total": 1
+    }
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - fully qualified: for example, ``@user:server.com``.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- ``pushers`` - An array containing the current pushers for the user
+
+  - ``app_display_name`` - string - A string that will allow the user to identify
+    what application owns this pusher.
+
+  - ``app_id`` - string - This is a reverse-DNS style identifier for the application.
+    Max length, 64 chars.
+
+  - ``data`` - A dictionary of information for the pusher implementation itself.
+
+    - ``url`` - string - Required if ``kind`` is ``http``. The URL to use to send
+      notifications to.
+
+    - ``format`` - string - The format to use when sending notifications to the
+      Push Gateway.
+
+  - ``device_display_name`` - string -  A string that will allow the user to identify
+    what device owns this pusher.
+
+  - ``profile_tag`` - string - This string determines which set of device specific rules
+    this pusher executes.
+
+  - ``kind`` - string -  The kind of pusher. "http" is a pusher that sends HTTP pokes.
+  - ``lang`` - string - The preferred language for receiving notifications
+    (e.g. 'en' or 'en-US')
+
+  - ``profile_tag`` - string - This string determines which set of device specific rules
+    this pusher executes.
+
+  - ``pushkey`` - string - This is a unique identifier for this pusher.
+    Max length, 512 bytes.
+
+- ``total`` - integer - Number of pushers.
+
+See also `Client-Server API Spec <https://matrix.org/docs/spec/client_server/latest#get-matrix-client-r0-pushers>`_
--- a/docs/manhole.md
+++ b/docs/manhole.md
@@ -5,8 +5,45 @@ The "manhole" allows server administrators to access a Python shell on a running
 Synapse installation. This is a very powerful mechanism for administration and
 debugging.

+**_Security Warning_**
+
+Note that this will give administrative access to synapse to **all users** with
+shell access to the server. It should therefore **not** be enabled in
+environments where untrusted users have shell access.
+
+***
+
 To enable it, first uncomment the `manhole` listener configuration in
-`homeserver.yaml`:
+`homeserver.yaml`. The configuration is slightly different if you're using docker.
+
+#### Docker config
+
+If you are using Docker, set `bind_addresses` to `['0.0.0.0']` as shown:
+
+```yaml
+listeners:
+  - port: 9000
+    bind_addresses: ['0.0.0.0']
+    type: manhole
+```
+
+When using `docker run` to start the server, you will then need to change the command to the following to include the
+`manhole` port forwarding. The `-p 127.0.0.1:9000:9000` below is important: it 
+ensures that access to the `manhole` is only possible for local users.
+
+```bash
+docker run -d --name synapse \
+    --mount type=volume,src=synapse-data,dst=/data \
+    -p 8008:8008 \
+    -p 127.0.0.1:9000:9000 \
+    matrixdotorg/synapse:latest
+```
+
+#### Native config
+
+If you are not using docker, set `bind_addresses` to `['::1', '127.0.0.1']` as shown.
+The `bind_addresses` in the example below is important: it ensures that access to the
+`manhole` is only possible for local users).

 ```yaml
 listeners:
@@ -15,12 +52,7 @@ listeners:
    type: manhole
 ```

-(`bind_addresses` in the above is important: it ensures that access to the
-manhole is only possible for local users).
-
-Note that this will give administrative access to synapse to **all users** with
-shell access to the server. It should therefore **not** be enabled in
-environments where untrusted users have shell access.
+#### Accessing synapse manhole

 Then restart synapse, and point an ssh client at port 9000 on localhost, using
 the username `matrix`:
--- a/docs/message_retention_policies.md
+++ b/docs/message_retention_policies.md
@@ -136,24 +136,34 @@ the server's database.

 ### Lifetime limits

-**Note: this feature is mainly useful within a closed federation or on
-servers that don't federate, because there currently is no way to
-enforce these limits in an open federation.**
-
-Server admins can restrict the values their local users are allowed to
-use for both `min_lifetime` and `max_lifetime`. These limits can be
-defined as such in the `retention` section of the configuration file:
+Server admins can set limits on the values of `max_lifetime` to use when
+purging old events in a room. These limits can be defined as such in the
+`retention` section of the configuration file:

 ```yaml
  allowed_lifetime_min: 1d
  allowed_lifetime_max: 1y
 ```

-Here, `allowed_lifetime_min` is the lowest value a local user can set
-for both `min_lifetime` and `max_lifetime`, and `allowed_lifetime_max`
-is the highest value. Both parameters are optional (e.g. setting
-`allowed_lifetime_min` but not `allowed_lifetime_max` only enforces a
-minimum and no maximum).
+The limits are considered when running purge jobs. If necessary, the
+effective value of `max_lifetime` will be brought between
+`allowed_lifetime_min` and `allowed_lifetime_max` (inclusive).
+This means that, if the value of `max_lifetime` defined in the room's state
+is lower than `allowed_lifetime_min`, the value of `allowed_lifetime_min`
+will be used instead. Likewise, if the value of `max_lifetime` is higher
+than `allowed_lifetime_max`, the value of `allowed_lifetime_max` will be
+used instead.
+
+In the example above, we ensure Synapse never deletes events that are less
+than one day old, and that it always deletes events that are over a year
+old.
+
+If a default policy is set, and its `max_lifetime` value is lower than
+`allowed_lifetime_min` or higher than `allowed_lifetime_max`, the same
+process applies.
+
+Both parameters are optional; if one is omitted Synapse won't use it to
+adjust the effective value of `max_lifetime`.

 Like other settings in this section, these parameters can be expressed
 either as a duration or as a number of milliseconds.
--- a/docs/metrics-howto.md
+++ b/docs/metrics-howto.md
@@ -60,6 +60,8 @@

 1.  Restart Prometheus.

+1.  Consider using the [grafana dashboard](https://github.com/matrix-org/synapse/tree/master/contrib/grafana/) and required [recording rules](https://github.com/matrix-org/synapse/tree/master/contrib/prometheus/) 
+
 ## Monitoring workers

 To monitor a Synapse installation using
--- a/docs/openid.md
+++ b/docs/openid.md
@@ -37,7 +37,7 @@ as follows:
   provided by `matrix.org` so no further action is needed.

 * If you installed Synapse into a virtualenv, run `/path/to/env/bin/pip
-   install synapse[oidc]` to install the necessary dependencies.
+   install matrix-synapse[oidc]` to install the necessary dependencies.

 * For other installation mechanisms, see the documentation provided by the
   maintainer.
@@ -52,14 +52,39 @@ specific providers.

 Here are a few configs for providers that should work with Synapse.

+### Microsoft Azure Active Directory
+Azure AD can act as an OpenID Connect Provider. Register a new application under 
+*App registrations* in the Azure AD management console. The RedirectURI for your
+application should point to your matrix server: `[synapse public baseurl]/_synapse/oidc/callback`
+
+Go to *Certificates & secrets* and register a new client secret. Make note of your 
+Directory (tenant) ID as it will be used in the Azure links.
+Edit your Synapse config file and change the `oidc_config` section:
+
+```yaml
+oidc_config:
+   enabled: true
+   issuer: "https://login.microsoftonline.com/<tenant id>/v2.0"
+   client_id: "<client id>"
+   client_secret: "<client secret>"
+   scopes: ["openid", "profile"]
+   authorization_endpoint: "https://login.microsoftonline.com/<tenant id>/oauth2/v2.0/authorize"
+   token_endpoint: "https://login.microsoftonline.com/<tenant id>/oauth2/v2.0/token"
+   userinfo_endpoint: "https://graph.microsoft.com/oidc/userinfo"
+
+   user_mapping_provider:
+     config:
+       localpart_template: "{{ user.preferred_username.split('@')[0] }}"
+       display_name_template: "{{ user.name }}"
+```
+
 ### [Dex][dex-idp]

 [Dex][dex-idp] is a simple, open-source, certified OpenID Connect Provider.
 Although it is designed to help building a full-blown provider with an
 external database, it can be configured with static passwords in a config file.

-Follow the [Getting Started
-guide](https://github.com/dexidp/dex/blob/master/Documentation/getting-started.md)
+Follow the [Getting Started guide](https://dexidp.io/docs/getting-started/)
 to install Dex.

 Edit `examples/config-dev.yaml` config file from the Dex repo to add a client:
@@ -73,7 +98,7 @@ staticClients:
  name: 'Synapse'
 ```

-Run with `dex serve examples/config-dex.yaml`.
+Run with `dex serve examples/config-dev.yaml`.

 Synapse config:

--- a/docs/reverse_proxy.md
+++ b/docs/reverse_proxy.md
@@ -54,7 +54,7 @@ server {
        proxy_set_header X-Forwarded-For $remote_addr;
        # Nginx by default only allows file uploads up to 1M in size
        # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
-        client_max_body_size 10M;
+        client_max_body_size 50M;
    }
 }
 ```
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -893,7 +893,7 @@ media_store_path: "DATADIR/media_store"

 # The largest allowed upload size in bytes
 #
-#max_upload_size: 10M
+#max_upload_size: 50M

 # Maximum number of pixels that will be thumbnailed
 #
@@ -1505,10 +1505,8 @@ trusted_key_servers:

 ## Single sign-on integration ##

-# Enable SAML2 for registration and login. Uses pysaml2.
-#
-# At least one of `sp_config` or `config_path` must be set in this section to
-# enable SAML login.
+# The following settings can be used to make Synapse use a single sign-on
+# provider for authentication, instead of its internal password database.
 #
 # You will probably also want to set the following options to `false` to
 # disable the regular login/registration flows:
@@ -1517,6 +1515,11 @@ trusted_key_servers:
 #
 # You will also want to investigate the settings under the "sso" configuration
 # section below.
+
+# Enable SAML2 for registration and login. Uses pysaml2.
+#
+# At least one of `sp_config` or `config_path` must be set in this section to
+# enable SAML login.
 #
 # Once SAML support is enabled, a metadata file will be exposed at
 # https://<server>:<port>/_matrix/saml2/metadata.xml, which you may be able to
@@ -1532,40 +1535,42 @@ saml2_config:
  # so it is not normally necessary to specify them unless you need to
  # override them.
  #
-  #sp_config:
-  #  # point this to the IdP's metadata. You can use either a local file or
-  #  # (preferably) a URL.
-  #  metadata:
-  #    #local: ["saml2/idp.xml"]
-  #    remote:
-  #      - url: https://our_idp/metadata.xml
-  #
-  #  # By default, the user has to go to our login page first. If you'd like
-  #  # to allow IdP-initiated login, set 'allow_unsolicited: true' in a
-  #  # 'service.sp' section:
-  #  #
-  #  #service:
-  #  #  sp:
-  #  #    allow_unsolicited: true
-  #
-  #  # The examples below are just used to generate our metadata xml, and you
-  #  # may well not need them, depending on your setup. Alternatively you
-  #  # may need a whole lot more detail - see the pysaml2 docs!
-  #
-  #  description: ["My awesome SP", "en"]
-  #  name: ["Test SP", "en"]
-  #
-  #  organization:
-  #    name: Example com
-  #    display_name:
-  #      - ["Example co", "en"]
-  #    url: "http://example.com"
-  #
-  #  contact_person:
-  #    - given_name: Bob
-  #      sur_name: "the Sysadmin"
-  #      email_address": ["admin@example.com"]
-  #      contact_type": technical
+  sp_config:
+    # Point this to the IdP's metadata. You must provide either a local
+    # file via the `local` attribute or (preferably) a URL via the
+    # `remote` attribute.
+    #
+    #metadata:
+    #  local: ["saml2/idp.xml"]
+    #  remote:
+    #    - url: https://our_idp/metadata.xml
+
+    # By default, the user has to go to our login page first. If you'd like
+    # to allow IdP-initiated login, set 'allow_unsolicited: true' in a
+    # 'service.sp' section:
+    #
+    #service:
+    #  sp:
+    #    allow_unsolicited: true
+
+    # The examples below are just used to generate our metadata xml, and you
+    # may well not need them, depending on your setup. Alternatively you
+    # may need a whole lot more detail - see the pysaml2 docs!
+
+    #description: ["My awesome SP", "en"]
+    #name: ["Test SP", "en"]
+
+    #organization:
+    #  name: Example com
+    #  display_name:
+    #    - ["Example co", "en"]
+    #  url: "http://example.com"
+
+    #contact_person:
+    #  - given_name: Bob
+    #    sur_name: "the Sysadmin"
+    #    email_address": ["admin@example.com"]
+    #    contact_type": technical

  # Instead of putting the config inline as above, you can specify a
  # separate pysaml2 configuration file:
@@ -1641,11 +1646,10 @@ saml2_config:
  #    value: "sales"


-# OpenID Connect integration. The following settings can be used to make Synapse
-# use an OpenID Connect Provider for authentication, instead of its internal
-# password database.
+# Enable OpenID Connect (OIDC) / OAuth 2.0 for registration and login.
 #
-# See https://github.com/matrix-org/synapse/blob/master/docs/openid.md.
+# See https://github.com/matrix-org/synapse/blob/master/docs/openid.md
+# for some example configurations.
 #
 oidc_config:
  # Uncomment the following to enable authorization against an OpenID Connect
@@ -1778,15 +1782,37 @@ oidc_config:



-# Enable CAS for registration and login.
+# Enable Central Authentication Service (CAS) for registration and login.
 #
-#cas_config:
-#   enabled: true
-#   server_url: "https://cas-server.com"
-#   service_url: "https://homeserver.domain.com:8448"
-#   #displayname_attribute: name
-#   #required_attributes:
-#   #    name: value
+cas_config:
+  # Uncomment the following to enable authorization against a CAS server.
+  # Defaults to false.
+  #
+  #enabled: true
+
+  # The URL of the CAS authorization endpoint.
+  #
+  #server_url: "https://cas-server.com"
+
+  # The public URL of the homeserver.
+  #
+  #service_url: "https://homeserver.domain.com:8448"
+
+  # The attribute of the CAS response to use as the display name.
+  #
+  # If unset, no displayname will be set.
+  #
+  #displayname_attribute: name
+
+  # It is possible to configure Synapse to only allow logins if CAS attributes
+  # match particular values. All of the keys in the mapping below must exist
+  # and the values must match the given value. Alternately if the given value
+  # is None then any value is allowed (the attribute just must exist).
+  # All of the listed attributes must match for the login to be permitted.
+  #
+  #required_attributes:
+  #  userGroup: "staff"
+  #  department: None


 # Additional settings to use with single-sign on systems such as OpenID Connect,
@@ -1886,7 +1912,7 @@ sso:
 # and issued at ("iat") claims are validated if present.
 #
 # Note that this is a non-standard login type and client support is
-# expected to be non-existant.
+# expected to be non-existent.
 #
 # See https://github.com/matrix-org/synapse/blob/master/docs/jwt.md.
 #
@@ -2402,7 +2428,7 @@ spam_checker:
 #
 # Options for the rules include:
 #
-#   user_id: Matches agaisnt the creator of the alias
+#   user_id: Matches against the creator of the alias
 #   room_id: Matches against the room ID being published
 #   alias: Matches against any current local or canonical aliases
 #            associated with the room
@@ -2448,7 +2474,7 @@ opentracing:
    # This is a list of regexes which are matched against the server_name of the
    # homeserver.
    #
-    # By defult, it is empty, so no servers are matched.
+    # By default, it is empty, so no servers are matched.
    #
    #homeserver_whitelist:
    #  - ".*"
--- a/docs/sample_log_config.yaml
+++ b/docs/sample_log_config.yaml
@@ -3,7 +3,11 @@
 # This is a YAML file containing a standard Python logging configuration
 # dictionary. See [1] for details on the valid settings.
 #
+# Synapse also supports structured logging for machine readable logs which can
+# be ingested by ELK stacks. See [2] for details.
+#
 # [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
+# [2]: https://github.com/matrix-org/synapse/blob/master/docs/structured_logging.md

 version: 1

@@ -59,7 +63,7 @@ root:
    # then write them to a file.
    #
    # Replace "buffer" with "console" to log to stderr instead. (Note that you'll
-    # also need to update the configuation for the `twisted` logger above, in
+    # also need to update the configuration for the `twisted` logger above, in
    # this case.)
    #
    handlers: [buffer]
--- a/docs/structured_logging.md
+++ b/docs/structured_logging.md
@@ -1,11 +1,116 @@
 # Structured Logging

-A structured logging system can be useful when your logs are destined for a machine to parse and process. By maintaining its machine-readable characteristics, it enables more efficient searching and aggregations when consumed by software such as the "ELK stack".
+A structured logging system can be useful when your logs are destined for a
+machine to parse and process. By maintaining its machine-readable characteristics,
+it enables more efficient searching and aggregations when consumed by software
+such as the "ELK stack".

-Synapse's structured logging system is configured via the file that Synapse's `log_config` config option points to. The file must be YAML and contain `structured: true`. It must contain a list of "drains" (places where logs go to).
+Synapse's structured logging system is configured via the file that Synapse's
+`log_config` config option points to. The file should include a formatter which
+uses the `synapse.logging.TerseJsonFormatter` class included with Synapse and a
+handler which uses the above formatter.
+
+There is also a `synapse.logging.JsonFormatter` option which does not include
+a timestamp in the resulting JSON. This is useful if the log ingester adds its
+own timestamp.

 A structured logging configuration looks similar to the following:

+```yaml
+version: 1
+
+formatters:
+    structured:
+        class: synapse.logging.TerseJsonFormatter
+
+handlers:
+    file:
+        class: logging.handlers.TimedRotatingFileHandler
+        formatter: structured
+        filename: /path/to/my/logs/homeserver.log
+        when: midnight
+        backupCount: 3  # Does not include the current log file.
+        encoding: utf8
+
+loggers:
+    synapse:
+        level: INFO
+        handlers: [remote]
+    synapse.storage.SQL:
+        level: WARNING
+```
+
+The above logging config will set Synapse as 'INFO' logging level by default,
+with the SQL layer at 'WARNING', and will log to a file, stored as JSON.
+
+It is also possible to figure Synapse to log to a remote endpoint by using the
+`synapse.logging.RemoteHandler` class included with Synapse. It takes the
+following arguments:
+
+- `host`: Hostname or IP address of the log aggregator.
+- `port`: Numerical port to contact on the host.
+- `maximum_buffer`: (Optional, defaults to 1000) The maximum buffer size to allow.
+
+A remote structured logging configuration looks similar to the following:
+
+```yaml
+version: 1
+
+formatters:
+    structured:
+        class: synapse.logging.TerseJsonFormatter
+
+handlers:
+    remote:
+        class: synapse.logging.RemoteHandler
+        formatter: structured
+        host: 10.1.2.3
+        port: 9999
+
+loggers:
+    synapse:
+        level: INFO
+        handlers: [remote]
+    synapse.storage.SQL:
+        level: WARNING
+```
+
+The above logging config will set Synapse as 'INFO' logging level by default,
+with the SQL layer at 'WARNING', and will log JSON formatted messages to a
+remote endpoint at 10.1.2.3:9999.
+
+## Upgrading from legacy structured logging configuration
+
+Versions of Synapse prior to v1.23.0 included a custom structured logging
+configuration which is deprecated. It used a `structured: true` flag and
+configured `drains` instead of ``handlers`` and `formatters`.
+
+Synapse currently automatically converts the old configuration to the new
+configuration, but this will be removed in a future version of Synapse. The
+following reference can be used to update your configuration. Based on the drain
+`type`, we can pick a new handler:
+
+1. For a type of `console`, `console_json`, or `console_json_terse`: a handler
+   with a class of `logging.StreamHandler` and a `stream` of `ext://sys.stdout`
+   or `ext://sys.stderr` should be used.
+2. For a type of `file` or `file_json`: a handler of `logging.FileHandler` with
+   a location of the file path should be used.
+3. For a type of `network_json_terse`: a handler of `synapse.logging.RemoteHandler`
+   with the host and port should be used.
+
+Then based on the drain `type` we can pick a new formatter:
+
+1. For a type of `console` or `file` no formatter is necessary.
+2. For a type of `console_json` or `file_json`: a formatter of
+   `synapse.logging.JsonFormatter` should be used.
+3. For a type of `console_json_terse` or `network_json_terse`: a formatter of
+   `synapse.logging.TerseJsonFormatter` should be used.
+
+For each new handler and formatter they should be added to the logging configuration
+and then assigned to either a logger or the root logger.
+
+An example legacy configuration:
+
 ```yaml
 structured: true

@@ -24,60 +129,33 @@ drains:
        location: homeserver.log
 ```

-The above logging config will set Synapse as 'INFO' logging level by default, with the SQL layer at 'WARNING', and will have two logging drains (to the console and to a file, stored as JSON).
+Would be converted into a new configuration:

-## Drain Types
+```yaml
+version: 1

-Drain types can be specified by the `type` key.
+formatters:
+    json:
+        class: synapse.logging.JsonFormatter

-### `console`
+handlers:
+    console:
+        class: logging.StreamHandler
+        location: ext://sys.stdout
+    file:
+        class: logging.FileHandler
+        formatter: json
+        filename: homeserver.log

-Outputs human-readable logs to the console.
+loggers:
+    synapse:
+        level: INFO
+        handlers: [console, file]
+    synapse.storage.SQL:
+        level: WARNING
+```

-Arguments:
-
- `location`: Either `stdout` or `stderr`.
-
-### `console_json`
-
-Outputs machine-readable JSON logs to the console.
-
-Arguments:
-
- `location`: Either `stdout` or `stderr`.
-
-### `console_json_terse`
-
-Outputs machine-readable JSON logs to the console, separated by newlines. This
-format is not designed to be read and re-formatted into human-readable text, but
-is optimal for a logging aggregation system.
-
-Arguments:
-
- `location`: Either `stdout` or `stderr`.
-
-### `file`
-
-Outputs human-readable logs to a file.
-
-Arguments:
-
- `location`: An absolute path to the file to log to.
-
-### `file_json`
-
-Outputs machine-readable logs to a file.
-
-Arguments:
-
- `location`: An absolute path to the file to log to.
-
-### `network_json_terse`
-
-Delivers machine-readable JSON logs to a log aggregator over TCP. This is
-compatible with LogStash's TCP input with the codec set to `json_lines`.
-
-Arguments:
-
- `host`: Hostname or IP address of the log aggregator.
- `port`: Numerical port to contact on the host.
+The new logging configuration is a bit more verbose, but significantly more
+flexible. It allows for configuration that were not previously possible, such as
+sending plain logs over the network, or using different handlers for different
+modules.
--- a/docs/tcp_replication.md
+++ b/docs/tcp_replication.md
@@ -15,7 +15,7 @@ example flow would be (where '>' indicates master to worker and

    > SERVER example.com
    < REPLICATE
-    > POSITION events master 53
+    > POSITION events master 53 53
    > RDATA events master 54 ["$foo1:bar.com", ...]
    > RDATA events master 55 ["$foo4:bar.com", ...]

@@ -138,9 +138,9 @@ the wire:
    < NAME synapse.app.appservice
    < PING 1490197665618
    < REPLICATE
-    > POSITION events master 1
-    > POSITION backfill master 1
-    > POSITION caches master 1
+    > POSITION events master 1 1
+    > POSITION backfill master 1 1
+    > POSITION caches master 1 1
    > RDATA caches master 2 ["get_user_by_id",["@01register-user:localhost:8823"],1490197670513]
    > RDATA events master 14 ["$149019767112vOHxz:localhost:8823",
        "!AFDCvgApUmpdfVjIXm:localhost:8823","m.room.guest_access","",null]
@@ -185,6 +185,11 @@ client (C):
   updates via HTTP API, rather than via the DB, then processes should make the
   request to the appropriate process.

+   Two positions are included, the "new" position and the last position sent respectively.
+   This allows servers to tell instances that the positions have advanced but no
+   data has been written, without clients needlessly checking to see if they
+   have missed any updates.
+
 #### ERROR (S, C)

   There was an error
--- a/mypy.ini
+++ b/mypy.ini
@@ -15,9 +15,15 @@ files =
  synapse/events/builder.py,
  synapse/events/spamcheck.py,
  synapse/federation,
+  synapse/handlers/_base.py,
+  synapse/handlers/account_data.py,
+  synapse/handlers/account_validity.py,
+  synapse/handlers/appservice.py,
  synapse/handlers/auth.py,
  synapse/handlers/cas_handler.py,
+  synapse/handlers/deactivate_account.py,
  synapse/handlers/device.py,
+  synapse/handlers/devicemessage.py,
  synapse/handlers/directory.py,
  synapse/handlers/events.py,
  synapse/handlers/federation.py,
@@ -26,7 +32,10 @@ files =
  synapse/handlers/message.py,
  synapse/handlers/oidc_handler.py,
  synapse/handlers/pagination.py,
+  synapse/handlers/password_policy.py,
  synapse/handlers/presence.py,
+  synapse/handlers/profile.py,
+  synapse/handlers/read_marker.py,
  synapse/handlers/room.py,
  synapse/handlers/room_member.py,
  synapse/handlers/room_member_worker.py,
@@ -48,7 +57,9 @@ files =
  synapse/server_notices,
  synapse/spam_checker_api,
  synapse/state,
+  synapse/storage/databases/main/appservice.py,
  synapse/storage/databases/main/events.py,
+  synapse/storage/databases/main/registration.py,
  synapse/storage/databases/main/stream.py,
  synapse/storage/databases/main/ui_auth.py,
  synapse/storage/database.py,
@@ -59,8 +70,7 @@ files =
  synapse/streams,
  synapse/types.py,
  synapse/util/async_helpers.py,
-  synapse/util/caches/descriptors.py,
-  synapse/util/caches/stream_change_cache.py,
+  synapse/util/caches,
  synapse/util/metrics.py,
  tests/replication,
  tests/test_utils,
@@ -73,6 +83,9 @@ ignore_missing_imports = True
 [mypy-zope]
 ignore_missing_imports = True

+[mypy-bcrypt]
+ignore_missing_imports = True
+
 [mypy-constantly]
 ignore_missing_imports = True

--- a/pyproject.toml
+++ b/pyproject.toml
@@ -35,7 +35,7 @@
        showcontent = true

 [tool.black]
-target-version = ['py34']
+target-version = ['py35']
 exclude = '''

 (
--- a/Show More
+++ b/Show More
				`@@ -1 +0,0 @@`
				`Add a configuration option for always using the "userinfo endpoint" for OpenID Connect. This fixes support for some identity providers, e.g. GitLab. Contributed by Benjamin Koch.`
				`@@ -1 +0,0 @@`
				Allow `ThirdPartyEventRules` modules to query and manipulate whether a room is in the public rooms directory.
				`@@ -1 +0,0 @@`
				`Add support for olm fallback keys ([MSC2732](https://github.com/matrix-org/matrix-doc/pull/2732)).`
				`@@ -1 +0,0 @@`
				`Allow running background tasks in a separate worker process.`
				`@@ -1 +0,0 @@`
				`Add support for device dehydration ([MSC2697](https://github.com/matrix-org/matrix-doc/pull/2697)).`
				`@@ -1 +0,0 @@`
				`Add typing information to the device handler.`
				`@@ -1 +0,0 @@`
				`Add unit test for event persister sharding.`
				`@@ -1 +0,0 @@`
				`Allow events to be sent to clients sooner when using sharded event persisters.`
				`@@ -1 +0,0 @@`
				Configure `public_baseurl` when using demo scripts.
				`@@ -1 +0,0 @@`
				`Add SQL logging on queries that happen during startup.`