wip access rules modification

.circleci/config.yml

@@ -1,36 +1,22 @@
-version: 2.1
+version: 2
 jobs:
   dockerhubuploadrelease:
-    docker:
-      - image: docker:git
+    machine: true
     steps:
       - checkout
-      - setup_remote_docker
-      - docker_prepare
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      - docker_build:
-          tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
-          platforms: linux/amd64
-      - docker_build:
-          tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
-          platforms: linux/amd64,linux/arm/v7,linux/arm64
-
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
   dockerhubuploadlatest:
-    docker:
-      - image: docker:git
+    machine: true
     steps:
       - checkout
-      - setup_remote_docker
-      - docker_prepare
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      - docker_build:
-          tag: -t matrixdotorg/synapse:latest
-          platforms: linux/amd64
-      - docker_build:
-          tag: -t matrixdotorg/synapse:latest
-          platforms: linux/amd64,linux/arm/v7,linux/arm64
+      - run: docker push matrixdotorg/synapse:latest
 
 workflows:
+  version: 2
   build:
     jobs:
       - dockerhubuploadrelease:
@@ -43,33 +29,3 @@ workflows:
           filters:
             branches:
               only: master
-
-commands:
-  docker_prepare:
-    description: Downloads the buildx cli plugin and enables multiarch images
-    parameters:
-      buildx_version:
-        type: string
-        default: "v0.4.1"
-    steps:
-      - run: apk add --no-cache curl
-      - run: mkdir -vp ~/.docker/cli-plugins/ ~/dockercache
-      - run: curl --silent -L "https://github.com/docker/buildx/releases/download/<< parameters.buildx_version >>/buildx-<< parameters.buildx_version >>.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
-      - run: chmod a+x ~/.docker/cli-plugins/docker-buildx
-      # install qemu links in /proc/sys/fs/binfmt_misc on the docker instance running the circleci job
-      - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-      # create a context named `builder` for the builds
-      - run: docker context create builder
-      # create a buildx builder using the new context, and set it as the default
-      - run: docker buildx create builder --use
-
-  docker_build:
-    description: Builds and pushed images to dockerhub using buildx
-    parameters:
-      platforms:
-        type: string
-        default: linux/amd64
-      tag:
-        type: string
-    steps:
-      - run: docker buildx build -f docker/Dockerfile --push --platform << parameters.platforms >> --label gitsha1=${CIRCLE_SHA1} << parameters.tag >> --progress=plain .

.gitignore

@@ -21,7 +21,6 @@ _trial_temp*/
 /.python-version
 /*.signing.key
 /env/
-/.venv*/
 /homeserver*.yaml
 /logs
 /media_store/

CHANGES.md

@@ -1,171 +1,3 @@
-Synapse 1.22.1 (2020-10-30)
-===========================
-
-Bugfixes
---------
-
-- Fix a bug where an appservice may not be forwarded events for a room it was recently invited to. Broke in v1.22.0. ([\#8676](https://github.com/matrix-org/synapse/issues/8676))
-- Fix `Object of type frozendict is not JSON serializable` exceptions when using third-party event rules. Broke in v1.22.0. ([\#8678](https://github.com/matrix-org/synapse/issues/8678))
-
-
-Synapse 1.22.0 (2020-10-27)
-===========================
-
-No significant changes.
-
-
-Synapse 1.22.0rc2 (2020-10-26)
-==============================
-
-Bugfixes
---------
-
-- Fix bugs where ephemeral events were not sent to appservices. Broke in v1.22.0rc1. ([\#8648](https://github.com/matrix-org/synapse/issues/8648), [\#8656](https://github.com/matrix-org/synapse/issues/8656))
-- Fix `user_daily_visits` table to not have duplicate rows per user/device due to multiple user agents. Broke in v1.22.0rc1. ([\#8654](https://github.com/matrix-org/synapse/issues/8654))
-
-Synapse 1.22.0rc1 (2020-10-22)
-==============================
-
-Features
---------
-
-- Add a configuration option for always using the "userinfo endpoint" for OpenID Connect. This fixes support for some identity providers, e.g. GitLab. Contributed by Benjamin Koch. ([\#7658](https://github.com/matrix-org/synapse/issues/7658))
-- Add ability for `ThirdPartyEventRules` modules to query and manipulate whether a room is in the public rooms directory. ([\#8292](https://github.com/matrix-org/synapse/issues/8292), [\#8467](https://github.com/matrix-org/synapse/issues/8467))
-- Add support for olm fallback keys ([MSC2732](https://github.com/matrix-org/matrix-doc/pull/2732)). ([\#8312](https://github.com/matrix-org/synapse/issues/8312), [\#8501](https://github.com/matrix-org/synapse/issues/8501))
-- Add support for running background tasks in a separate worker process. ([\#8369](https://github.com/matrix-org/synapse/issues/8369), [\#8458](https://github.com/matrix-org/synapse/issues/8458), [\#8489](https://github.com/matrix-org/synapse/issues/8489), [\#8513](https://github.com/matrix-org/synapse/issues/8513), [\#8544](https://github.com/matrix-org/synapse/issues/8544), [\#8599](https://github.com/matrix-org/synapse/issues/8599))
-- Add support for device dehydration ([MSC2697](https://github.com/matrix-org/matrix-doc/pull/2697)). ([\#8380](https://github.com/matrix-org/synapse/issues/8380))
-- Add support for [MSC2409](https://github.com/matrix-org/matrix-doc/pull/2409), which allows sending typing, read receipts, and presence events to appservices. ([\#8437](https://github.com/matrix-org/synapse/issues/8437), [\#8590](https://github.com/matrix-org/synapse/issues/8590))
-- Change default room version to "6", per [MSC2788](https://github.com/matrix-org/matrix-doc/pull/2788). ([\#8461](https://github.com/matrix-org/synapse/issues/8461))
-- Add the ability to send non-membership events into a room via the `ModuleApi`. ([\#8479](https://github.com/matrix-org/synapse/issues/8479))
-- Increase default upload size limit from 10M to 50M. Contributed by @Akkowicz. ([\#8502](https://github.com/matrix-org/synapse/issues/8502))
-- Add support for modifying event content in `ThirdPartyRules` modules. ([\#8535](https://github.com/matrix-org/synapse/issues/8535), [\#8564](https://github.com/matrix-org/synapse/issues/8564))
-
-
-Bugfixes
---------
-
-- Fix a longstanding bug where invalid ignored users in account data could break clients. ([\#8454](https://github.com/matrix-org/synapse/issues/8454))
-- Fix a bug where backfilling a room with an event that was missing the `redacts` field would break. ([\#8457](https://github.com/matrix-org/synapse/issues/8457))
-- Don't attempt to respond to some requests if the client has already disconnected. ([\#8465](https://github.com/matrix-org/synapse/issues/8465))
-- Fix message duplication if something goes wrong after persisting the event. ([\#8476](https://github.com/matrix-org/synapse/issues/8476))
-- Fix incremental sync returning an incorrect `prev_batch` token in timeline section, which when used to paginate returned events that were included in the incremental sync. Broken since v0.16.0. ([\#8486](https://github.com/matrix-org/synapse/issues/8486))
-- Expose the `uk.half-shot.msc2778.login.application_service` to clients from the login API. This feature was added in v1.21.0, but was not exposed as a potential login flow. ([\#8504](https://github.com/matrix-org/synapse/issues/8504))
-- Fix error code for `/profile/{userId}/displayname` to be `M_BAD_JSON`. ([\#8517](https://github.com/matrix-org/synapse/issues/8517))
-- Fix a bug introduced in v1.7.0 that could cause Synapse to insert values from non-state `m.room.retention` events into the `room_retention` database table. ([\#8527](https://github.com/matrix-org/synapse/issues/8527))
-- Fix not sending events over federation when using sharded event writers. ([\#8536](https://github.com/matrix-org/synapse/issues/8536))
-- Fix a long standing bug where email notifications for encrypted messages were blank. ([\#8545](https://github.com/matrix-org/synapse/issues/8545))
-- Fix increase in the number of `There was no active span...` errors logged when using OpenTracing. ([\#8567](https://github.com/matrix-org/synapse/issues/8567))
-- Fix a bug that prevented errors encountered during execution of the `synapse_port_db` from being correctly printed. ([\#8585](https://github.com/matrix-org/synapse/issues/8585))
-- Fix appservice transactions to only include a maximum of 100 persistent and 100 ephemeral events. ([\#8606](https://github.com/matrix-org/synapse/issues/8606))
-
-
-Updates to the Docker image
----------------------------
-
-- Added multi-arch support (arm64,arm/v7) for the docker images. Contributed by @maquis196. ([\#7921](https://github.com/matrix-org/synapse/issues/7921))
-- Add support for passing commandline args to the synapse process. Contributed by @samuel-p. ([\#8390](https://github.com/matrix-org/synapse/issues/8390))
-
-
-Improved Documentation
-----------------------
-
-- Update the directions for using the manhole with coroutines. ([\#8462](https://github.com/matrix-org/synapse/issues/8462))
-- Improve readme by adding new shield.io badges. ([\#8493](https://github.com/matrix-org/synapse/issues/8493))
-- Added note about docker in manhole.md regarding which ip address to bind to. Contributed by @Maquis196. ([\#8526](https://github.com/matrix-org/synapse/issues/8526))
-- Document the new behaviour of the `allowed_lifetime_min` and `allowed_lifetime_max` settings in the room retention configuration. ([\#8529](https://github.com/matrix-org/synapse/issues/8529))
-
-
-Deprecations and Removals
--------------------------
-
-- Drop unused `device_max_stream_id` table. ([\#8589](https://github.com/matrix-org/synapse/issues/8589))
-
-
-Internal Changes
-----------------
-
-- Check for unreachable code with mypy. ([\#8432](https://github.com/matrix-org/synapse/issues/8432))
-- Add unit test for event persister sharding. ([\#8433](https://github.com/matrix-org/synapse/issues/8433))
-- Allow events to be sent to clients sooner when using sharded event persisters. ([\#8439](https://github.com/matrix-org/synapse/issues/8439), [\#8488](https://github.com/matrix-org/synapse/issues/8488), [\#8496](https://github.com/matrix-org/synapse/issues/8496), [\#8499](https://github.com/matrix-org/synapse/issues/8499))
-- Configure `public_baseurl` when using demo scripts. ([\#8443](https://github.com/matrix-org/synapse/issues/8443))
-- Add SQL logging on queries that happen during startup. ([\#8448](https://github.com/matrix-org/synapse/issues/8448))
-- Speed up unit tests when using PostgreSQL. ([\#8450](https://github.com/matrix-org/synapse/issues/8450))
-- Remove redundant database loads of stream_ordering for events we already have. ([\#8452](https://github.com/matrix-org/synapse/issues/8452))
-- Reduce inconsistencies between codepaths for membership and non-membership events. ([\#8463](https://github.com/matrix-org/synapse/issues/8463))
-- Combine `SpamCheckerApi` with the more generic `ModuleApi`. ([\#8464](https://github.com/matrix-org/synapse/issues/8464))
-- Additional testing for `ThirdPartyEventRules`. ([\#8468](https://github.com/matrix-org/synapse/issues/8468))
-- Add `-d` option to `./scripts-dev/lint.sh` to lint files that have changed since the last git commit. ([\#8472](https://github.com/matrix-org/synapse/issues/8472))
-- Unblacklist some sytests. ([\#8474](https://github.com/matrix-org/synapse/issues/8474))
-- Include the log level in the phone home stats. ([\#8477](https://github.com/matrix-org/synapse/issues/8477))
-- Remove outdated sphinx documentation, scripts and configuration. ([\#8480](https://github.com/matrix-org/synapse/issues/8480))
-- Clarify error message when plugin config parsers raise an error. ([\#8492](https://github.com/matrix-org/synapse/issues/8492))
-- Remove the deprecated `Handlers` object. ([\#8494](https://github.com/matrix-org/synapse/issues/8494))
-- Fix a threadsafety bug in unit tests. ([\#8497](https://github.com/matrix-org/synapse/issues/8497))
-- Add user agent to user_daily_visits table. ([\#8503](https://github.com/matrix-org/synapse/issues/8503))
-- Add type hints to various parts of the code base. ([\#8407](https://github.com/matrix-org/synapse/issues/8407), [\#8505](https://github.com/matrix-org/synapse/issues/8505), [\#8507](https://github.com/matrix-org/synapse/issues/8507), [\#8547](https://github.com/matrix-org/synapse/issues/8547), [\#8562](https://github.com/matrix-org/synapse/issues/8562), [\#8609](https://github.com/matrix-org/synapse/issues/8609))
-- Remove unused code from the test framework. ([\#8514](https://github.com/matrix-org/synapse/issues/8514))
-- Apply some internal fixes to the `HomeServer` class to make its code more idiomatic and statically-verifiable. ([\#8515](https://github.com/matrix-org/synapse/issues/8515))
-- Factor out common code between `RoomMemberHandler._locally_reject_invite` and `EventCreationHandler.create_event`. ([\#8537](https://github.com/matrix-org/synapse/issues/8537))
-- Improve database performance by executing more queries without starting transactions. ([\#8542](https://github.com/matrix-org/synapse/issues/8542))
-- Rename `Cache` to `DeferredCache`, to better reflect its purpose. ([\#8548](https://github.com/matrix-org/synapse/issues/8548))
-- Move metric registration code down into `LruCache`. ([\#8561](https://github.com/matrix-org/synapse/issues/8561), [\#8591](https://github.com/matrix-org/synapse/issues/8591))
-- Replace `DeferredCache` with the lighter-weight `LruCache` where possible. ([\#8563](https://github.com/matrix-org/synapse/issues/8563))
-- Add virtualenv-generated folders to `.gitignore`. ([\#8566](https://github.com/matrix-org/synapse/issues/8566))
-- Add `get_immediate` method to `DeferredCache`. ([\#8568](https://github.com/matrix-org/synapse/issues/8568))
-- Fix mypy not properly checking across the codebase, additionally, fix a typing assertion error in `handlers/auth.py`. ([\#8569](https://github.com/matrix-org/synapse/issues/8569))
-- Fix `synmark` benchmark runner. ([\#8571](https://github.com/matrix-org/synapse/issues/8571))
-- Modify `DeferredCache.get()` to return `Deferred`s instead of `ObservableDeferred`s. ([\#8572](https://github.com/matrix-org/synapse/issues/8572))
-- Adjust a protocol-type definition to fit `sqlite3` assertions. ([\#8577](https://github.com/matrix-org/synapse/issues/8577))
-- Support macOS on the `synmark` benchmark runner. ([\#8578](https://github.com/matrix-org/synapse/issues/8578))
-- Update `mypy` static type checker to 0.790. ([\#8583](https://github.com/matrix-org/synapse/issues/8583), [\#8600](https://github.com/matrix-org/synapse/issues/8600))
-- Re-organize the structured logging code to separate the TCP transport handling from the JSON formatting. ([\#8587](https://github.com/matrix-org/synapse/issues/8587))
-- Remove extraneous unittest logging decorators from unit tests. ([\#8592](https://github.com/matrix-org/synapse/issues/8592))
-- Minor optimisations in caching code. ([\#8593](https://github.com/matrix-org/synapse/issues/8593), [\#8594](https://github.com/matrix-org/synapse/issues/8594))
-
-
-Synapse 1.21.2 (2020-10-15)
-===========================
-
-Debian packages and Docker images have been rebuilt using the latest versions of dependency libraries, including authlib 0.15.1. Please see bugfixes below.
-
-Security advisory
------------------
-
-* HTML pages served via Synapse were vulnerable to cross-site scripting (XSS)
-  attacks. All server administrators are encouraged to upgrade.
-  ([\#8444](https://github.com/matrix-org/synapse/pull/8444))
-  ([CVE-2020-26891](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26891))
-
-  This fix was originally included in v1.21.0 but was missing a security advisory.
-
-  This was reported by [Denis Kasak](https://github.com/dkasak).
-
-Bugfixes
---------
-
-- Fix rare bug where sending an event would fail due to a racey assertion. ([\#8530](https://github.com/matrix-org/synapse/issues/8530))
-- An updated version of the authlib dependency is included in the Docker and Debian images to fix an issue using OpenID Connect. See [\#8534](https://github.com/matrix-org/synapse/issues/8534) for details.
-
-
-Synapse 1.21.1 (2020-10-13)
-===========================
-
-This release fixes a regression in v1.21.0 that prevented debian packages from being built.
-It is otherwise identical to v1.21.0.
-
-Synapse 1.21.0 (2020-10-12)
-===========================
-
-No significant changes since v1.21.0rc3.
-
-As [noted in
-v1.20.0](https://github.com/matrix-org/synapse/blob/release-v1.21.0/CHANGES.md#synapse-1200-2020-09-22),
-a future release will drop support for accessing Synapse's
-[Admin API](https://github.com/matrix-org/synapse/tree/master/docs/admin_api) under the
-`/_matrix/client/*` endpoint prefixes. At that point, the Admin API will only
-be accessible under `/_synapse/admin`.
-
-
 Synapse 1.21.0rc3 (2020-10-08)
 ==============================
 
@@ -322,11 +154,9 @@ API](https://github.com/matrix-org/synapse/tree/master/docs) has been
 accessible under the `/_matrix/client/api/v1/admin`,
 `/_matrix/client/unstable/admin`, `/_matrix/client/r0/admin` and
 `/_synapse/admin` prefixes. In a future release, we will be dropping support
-for accessing Synapse's Admin API using the `/_matrix/client/*` prefixes.
-
-From that point, the Admin API will only be accessible under `/_synapse/admin`.
-This makes it easier for homeserver admins to lock down external access to the
-Admin API endpoints.
+for accessing Synapse's Admin API using the `/_matrix/client/*` prefixes. This
+makes it easier for homeserver admins to lock down external access to the Admin
+API endpoints.
 
 Synapse 1.20.0rc5 (2020-09-18)
 ==============================

CONTRIBUTING.md

@@ -46,7 +46,7 @@ locally. You'll need python 3.6 or later, and to install a number of tools:
 
 ```
 # Install the dependencies
-pip install -e ".[lint,mypy]"
+pip install -e ".[lint]"
 
 # Run the linter script
 ./scripts-dev/lint.sh
@@ -63,10 +63,6 @@ run-time:
 ./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder
 ```
 
-You can also provide the `-d` option, which will lint the files that have been
-changed since the last git commit. This will often be significantly faster than
-linting the whole codebase.
-
 Before pushing new changes, ensure they don't produce linting errors. Commit any
 files that were corrected.
 

INSTALL.md

@@ -57,7 +57,7 @@ light workloads.
 System requirements:
 
 - POSIX-compliant system (tested on Linux & OS X)
-- Python 3.5.2 or later, up to Python 3.9.
+- Python 3.5.2 or later, up to Python 3.8.
 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
 
 Synapse is written in Python but some of the libraries it uses are written in

README.rst

@@ -256,9 +256,9 @@ directory of your choice::
 Synapse has a number of external dependencies, that are easiest
 to install using pip and a virtualenv::
 
-    python3 -m venv ./env
-    source ./env/bin/activate
-    pip install -e ".[all,test]"
+    virtualenv -p python3 env
+    source env/bin/activate
+    python -m pip install --no-use-pep517 -e ".[all]"
 
 This will run a process of downloading and installing all the needed
 dependencies into a virtual env.
@@ -270,9 +270,9 @@ check that everything is installed as it should be::
 
 This should end with a 'PASSED' result::
 
-    Ran 1266 tests in 643.930s
+    Ran 143 tests in 0.601s
 
-    PASSED (skips=15, successes=1251)
+    PASSED (successes=143)
 
 Running the Integration Tests
 =============================

UPGRADE.rst

@@ -75,22 +75,6 @@ for example:
      wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
      dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
 
-Upgrading to v1.23.0
-====================
-
-Structured logging configuration breaking changes
--------------------------------------------------
-
-This release deprecates use of the ``structured: true`` logging configuration for
-structured logging. If your logging configuration contains ``structured: true``
-then it should be modified based on the `structured logging documentation
-<https://github.com/matrix-org/synapse/blob/master/docs/structured_logging.md>`_.
-
-The ``structured`` and ``drains`` logging options are now deprecated and should
-be replaced by standard logging configuration of ``handlers`` and ``formatters`.
-
-A future will release of Synapse will make using ``structured: true`` an error.
-
 Upgrading to v1.22.0
 ====================
 

changelog.d/6739.feature

@@ -0,0 +1 @@
+Implement "knock" feature as per MSC2403. Contributed by Sorunome.

changelog.d/7658.feature

@@ -0,0 +1 @@
+Add a configuration option for always using the "userinfo endpoint" for OpenID Connect. This fixes support for some identity providers, e.g. GitLab. Contributed by Benjamin Koch.

changelog.d/8292.feature

@@ -0,0 +1 @@
+Allow `ThirdPartyEventRules` modules to query and manipulate whether a room is in the public rooms directory.

changelog.d/8312.feature

@@ -0,0 +1 @@
+Add support for olm fallback keys ([MSC2732](https://github.com/matrix-org/matrix-doc/pull/2732)).

changelog.d/8369.feature

@@ -0,0 +1 @@
+Allow running background tasks in a separate worker process.

changelog.d/8380.feature

@@ -0,0 +1 @@
+Add support for device dehydration ([MSC2697](https://github.com/matrix-org/matrix-doc/pull/2697)).

changelog.d/8407.misc

@@ -0,0 +1 @@
+Add typing information to the device handler.

changelog.d/8432.misc

@@ -0,0 +1 @@
+Check for unreachable code with mypy.

changelog.d/8433.misc

@@ -0,0 +1 @@
+Add unit test for event persister sharding.

changelog.d/8439.misc

@@ -0,0 +1 @@
+Allow events to be sent to clients sooner when using sharded event persisters.

changelog.d/8443.misc

@@ -0,0 +1 @@
+Configure `public_baseurl` when using demo scripts.

changelog.d/8448.misc

@@ -0,0 +1 @@
+Add SQL logging on queries that happen during startup.

changelog.d/8450.misc

@@ -0,0 +1 @@
+Speed up unit tests when using PostgreSQL.

changelog.d/8452.misc

@@ -0,0 +1 @@
+Remove redundant databae loads of stream_ordering for events we already have.

changelog.d/8454.bugfix

@@ -0,0 +1 @@
+Fix a longstanding bug where invalid ignored users in account data could break clients.

changelog.d/8455.bugfix

@@ -1 +0,0 @@
-Fix fetching of E2E cross signing keys over federation when only one of the master key and device signing key is cached already.

changelog.d/8457.bugfix

@@ -0,0 +1 @@
+Fix a bug where backfilling a room with an event that was missing the `redacts` field would break.

changelog.d/8458.feature

@@ -0,0 +1 @@
+Allow running background tasks in a separate worker process.

changelog.d/8461.feature

@@ -0,0 +1 @@
+Change default room version to "6", per [MSC2788](https://github.com/matrix-org/matrix-doc/pull/2788).

changelog.d/8462.doc

@@ -0,0 +1 @@
+Update the directions for using the manhole with coroutines.

changelog.d/8463.misc

@@ -0,0 +1 @@
+Reduce inconsistencies between codepaths for membership and non-membership events.

changelog.d/8464.misc

@@ -0,0 +1 @@
+Combine `SpamCheckerApi` with the more generic `ModuleApi`.

changelog.d/8465.bugfix

@@ -0,0 +1 @@
+Don't attempt to respond to some requests if the client has already disconnected.

changelog.d/8467.feature

@@ -0,0 +1 @@
+Allow `ThirdPartyEventRules` modules to query and manipulate whether a room is in the public rooms directory.

changelog.d/8468.misc

@@ -0,0 +1 @@
+Additional testing for `ThirdPartyEventRules`.

changelog.d/8474.misc

@@ -0,0 +1 @@
+Unblacklist some sytests.

changelog.d/8477.misc

@@ -0,0 +1 @@
+Include the log level in the phone home stats.

changelog.d/8480.misc

@@ -0,0 +1 @@
+Remove outdated sphinx documentation, scripts and configuration.

changelog.d/8486.bugfix

@@ -0,0 +1 @@
+Fix incremental sync returning an incorrect `prev_batch` token in timeline section, which when used to paginate returned events that were included in the incremental sync. Broken since v0.16.0.

changelog.d/8492.misc

@@ -0,0 +1 @@
+Clarify error message when plugin config parsers raise an error.

changelog.d/8493.doc

@@ -0,0 +1 @@
+Improve readme by adding new shield.io badges.

changelog.d/8519.feature

@@ -1 +0,0 @@
-Add an admin api to delete a single file or files were not used for a defined time from server. Contributed by @dklimpel.

changelog.d/8539.feature

@@ -1 +0,0 @@
-Split admin API for reported events (`GET /_synapse/admin/v1/event_reports`) into detail and list endpoints. This is a breaking change to #8217 which was introduced in Synapse v1.21.0. Those who already use this API should check their scripts. Contributed by @dklimpel.

changelog.d/8559.misc

@@ -1 +0,0 @@
-Optimise `/createRoom` with multiple invited users.

changelog.d/8580.bugfix

@@ -1 +0,0 @@
-Fix a bug where Synapse would blindly forward bad responses from federation to clients when retrieving profile information.

changelog.d/8582.doc

@@ -1 +0,0 @@
-Instructions for Azure AD in the OpenID Connect documentation. Contributed by peterk.

changelog.d/8595.misc

@@ -1 +0,0 @@
-Implement and use an @lru_cache decorator.

changelog.d/8607.feature

@@ -1 +0,0 @@
-Support generating structured logs via the standard logging configuration.

changelog.d/8610.feature

@@ -1 +0,0 @@
-Add an admin APIs to allow server admins to list users' pushers. Contributed by @dklimpel.

changelog.d/8614.misc

@@ -1 +0,0 @@
-Don't instansiate Requester directly.

changelog.d/8615.misc

@@ -1 +0,0 @@
-Type hints for `RegistrationStore`.

changelog.d/8616.misc

@@ -1 +0,0 @@
-Change schema to support access tokens belonging to one user but granting access to another.

changelog.d/8620.bugfix

@@ -1 +0,0 @@
-Fix a bug where the account validity endpoint would silently fail if the user ID did not have an expiration time. It now returns a 400 error.

changelog.d/8621.misc

@@ -1 +0,0 @@
-Remove unused OPTIONS handlers.

changelog.d/8627.bugfix

@@ -1 +0,0 @@
-Fix email notifications for invites without local state.

changelog.d/8628.bugfix

@@ -1 +0,0 @@
-Fix handling of invalid group IDs to return a 400 rather than log an exception and return a 500.

changelog.d/8632.bugfix

@@ -1 +0,0 @@
-Fix handling of User-Agent headers that are invalid UTF-8, which caused user agents of users to not get correctly recorded.

changelog.d/8633.misc

@@ -1 +0,0 @@
-Run `mypy` as part of the lint.sh script.

changelog.d/8634.misc

@@ -1 +0,0 @@
-Correct Synapse's PyPI package name in the OpenID Connect installation instructions.

changelog.d/8635.doc

@@ -1 +0,0 @@
-Improve the sample configuration for single sign-on providers.

changelog.d/8639.misc

@@ -1 +0,0 @@
-Fix typos and spelling errors in the code.

changelog.d/8640.misc

@@ -1 +0,0 @@
-Reduce number of OpenTracing spans started.

changelog.d/8643.bugfix

@@ -1 +0,0 @@
-Fix a bug in the `joined_rooms` admin API if the user has never joined any rooms. The bug was introduced, along with the API, in v1.21.0.

changelog.d/8644.misc

@@ -1 +0,0 @@
-Add field `total` to device list in admin API.

changelog.d/8647.feature

@@ -1 +0,0 @@
-Add an admin API `GET /_synapse/admin/v1/users/<user_id>/media` to get information about uploaded media. Contributed by @dklimpel.

changelog.d/8655.misc

@@ -1 +0,0 @@
-Add more type hints to the application services code.

changelog.d/8657.doc

@@ -1 +0,0 @@
-Fix the filepath of Dex's example config and the link to Dex's Getting Started guide in the OpenID Connect docs.

changelog.d/8664.misc

@@ -1 +0,0 @@
-Tell Black to format code for Python 3.5.

changelog.d/8665.doc

@@ -1 +0,0 @@
-Note support for Python 3.9.

changelog.d/8666.doc

@@ -1 +0,0 @@
-Minor updates to docs on running tests.

changelog.d/8667.doc

@@ -1 +0,0 @@
-Interlink prometheus/grafana documentation.

changelog.d/8668.misc

@@ -1 +0,0 @@
-Reduce number of OpenTracing spans started.

changelog.d/8669.misc

@@ -1 +0,0 @@
-Don't pull event from DB when handling replication traffic.

changelog.d/8670.misc

@@ -1 +0,0 @@
-Reduce number of OpenTracing spans started.

changelog.d/8671.misc

@@ -1 +0,0 @@
-Abstract some invite-related code in preparation for landing knocking.

changelog.d/8679.misc

@@ -1 +0,0 @@
-Clarify representation of events in logfiles.

changelog.d/8680.misc

@@ -1 +0,0 @@
-Don't require `hiredis` package to be installed to run unit tests.

changelog.d/8682.bugfix

@@ -1 +0,0 @@
-Fix exception during handling multiple concurrent requests for remote media when using multiple media repositories.

changelog.d/8684.misc

@@ -1 +0,0 @@
-Fix typing info on cache call signature to accept `on_invalidate`.

changelog.d/8685.feature

@@ -1 +0,0 @@
-Support generating structured logs via the standard logging configuration.

changelog.d/8688.misc

@@ -1 +0,0 @@
-Abstract some invite-related code in preparation for landing knocking.

changelog.d/8689.feature

@@ -1 +0,0 @@
-Add an admin APIs to allow server admins to list users' pushers. Contributed by @dklimpel.

changelog.d/8690.misc

@@ -1 +0,0 @@
-Fail tests if they do not await coroutines.

changelog.d/8693.misc

@@ -1 +0,0 @@
-Add more type hints to the application services code.

contrib/grafana/README.md

@@ -3,4 +3,4 @@
 0. Set up Prometheus and Grafana. Out of scope for this readme. Useful documentation about using Grafana with Prometheus: http://docs.grafana.org/features/datasources/prometheus/
 1. Have your Prometheus scrape your Synapse. https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md
 2. Import dashboard into Grafana. Download `synapse.json`. Import it to Grafana and select the correct Prometheus datasource. http://docs.grafana.org/reference/export_import/
-3. Set up required recording rules. https://github.com/matrix-org/synapse/tree/master/contrib/prometheus
+3. Set up additional recording rules

debian/build_virtualenv

@@ -42,7 +42,7 @@ dh_virtualenv \
     --preinstall="mock" \
     --extra-pip-arg="--no-cache-dir" \
     --extra-pip-arg="--compile" \
-    --extras="all,systemd,test"
+    --extras="all,systemd"
 
 PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
 VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"

debian/changelog

@@ -1,38 +1,3 @@
-matrix-synapse-py3 (1.22.1) stable; urgency=medium
-
-  * New synapse release 1.22.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 30 Oct 2020 15:25:37 +0000
-
-matrix-synapse-py3 (1.22.0) stable; urgency=medium
-
-  * New synapse release 1.22.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 27 Oct 2020 12:07:12 +0000
-
-matrix-synapse-py3 (1.21.2) stable; urgency=medium
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.21.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 15 Oct 2020 09:23:27 -0400
-
-matrix-synapse-py3 (1.21.1) stable; urgency=medium
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.21.1.
-
-  [ Andrew Morgan ]
-  * Explicitly install "test" python dependencies.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Oct 2020 10:24:13 +0100
-
-matrix-synapse-py3 (1.21.0) stable; urgency=medium
-
-  * New synapse release 1.21.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 12 Oct 2020 15:47:44 +0100
-
 matrix-synapse-py3 (1.20.1) stable; urgency=medium
 
   * New synapse release 1.20.1.

docker/README.md

@@ -83,7 +83,7 @@ docker logs synapse
 If all is well, you should now be able to connect to http://localhost:8008 and
 see a confirmation message.
 
-The following environment variables are supported in `run` mode:
+The following environment variables are supported in run mode:
 
 * `SYNAPSE_CONFIG_DIR`: where additional config files are stored. Defaults to
   `/data`.
@@ -94,20 +94,6 @@ The following environment variables are supported in `run` mode:
 * `UID`, `GID`: the user and group id to run Synapse as. Defaults to `991`, `991`.
 * `TZ`: the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) the container will run with. Defaults to `UTC`.
 
-For more complex setups (e.g. for workers) you can also pass your args directly to synapse using `run` mode. For example like this:
-
-```
-docker run -d --name synapse \
-    --mount type=volume,src=synapse-data,dst=/data \
-    -p 8008:8008 \
-    matrixdotorg/synapse:latest run \
-    -m synapse.app.generic_worker \
-    --config-path=/data/homeserver.yaml \
-    --config-path=/data/generic_worker.yaml
-```
-
-If you do not provide `-m`, the value of the `SYNAPSE_WORKER` environment variable is used. If you do not provide at least one `--config-path` or `-c`, the value of the `SYNAPSE_CONFIG_PATH` environment variable is used instead.
-
 ## Generating an (admin) user
 
 After synapse is running, you may wish to create a user via `register_new_matrix_user`.

docker/conf/homeserver.yaml

@@ -90,7 +90,7 @@ federation_rc_concurrent: 3
 
 media_store_path: "/data/media"
 uploads_path: "/data/uploads"
-max_upload_size: "{{ SYNAPSE_MAX_UPLOAD_SIZE or "50M" }}"
+max_upload_size: "{{ SYNAPSE_MAX_UPLOAD_SIZE or "10M" }}"
 max_image_pixels: "32M"
 dynamic_thumbnails: false
 

docker/start.py

@@ -179,7 +179,7 @@ def run_generate_config(environ, ownership):
 
 
 def main(args, environ):
-    mode = args[1] if len(args) > 1 else "run"
+    mode = args[1] if len(args) > 1 else None
     desired_uid = int(environ.get("UID", "991"))
     desired_gid = int(environ.get("GID", "991"))
     synapse_worker = environ.get("SYNAPSE_WORKER", "synapse.app.homeserver")
@@ -205,47 +205,36 @@ def main(args, environ):
             config_dir, config_path, environ, ownership
         )
 
-    if mode != "run":
+    if mode is not None:
         error("Unknown execution mode '%s'" % (mode,))
 
-    args = args[2:]
+    config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
+    config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
 
-    if "-m" not in args:
-        args = ["-m", synapse_worker] + args
-
-    # if there are no config files passed to synapse, try adding the default file
-    if not any(p.startswith("--config-path") or p.startswith("-c") for p in args):
-        config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
-        config_path = environ.get(
-            "SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml"
-        )
-
-        if not os.path.exists(config_path):
-            if "SYNAPSE_SERVER_NAME" in environ:
-                error(
-                    """\
+    if not os.path.exists(config_path):
+        if "SYNAPSE_SERVER_NAME" in environ:
+            error(
+                """\
 Config file '%s' does not exist.
 
 The synapse docker image no longer supports generating a config file on-the-fly
 based on environment variables. You can migrate to a static config file by
 running with 'migrate_config'. See the README for more details.
 """
-                    % (config_path,)
-                )
-
-            error(
-                "Config file '%s' does not exist. You should either create a new "
-                "config file by running with the `generate` argument (and then edit "
-                "the resulting file before restarting) or specify the path to an "
-                "existing config file with the SYNAPSE_CONFIG_PATH variable."
                 % (config_path,)
             )
 
-        args += ["--config-path", config_path]
+        error(
+            "Config file '%s' does not exist. You should either create a new "
+            "config file by running with the `generate` argument (and then edit "
+            "the resulting file before restarting) or specify the path to an "
+            "existing config file with the SYNAPSE_CONFIG_PATH variable."
+            % (config_path,)
+        )
 
-    log("Starting synapse with args " + " ".join(args))
+    log("Starting synapse with config file " + config_path)
 
-    args = ["python"] + args
+    args = ["python", "-m", synapse_worker, "--config-path", config_path]
     if ownership is not None:
         args = ["gosu", ownership] + args
         os.execv("/usr/sbin/gosu", args)

docs/admin_api/event_reports.rst

@@ -17,26 +17,67 @@ It returns a JSON body like the following:
     {
         "event_reports": [
             {
+                "content": {
+                    "reason": "foo",
+                    "score": -100
+                },
                 "event_id": "$bNUFCwGzWca1meCGkjp-zwslF-GfVcXukvRLI1_FaVY",
+                "event_json": {
+                    "auth_events": [
+                        "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M",
+                        "$oggsNXxzPFRE3y53SUNd7nsj69-QzKv03a1RucHu-ws"
+                    ],
+                    "content": {
+                        "body": "matrix.org: This Week in Matrix",
+                        "format": "org.matrix.custom.html",
+                        "formatted_body": "<strong>matrix.org</strong>:<br><a href=\"https://matrix.org/blog/\"><strong>This Week in Matrix</strong></a>",
+                        "msgtype": "m.notice"
+                    },
+                    "depth": 546,
+                    "hashes": {
+                        "sha256": "xK1//xnmvHJIOvbgXlkI8eEqdvoMmihVDJ9J4SNlsAw"
+                    },
+                    "origin": "matrix.org",
+                    "origin_server_ts": 1592291711430,
+                    "prev_events": [
+                        "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M"
+                    ],
+                    "prev_state": [],
+                    "room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org",
+                    "sender": "@foobar:matrix.org",
+                    "signatures": {
+                        "matrix.org": {
+                            "ed25519:a_JaEG": "cs+OUKW/iHx5pEidbWxh0UiNNHwe46Ai9LwNz+Ah16aWDNszVIe2gaAcVZfvNsBhakQTew51tlKmL2kspXk/Dg"
+                        }
+                    },
+                    "type": "m.room.message",
+                    "unsigned": {
+                        "age_ts": 1592291711430,
+                    }
+                },
                 "id": 2,
                 "reason": "foo",
-                "score": -100,
                 "received_ts": 1570897107409,
-                "canonical_alias": "#alias1:matrix.org",
+                "room_alias": "#alias1:matrix.org",
                 "room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org",
-                "name": "Matrix HQ",
                 "sender": "@foobar:matrix.org",
                 "user_id": "@foo:matrix.org"
             },
             {
+                "content": {
+                    "reason": "bar",
+                    "score": -100
+                },
                 "event_id": "$3IcdZsDaN_En-S1DF4EMCy3v4gNRKeOJs8W5qTOKj4I",
+                "event_json": {
+                    // hidden items
+                    // see above
+                },
                 "id": 3,
                 "reason": "bar",
-                "score": -100,
                 "received_ts": 1598889612059,
-                "canonical_alias": "#alias2:matrix.org",
+                "room_alias": "#alias2:matrix.org",
                 "room_id": "!eGvUQuTCkHGVwNMOjv:matrix.org",
-                "name": "Your room name here",
                 "sender": "@foobar:matrix.org",
                 "user_id": "@bar:matrix.org"
             }
@@ -72,94 +113,17 @@ The following fields are returned in the JSON response body:
 - ``id``: integer - ID of event report.
 - ``received_ts``: integer - The timestamp (in milliseconds since the unix epoch) when this report was sent.
 - ``room_id``: string - The ID of the room in which the event being reported is located.
-- ``name``: string - The name of the room.
 - ``event_id``: string - The ID of the reported event.
 - ``user_id``: string - This is the user who reported the event and wrote the reason.
 - ``reason``: string - Comment made by the ``user_id`` in this report. May be blank.
-- ``score``: integer - Content is reported based upon a negative score, where -100 is "most offensive" and 0 is "inoffensive".
+- ``content``: object - Content of reported event.
+
+  - ``reason``: string - Comment made by the ``user_id`` in this report. May be blank.
+  - ``score``: integer - Content is reported based upon a negative score, where -100 is "most offensive" and 0 is "inoffensive".
+
 - ``sender``: string - This is the ID of the user who sent the original message/event that was reported.
-- ``canonical_alias``: string - The canonical alias of the room. ``null`` if the room does not have a canonical alias set.
+- ``room_alias``: string - The alias of the room. ``null`` if the room does not have a canonical alias set.
+- ``event_json``: object - Details of the original event that was reported.
 - ``next_token``: integer - Indication for pagination. See above.
 - ``total``: integer - Total number of event reports related to the query (``user_id`` and ``room_id``).
 
-Show details of a specific event report
-=======================================
-
-This API returns information about a specific event report.
-
-The api is::
-
-    GET /_synapse/admin/v1/event_reports/<report_id>
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-It returns a JSON body like the following:
-
-.. code:: jsonc
-
-    {
-        "event_id": "$bNUFCwGzWca1meCGkjp-zwslF-GfVcXukvRLI1_FaVY",
-        "event_json": {
-            "auth_events": [
-                "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M",
-                "$oggsNXxzPFRE3y53SUNd7nsj69-QzKv03a1RucHu-ws"
-            ],
-            "content": {
-                "body": "matrix.org: This Week in Matrix",
-                "format": "org.matrix.custom.html",
-                "formatted_body": "<strong>matrix.org</strong>:<br><a href=\"https://matrix.org/blog/\"><strong>This Week in Matrix</strong></a>",
-                "msgtype": "m.notice"
-            },
-            "depth": 546,
-            "hashes": {
-                "sha256": "xK1//xnmvHJIOvbgXlkI8eEqdvoMmihVDJ9J4SNlsAw"
-            },
-            "origin": "matrix.org",
-            "origin_server_ts": 1592291711430,
-            "prev_events": [
-                "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M"
-            ],
-            "prev_state": [],
-            "room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org",
-            "sender": "@foobar:matrix.org",
-            "signatures": {
-                "matrix.org": {
-                    "ed25519:a_JaEG": "cs+OUKW/iHx5pEidbWxh0UiNNHwe46Ai9LwNz+Ah16aWDNszVIe2gaAcVZfvNsBhakQTew51tlKmL2kspXk/Dg"
-                }
-            },
-            "type": "m.room.message",
-            "unsigned": {
-                "age_ts": 1592291711430,
-            }
-        },
-        "id": <report_id>,
-        "reason": "foo",
-        "score": -100,
-        "received_ts": 1570897107409,
-        "canonical_alias": "#alias1:matrix.org",
-        "room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org",
-        "name": "Matrix HQ",
-        "sender": "@foobar:matrix.org",
-        "user_id": "@foo:matrix.org"
-    }
-
-**URL parameters:**
-
-- ``report_id``: string - The ID of the event report.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``id``: integer - ID of event report.
-- ``received_ts``: integer - The timestamp (in milliseconds since the unix epoch) when this report was sent.
-- ``room_id``: string - The ID of the room in which the event being reported is located.
-- ``name``: string - The name of the room.
-- ``event_id``: string - The ID of the reported event.
-- ``user_id``: string - This is the user who reported the event and wrote the reason.
-- ``reason``: string - Comment made by the ``user_id`` in this report. May be blank.
-- ``score``: integer - Content is reported based upon a negative score, where -100 is "most offensive" and 0 is "inoffensive".
-- ``sender``: string - This is the ID of the user who sent the original message/event that was reported.
-- ``canonical_alias``: string - The canonical alias of the room. ``null`` if the room does not have a canonical alias set.
-- ``event_json``: object - Details of the original event that was reported.

docs/admin_api/media_admin_api.md

@@ -100,82 +100,3 @@ Response:
   "num_quarantined": 10  # The number of media items successfully quarantined
 }
 ```
-
-# Delete local media
-This API deletes the *local* media from the disk of your own server.
-This includes any local thumbnails and copies of media downloaded from
-remote homeservers.
-This API will not affect media that has been uploaded to external
-media repositories (e.g https://github.com/turt2live/matrix-media-repo/).
-See also [purge_remote_media.rst](purge_remote_media.rst).
-
-## Delete a specific local media
-Delete a specific `media_id`.
-
-Request:
-
-```
-DELETE /_synapse/admin/v1/media/<server_name>/<media_id>
-
-{}
-```
-
-URL Parameters
-
-* `server_name`: string - The name of your local server (e.g `matrix.org`)
-* `media_id`: string - The ID of the media (e.g `abcdefghijklmnopqrstuvwx`)
-
-Response:
-
-```json
-    {
-       "deleted_media": [
-          "abcdefghijklmnopqrstuvwx"
-       ],
-       "total": 1
-    }
-```
-
-The following fields are returned in the JSON response body:
-
-* `deleted_media`: an array of strings - List of deleted `media_id`
-* `total`: integer - Total number of deleted `media_id`
-
-## Delete local media by date or size
-
-Request:
-
-```
-POST /_synapse/admin/v1/media/<server_name>/delete?before_ts=<before_ts>
-
-{}
-```
-
-URL Parameters
-
-* `server_name`: string - The name of your local server (e.g `matrix.org`).
-* `before_ts`: string representing a positive integer - Unix timestamp in ms.
-Files that were last used before this timestamp will be deleted. It is the timestamp of
-last access and not the timestamp creation. 
-* `size_gt`: Optional - string representing a positive integer - Size of the media in bytes.
-Files that are larger will be deleted. Defaults to `0`.
-* `keep_profiles`: Optional - string representing a boolean - Switch to also delete files
-that are still used in image data (e.g user profile, room avatar).
-If `false` these files will be deleted. Defaults to `true`.
-
-Response:
-
-```json
-    {
-       "deleted_media": [
-          "abcdefghijklmnopqrstuvwx",
-          "abcdefghijklmnopqrstuvwz"
-       ],
-       "total": 2
-    }
-```
-
-The following fields are returned in the JSON response body:
-
-* `deleted_media`: an array of strings - List of deleted `media_id`
-* `total`: integer - Total number of deleted `media_id`

docs/admin_api/user_admin_api.rst

@@ -341,89 +341,6 @@ The following fields are returned in the JSON response body:
 - ``total`` - Number of rooms.
 
 
-List media of an user
-================================
-Gets a list of all local media that a specific ``user_id`` has created.
-The response is ordered by creation date descending and media ID descending.
-The newest media is on top.
-
-The API is::
-
-  GET /_synapse/admin/v1/users/<user_id>/media
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-      "media": [
-        {
-          "created_ts": 100400,
-          "last_access_ts": null,
-          "media_id": "qXhyRzulkwLsNHTbpHreuEgo",
-          "media_length": 67,
-          "media_type": "image/png",
-          "quarantined_by": null,
-          "safe_from_quarantine": false,
-          "upload_name": "test1.png"
-        },
-        {
-          "created_ts": 200400,
-          "last_access_ts": null,
-          "media_id": "FHfiSnzoINDatrXHQIXBtahw",
-          "media_length": 67,
-          "media_type": "image/png",
-          "quarantined_by": null,
-          "safe_from_quarantine": false,
-          "upload_name": "test2.png"
-        }
-      ],
-      "next_token": 3,
-      "total": 2
-    }
-
-To paginate, check for ``next_token`` and if present, call the endpoint again
-with ``from`` set to the value of ``next_token``. This will return a new page.
-
-If the endpoint does not return a ``next_token`` then there are no more
-reports to paginate through.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - string - fully qualified: for example, ``@user:server.com``.
-- ``limit``: string representing a positive integer - Is optional but is used for pagination,
-  denoting the maximum number of items to return in this call. Defaults to ``100``.
-- ``from``: string representing a positive integer - Is optional but used for pagination,
-  denoting the offset in the returned results. This should be treated as an opaque value and
-  not explicitly set to anything other than the return value of ``next_token`` from a previous call.
-  Defaults to ``0``.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``media`` - An array of objects, each containing information about a media.
-  Media objects contain the following fields:
-
-  - ``created_ts`` - integer - Timestamp when the content was uploaded in ms.
-  - ``last_access_ts`` - integer - Timestamp when the content was last accessed in ms.
-  - ``media_id`` - string - The id used to refer to the media.
-  - ``media_length`` - integer - Length of the media in bytes.
-  - ``media_type`` - string - The MIME-type of the media.
-  - ``quarantined_by`` - string - The user ID that initiated the quarantine request
-    for this media.
-
-  - ``safe_from_quarantine`` - bool - Status if this media is safe from quarantining.
-  - ``upload_name`` - string - The name the media was uploaded with.
-
-- ``next_token``: integer - Indication for pagination. See above.
-- ``total`` - integer - Total number of media.
-
 User devices
 ============
 
@@ -458,8 +375,7 @@ A response body like the following is returned:
           "last_seen_ts": 1474491775025,
           "user_id": "<user_id>"
         }
-      ],
-      "total": 2
+      ]
     }
 
 **Parameters**
@@ -484,8 +400,6 @@ The following fields are returned in the JSON response body:
     devices was last seen. (May be a few minutes out of date, for efficiency reasons).
   - ``user_id`` - Owner of  device.
 
-- ``total`` - Total number of user's devices.
-
 Delete multiple devices
 ------------------
 Deletes the given devices for a specific ``user_id``, and invalidates
@@ -611,82 +525,3 @@ The following parameters should be set in the URL:
 
 - ``user_id`` - fully qualified: for example, ``@user:server.com``.
 - ``device_id`` - The device to delete.
-
-List all pushers
-================
-Gets information about all pushers for a specific ``user_id``.
-
-The API is::
-
-  GET /_synapse/admin/v1/users/<user_id>/pushers
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-      "pushers": [
-        {
-          "app_display_name":"HTTP Push Notifications",
-          "app_id":"m.http",
-          "data": {
-            "url":"example.com"
-          },
-          "device_display_name":"pushy push",
-          "kind":"http",
-          "lang":"None",
-          "profile_tag":"",
-          "pushkey":"a@example.com"
-        }
-      ],
-      "total": 1
-    }
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``pushers`` - An array containing the current pushers for the user
-
-  - ``app_display_name`` - string - A string that will allow the user to identify
-    what application owns this pusher.
-
-  - ``app_id`` - string - This is a reverse-DNS style identifier for the application.
-    Max length, 64 chars.
-
-  - ``data`` - A dictionary of information for the pusher implementation itself.
-
-    - ``url`` - string - Required if ``kind`` is ``http``. The URL to use to send
-      notifications to.
-
-    - ``format`` - string - The format to use when sending notifications to the
-      Push Gateway.
-
-  - ``device_display_name`` - string -  A string that will allow the user to identify
-    what device owns this pusher.
-
-  - ``profile_tag`` - string - This string determines which set of device specific rules
-    this pusher executes.
-
-  - ``kind`` - string -  The kind of pusher. "http" is a pusher that sends HTTP pokes.
-  - ``lang`` - string - The preferred language for receiving notifications
-    (e.g. 'en' or 'en-US')
-
-  - ``profile_tag`` - string - This string determines which set of device specific rules
-    this pusher executes.
-
-  - ``pushkey`` - string - This is a unique identifier for this pusher.
-    Max length, 512 bytes.
-
-- ``total`` - integer - Number of pushers.
-
-See also `Client-Server API Spec <https://matrix.org/docs/spec/client_server/latest#get-matrix-client-r0-pushers>`_

docs/manhole.md

@@ -5,45 +5,8 @@ The "manhole" allows server administrators to access a Python shell on a running
 Synapse installation. This is a very powerful mechanism for administration and
 debugging.
 
-**_Security Warning_**
-
-Note that this will give administrative access to synapse to **all users** with
-shell access to the server. It should therefore **not** be enabled in
-environments where untrusted users have shell access.
-
-***
-
 To enable it, first uncomment the `manhole` listener configuration in
-`homeserver.yaml`. The configuration is slightly different if you're using docker.
-
-#### Docker config
-
-If you are using Docker, set `bind_addresses` to `['0.0.0.0']` as shown:
-
-```yaml
-listeners:
-  - port: 9000
-    bind_addresses: ['0.0.0.0']
-    type: manhole
-```
-
-When using `docker run` to start the server, you will then need to change the command to the following to include the
-`manhole` port forwarding. The `-p 127.0.0.1:9000:9000` below is important: it 
-ensures that access to the `manhole` is only possible for local users.
-
-```bash
-docker run -d --name synapse \
-    --mount type=volume,src=synapse-data,dst=/data \
-    -p 8008:8008 \
-    -p 127.0.0.1:9000:9000 \
-    matrixdotorg/synapse:latest
-```
-
-#### Native config
-
-If you are not using docker, set `bind_addresses` to `['::1', '127.0.0.1']` as shown.
-The `bind_addresses` in the example below is important: it ensures that access to the
-`manhole` is only possible for local users).
+`homeserver.yaml`:
 
 ```yaml
 listeners:
@@ -52,7 +15,12 @@ listeners:
     type: manhole
 ```
 
-#### Accessing synapse manhole
+(`bind_addresses` in the above is important: it ensures that access to the
+manhole is only possible for local users).
+
+Note that this will give administrative access to synapse to **all users** with
+shell access to the server. It should therefore **not** be enabled in
+environments where untrusted users have shell access.
 
 Then restart synapse, and point an ssh client at port 9000 on localhost, using
 the username `matrix`:

docs/message_retention_policies.md

@@ -136,34 +136,24 @@ the server's database.
 
 ### Lifetime limits
 
-Server admins can set limits on the values of `max_lifetime` to use when
-purging old events in a room. These limits can be defined as such in the
-`retention` section of the configuration file:
+**Note: this feature is mainly useful within a closed federation or on
+servers that don't federate, because there currently is no way to
+enforce these limits in an open federation.**
+
+Server admins can restrict the values their local users are allowed to
+use for both `min_lifetime` and `max_lifetime`. These limits can be
+defined as such in the `retention` section of the configuration file:
 
 ```yaml
   allowed_lifetime_min: 1d
   allowed_lifetime_max: 1y
 ```
 
-The limits are considered when running purge jobs. If necessary, the
-effective value of `max_lifetime` will be brought between
-`allowed_lifetime_min` and `allowed_lifetime_max` (inclusive).
-This means that, if the value of `max_lifetime` defined in the room's state
-is lower than `allowed_lifetime_min`, the value of `allowed_lifetime_min`
-will be used instead. Likewise, if the value of `max_lifetime` is higher
-than `allowed_lifetime_max`, the value of `allowed_lifetime_max` will be
-used instead.
-
-In the example above, we ensure Synapse never deletes events that are less
-than one day old, and that it always deletes events that are over a year
-old.
-
-If a default policy is set, and its `max_lifetime` value is lower than
-`allowed_lifetime_min` or higher than `allowed_lifetime_max`, the same
-process applies.
-
-Both parameters are optional; if one is omitted Synapse won't use it to
-adjust the effective value of `max_lifetime`.
+Here, `allowed_lifetime_min` is the lowest value a local user can set
+for both `min_lifetime` and `max_lifetime`, and `allowed_lifetime_max`
+is the highest value. Both parameters are optional (e.g. setting
+`allowed_lifetime_min` but not `allowed_lifetime_max` only enforces a
+minimum and no maximum).
 
 Like other settings in this section, these parameters can be expressed
 either as a duration or as a number of milliseconds.

docs/metrics-howto.md

@@ -60,8 +60,6 @@
 
 1.  Restart Prometheus.
 
-1.  Consider using the [grafana dashboard](https://github.com/matrix-org/synapse/tree/master/contrib/grafana/) and required [recording rules](https://github.com/matrix-org/synapse/tree/master/contrib/prometheus/) 
-
 ## Monitoring workers
 
 To monitor a Synapse installation using

docs/openid.md

@@ -37,7 +37,7 @@ as follows:
    provided by `matrix.org` so no further action is needed.
 
  * If you installed Synapse into a virtualenv, run `/path/to/env/bin/pip
-   install matrix-synapse[oidc]` to install the necessary dependencies.
+   install synapse[oidc]` to install the necessary dependencies.
 
  * For other installation mechanisms, see the documentation provided by the
    maintainer.
@@ -52,39 +52,14 @@ specific providers.
 
 Here are a few configs for providers that should work with Synapse.
 
-### Microsoft Azure Active Directory
-Azure AD can act as an OpenID Connect Provider. Register a new application under 
-*App registrations* in the Azure AD management console. The RedirectURI for your
-application should point to your matrix server: `[synapse public baseurl]/_synapse/oidc/callback`
-
-Go to *Certificates & secrets* and register a new client secret. Make note of your 
-Directory (tenant) ID as it will be used in the Azure links.
-Edit your Synapse config file and change the `oidc_config` section:
-
-```yaml
-oidc_config:
-   enabled: true
-   issuer: "https://login.microsoftonline.com/<tenant id>/v2.0"
-   client_id: "<client id>"
-   client_secret: "<client secret>"
-   scopes: ["openid", "profile"]
-   authorization_endpoint: "https://login.microsoftonline.com/<tenant id>/oauth2/v2.0/authorize"
-   token_endpoint: "https://login.microsoftonline.com/<tenant id>/oauth2/v2.0/token"
-   userinfo_endpoint: "https://graph.microsoft.com/oidc/userinfo"
-
-   user_mapping_provider:
-     config:
-       localpart_template: "{{ user.preferred_username.split('@')[0] }}"
-       display_name_template: "{{ user.name }}"
-```
-
 ### [Dex][dex-idp]
 
 [Dex][dex-idp] is a simple, open-source, certified OpenID Connect Provider.
 Although it is designed to help building a full-blown provider with an
 external database, it can be configured with static passwords in a config file.
 
-Follow the [Getting Started guide](https://dexidp.io/docs/getting-started/)
+Follow the [Getting Started
+guide](https://github.com/dexidp/dex/blob/master/Documentation/getting-started.md)
 to install Dex.
 
 Edit `examples/config-dev.yaml` config file from the Dex repo to add a client:
@@ -98,7 +73,7 @@ staticClients:
   name: 'Synapse'
 ```
 
-Run with `dex serve examples/config-dev.yaml`.
+Run with `dex serve examples/config-dex.yaml`.
 
 Synapse config:
 

docs/reverse_proxy.md

@@ -54,7 +54,7 @@ server {
         proxy_set_header X-Forwarded-For $remote_addr;
         # Nginx by default only allows file uploads up to 1M in size
         # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
-        client_max_body_size 50M;
+        client_max_body_size 10M;
     }
 }
 ```

docs/sample_config.yaml

@@ -893,7 +893,7 @@ media_store_path: "DATADIR/media_store"
 
 # The largest allowed upload size in bytes
 #
-#max_upload_size: 50M
+#max_upload_size: 10M
 
 # Maximum number of pixels that will be thumbnailed
 #
@@ -1505,8 +1505,10 @@ trusted_key_servers:
 
 ## Single sign-on integration ##
 
-# The following settings can be used to make Synapse use a single sign-on
-# provider for authentication, instead of its internal password database.
+# Enable SAML2 for registration and login. Uses pysaml2.
+#
+# At least one of `sp_config` or `config_path` must be set in this section to
+# enable SAML login.
 #
 # You will probably also want to set the following options to `false` to
 # disable the regular login/registration flows:
@@ -1515,11 +1517,6 @@ trusted_key_servers:
 #
 # You will also want to investigate the settings under the "sso" configuration
 # section below.
-
-# Enable SAML2 for registration and login. Uses pysaml2.
-#
-# At least one of `sp_config` or `config_path` must be set in this section to
-# enable SAML login.
 #
 # Once SAML support is enabled, a metadata file will be exposed at
 # https://<server>:<port>/_matrix/saml2/metadata.xml, which you may be able to
@@ -1535,42 +1532,40 @@ saml2_config:
   # so it is not normally necessary to specify them unless you need to
   # override them.
   #
-  sp_config:
-    # Point this to the IdP's metadata. You must provide either a local
-    # file via the `local` attribute or (preferably) a URL via the
-    # `remote` attribute.
-    #
-    #metadata:
-    #  local: ["saml2/idp.xml"]
-    #  remote:
-    #    - url: https://our_idp/metadata.xml
-
-    # By default, the user has to go to our login page first. If you'd like
-    # to allow IdP-initiated login, set 'allow_unsolicited: true' in a
-    # 'service.sp' section:
-    #
-    #service:
-    #  sp:
-    #    allow_unsolicited: true
-
-    # The examples below are just used to generate our metadata xml, and you
-    # may well not need them, depending on your setup. Alternatively you
-    # may need a whole lot more detail - see the pysaml2 docs!
-
-    #description: ["My awesome SP", "en"]
-    #name: ["Test SP", "en"]
-
-    #organization:
-    #  name: Example com
-    #  display_name:
-    #    - ["Example co", "en"]
-    #  url: "http://example.com"
-
-    #contact_person:
-    #  - given_name: Bob
-    #    sur_name: "the Sysadmin"
-    #    email_address": ["admin@example.com"]
-    #    contact_type": technical
+  #sp_config:
+  #  # point this to the IdP's metadata. You can use either a local file or
+  #  # (preferably) a URL.
+  #  metadata:
+  #    #local: ["saml2/idp.xml"]
+  #    remote:
+  #      - url: https://our_idp/metadata.xml
+  #
+  #  # By default, the user has to go to our login page first. If you'd like
+  #  # to allow IdP-initiated login, set 'allow_unsolicited: true' in a
+  #  # 'service.sp' section:
+  #  #
+  #  #service:
+  #  #  sp:
+  #  #    allow_unsolicited: true
+  #
+  #  # The examples below are just used to generate our metadata xml, and you
+  #  # may well not need them, depending on your setup. Alternatively you
+  #  # may need a whole lot more detail - see the pysaml2 docs!
+  #
+  #  description: ["My awesome SP", "en"]
+  #  name: ["Test SP", "en"]
+  #
+  #  organization:
+  #    name: Example com
+  #    display_name:
+  #      - ["Example co", "en"]
+  #    url: "http://example.com"
+  #
+  #  contact_person:
+  #    - given_name: Bob
+  #      sur_name: "the Sysadmin"
+  #      email_address": ["admin@example.com"]
+  #      contact_type": technical
 
   # Instead of putting the config inline as above, you can specify a
   # separate pysaml2 configuration file:
@@ -1646,10 +1641,11 @@ saml2_config:
   #    value: "sales"
 
 
-# Enable OpenID Connect (OIDC) / OAuth 2.0 for registration and login.
+# OpenID Connect integration. The following settings can be used to make Synapse
+# use an OpenID Connect Provider for authentication, instead of its internal
+# password database.
 #
-# See https://github.com/matrix-org/synapse/blob/master/docs/openid.md
-# for some example configurations.
+# See https://github.com/matrix-org/synapse/blob/master/docs/openid.md.
 #
 oidc_config:
   # Uncomment the following to enable authorization against an OpenID Connect
@@ -1782,37 +1778,15 @@ oidc_config:
 
 
 
-# Enable Central Authentication Service (CAS) for registration and login.
+# Enable CAS for registration and login.
 #
-cas_config:
-  # Uncomment the following to enable authorization against a CAS server.
-  # Defaults to false.
-  #
-  #enabled: true
-
-  # The URL of the CAS authorization endpoint.
-  #
-  #server_url: "https://cas-server.com"
-
-  # The public URL of the homeserver.
-  #
-  #service_url: "https://homeserver.domain.com:8448"
-
-  # The attribute of the CAS response to use as the display name.
-  #
-  # If unset, no displayname will be set.
-  #
-  #displayname_attribute: name
-
-  # It is possible to configure Synapse to only allow logins if CAS attributes
-  # match particular values. All of the keys in the mapping below must exist
-  # and the values must match the given value. Alternately if the given value
-  # is None then any value is allowed (the attribute just must exist).
-  # All of the listed attributes must match for the login to be permitted.
-  #
-  #required_attributes:
-  #  userGroup: "staff"
-  #  department: None
+#cas_config:
+#   enabled: true
+#   server_url: "https://cas-server.com"
+#   service_url: "https://homeserver.domain.com:8448"
+#   #displayname_attribute: name
+#   #required_attributes:
+#   #    name: value
 
 
 # Additional settings to use with single-sign on systems such as OpenID Connect,
@@ -1912,7 +1886,7 @@ sso:
 # and issued at ("iat") claims are validated if present.
 #
 # Note that this is a non-standard login type and client support is
-# expected to be non-existent.
+# expected to be non-existant.
 #
 # See https://github.com/matrix-org/synapse/blob/master/docs/jwt.md.
 #
@@ -2428,7 +2402,7 @@ spam_checker:
 #
 # Options for the rules include:
 #
-#   user_id: Matches against the creator of the alias
+#   user_id: Matches agaisnt the creator of the alias
 #   room_id: Matches against the room ID being published
 #   alias: Matches against any current local or canonical aliases
 #            associated with the room
@@ -2474,7 +2448,7 @@ opentracing:
     # This is a list of regexes which are matched against the server_name of the
     # homeserver.
     #
-    # By default, it is empty, so no servers are matched.
+    # By defult, it is empty, so no servers are matched.
     #
     #homeserver_whitelist:
     #  - ".*"

docs/sample_log_config.yaml

@@ -3,11 +3,7 @@
 # This is a YAML file containing a standard Python logging configuration
 # dictionary. See [1] for details on the valid settings.
 #
-# Synapse also supports structured logging for machine readable logs which can
-# be ingested by ELK stacks. See [2] for details.
-#
 # [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
-# [2]: https://github.com/matrix-org/synapse/blob/master/docs/structured_logging.md
 
 version: 1
 
@@ -63,7 +59,7 @@ root:
     # then write them to a file.
     #
     # Replace "buffer" with "console" to log to stderr instead. (Note that you'll
-    # also need to update the configuration for the `twisted` logger above, in
+    # also need to update the configuation for the `twisted` logger above, in
     # this case.)
     #
     handlers: [buffer]

docs/structured_logging.md

@@ -1,116 +1,11 @@
 # Structured Logging
 
-A structured logging system can be useful when your logs are destined for a
-machine to parse and process. By maintaining its machine-readable characteristics,
-it enables more efficient searching and aggregations when consumed by software
-such as the "ELK stack".
+A structured logging system can be useful when your logs are destined for a machine to parse and process. By maintaining its machine-readable characteristics, it enables more efficient searching and aggregations when consumed by software such as the "ELK stack".
 
-Synapse's structured logging system is configured via the file that Synapse's
-`log_config` config option points to. The file should include a formatter which
-uses the `synapse.logging.TerseJsonFormatter` class included with Synapse and a
-handler which uses the above formatter.
-
-There is also a `synapse.logging.JsonFormatter` option which does not include
-a timestamp in the resulting JSON. This is useful if the log ingester adds its
-own timestamp.
+Synapse's structured logging system is configured via the file that Synapse's `log_config` config option points to. The file must be YAML and contain `structured: true`. It must contain a list of "drains" (places where logs go to).
 
 A structured logging configuration looks similar to the following:
 
-```yaml
-version: 1
-
-formatters:
-    structured:
-        class: synapse.logging.TerseJsonFormatter
-
-handlers:
-    file:
-        class: logging.handlers.TimedRotatingFileHandler
-        formatter: structured
-        filename: /path/to/my/logs/homeserver.log
-        when: midnight
-        backupCount: 3  # Does not include the current log file.
-        encoding: utf8
-
-loggers:
-    synapse:
-        level: INFO
-        handlers: [remote]
-    synapse.storage.SQL:
-        level: WARNING
-```
-
-The above logging config will set Synapse as 'INFO' logging level by default,
-with the SQL layer at 'WARNING', and will log to a file, stored as JSON.
-
-It is also possible to figure Synapse to log to a remote endpoint by using the
-`synapse.logging.RemoteHandler` class included with Synapse. It takes the
-following arguments:
-
-- `host`: Hostname or IP address of the log aggregator.
-- `port`: Numerical port to contact on the host.
-- `maximum_buffer`: (Optional, defaults to 1000) The maximum buffer size to allow.
-
-A remote structured logging configuration looks similar to the following:
-
-```yaml
-version: 1
-
-formatters:
-    structured:
-        class: synapse.logging.TerseJsonFormatter
-
-handlers:
-    remote:
-        class: synapse.logging.RemoteHandler
-        formatter: structured
-        host: 10.1.2.3
-        port: 9999
-
-loggers:
-    synapse:
-        level: INFO
-        handlers: [remote]
-    synapse.storage.SQL:
-        level: WARNING
-```
-
-The above logging config will set Synapse as 'INFO' logging level by default,
-with the SQL layer at 'WARNING', and will log JSON formatted messages to a
-remote endpoint at 10.1.2.3:9999.
-
-## Upgrading from legacy structured logging configuration
-
-Versions of Synapse prior to v1.23.0 included a custom structured logging
-configuration which is deprecated. It used a `structured: true` flag and
-configured `drains` instead of ``handlers`` and `formatters`.
-
-Synapse currently automatically converts the old configuration to the new
-configuration, but this will be removed in a future version of Synapse. The
-following reference can be used to update your configuration. Based on the drain
-`type`, we can pick a new handler:
-
-1. For a type of `console`, `console_json`, or `console_json_terse`: a handler
-   with a class of `logging.StreamHandler` and a `stream` of `ext://sys.stdout`
-   or `ext://sys.stderr` should be used.
-2. For a type of `file` or `file_json`: a handler of `logging.FileHandler` with
-   a location of the file path should be used.
-3. For a type of `network_json_terse`: a handler of `synapse.logging.RemoteHandler`
-   with the host and port should be used.
-
-Then based on the drain `type` we can pick a new formatter:
-
-1. For a type of `console` or `file` no formatter is necessary.
-2. For a type of `console_json` or `file_json`: a formatter of
-   `synapse.logging.JsonFormatter` should be used.
-3. For a type of `console_json_terse` or `network_json_terse`: a formatter of
-   `synapse.logging.TerseJsonFormatter` should be used.
-
-For each new handler and formatter they should be added to the logging configuration
-and then assigned to either a logger or the root logger.
-
-An example legacy configuration:
-
 ```yaml
 structured: true
 
@@ -129,33 +24,60 @@ drains:
         location: homeserver.log
 ```
 
-Would be converted into a new configuration:
+The above logging config will set Synapse as 'INFO' logging level by default, with the SQL layer at 'WARNING', and will have two logging drains (to the console and to a file, stored as JSON).
 
-```yaml
-version: 1
+## Drain Types
 
-formatters:
-    json:
-        class: synapse.logging.JsonFormatter
+Drain types can be specified by the `type` key.
 
-handlers:
-    console:
-        class: logging.StreamHandler
-        location: ext://sys.stdout
-    file:
-        class: logging.FileHandler
-        formatter: json
-        filename: homeserver.log
+### `console`
 
-loggers:
-    synapse:
-        level: INFO
-        handlers: [console, file]
-    synapse.storage.SQL:
-        level: WARNING
-```
+Outputs human-readable logs to the console.
 
-The new logging configuration is a bit more verbose, but significantly more
-flexible. It allows for configuration that were not previously possible, such as
-sending plain logs over the network, or using different handlers for different
-modules.
+Arguments:
+
+- `location`: Either `stdout` or `stderr`.
+
+### `console_json`
+
+Outputs machine-readable JSON logs to the console.
+
+Arguments:
+
+- `location`: Either `stdout` or `stderr`.
+
+### `console_json_terse`
+
+Outputs machine-readable JSON logs to the console, separated by newlines. This
+format is not designed to be read and re-formatted into human-readable text, but
+is optimal for a logging aggregation system.
+
+Arguments:
+
+- `location`: Either `stdout` or `stderr`.
+
+### `file`
+
+Outputs human-readable logs to a file.
+
+Arguments:
+
+- `location`: An absolute path to the file to log to.
+
+### `file_json`
+
+Outputs machine-readable logs to a file.
+
+Arguments:
+
+- `location`: An absolute path to the file to log to.
+
+### `network_json_terse`
+
+Delivers machine-readable JSON logs to a log aggregator over TCP. This is
+compatible with LogStash's TCP input with the codec set to `json_lines`.
+
+Arguments:
+
+- `host`: Hostname or IP address of the log aggregator.
+- `port`: Numerical port to contact on the host.

docs/tcp_replication.md

@@ -15,7 +15,7 @@ example flow would be (where '>' indicates master to worker and
 
     > SERVER example.com
     < REPLICATE
-    > POSITION events master 53 53
+    > POSITION events master 53
     > RDATA events master 54 ["$foo1:bar.com", ...]
     > RDATA events master 55 ["$foo4:bar.com", ...]
 
@@ -138,9 +138,9 @@ the wire:
     < NAME synapse.app.appservice
     < PING 1490197665618
     < REPLICATE
-    > POSITION events master 1 1
-    > POSITION backfill master 1 1
-    > POSITION caches master 1 1
+    > POSITION events master 1
+    > POSITION backfill master 1
+    > POSITION caches master 1
     > RDATA caches master 2 ["get_user_by_id",["@01register-user:localhost:8823"],1490197670513]
     > RDATA events master 14 ["$149019767112vOHxz:localhost:8823",
         "!AFDCvgApUmpdfVjIXm:localhost:8823","m.room.guest_access","",null]
@@ -185,11 +185,6 @@ client (C):
    updates via HTTP API, rather than via the DB, then processes should make the
    request to the appropriate process.
 
-   Two positions are included, the "new" position and the last position sent respectively.
-   This allows servers to tell instances that the positions have advanced but no
-   data has been written, without clients needlessly checking to see if they
-   have missed any updates.
-
 #### ERROR (S, C)
 
    There was an error

mypy.ini

@@ -15,15 +15,9 @@ files =
   synapse/events/builder.py,
   synapse/events/spamcheck.py,
   synapse/federation,
-  synapse/handlers/_base.py,
-  synapse/handlers/account_data.py,
-  synapse/handlers/account_validity.py,
-  synapse/handlers/appservice.py,
   synapse/handlers/auth.py,
   synapse/handlers/cas_handler.py,
-  synapse/handlers/deactivate_account.py,
   synapse/handlers/device.py,
-  synapse/handlers/devicemessage.py,
   synapse/handlers/directory.py,
   synapse/handlers/events.py,
   synapse/handlers/federation.py,
@@ -32,10 +26,7 @@ files =
   synapse/handlers/message.py,
   synapse/handlers/oidc_handler.py,
   synapse/handlers/pagination.py,
-  synapse/handlers/password_policy.py,
   synapse/handlers/presence.py,
-  synapse/handlers/profile.py,
-  synapse/handlers/read_marker.py,
   synapse/handlers/room.py,
   synapse/handlers/room_member.py,
   synapse/handlers/room_member_worker.py,
@@ -57,9 +48,7 @@ files =
   synapse/server_notices,
   synapse/spam_checker_api,
   synapse/state,
-  synapse/storage/databases/main/appservice.py,
   synapse/storage/databases/main/events.py,
-  synapse/storage/databases/main/registration.py,
   synapse/storage/databases/main/stream.py,
   synapse/storage/databases/main/ui_auth.py,
   synapse/storage/database.py,
@@ -70,7 +59,8 @@ files =
   synapse/streams,
   synapse/types.py,
   synapse/util/async_helpers.py,
-  synapse/util/caches,
+  synapse/util/caches/descriptors.py,
+  synapse/util/caches/stream_change_cache.py,
   synapse/util/metrics.py,
   tests/replication,
   tests/test_utils,
@@ -83,9 +73,6 @@ ignore_missing_imports = True
 [mypy-zope]
 ignore_missing_imports = True
 
-[mypy-bcrypt]
-ignore_missing_imports = True
-
 [mypy-constantly]
 ignore_missing_imports = True