Don't mention other PR

.buildkite/merge_base_branch.sh

@@ -2,7 +2,7 @@
 
 set -ex
 
-if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs-.*|release-.*)$ ]]; then
+if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs|release-.*)$ ]]; then
     echo "Not merging forward, as this is a release branch"
     exit 0
 fi

.buildkite/pipeline.yml

@@ -31,7 +31,7 @@ steps:
       - "python -m pip install tox"
       - "scripts-dev/check-newsfragment"
     label: ":newspaper: Newsfile"
-    branches: "!master !develop !release-* !shhs-v*"
+    branches: "!master !develop !release-*"
     plugins:
       - docker#v3.0.1:
           image: "python:3.6"
@@ -47,11 +47,11 @@ steps:
 
   - wait
 
+
   - command:
       - "python -m pip install tox"
       - "tox -e py35-old,codecov"
     label: ":python: 3.5 / SQLite / Old Deps"
-    branches: "!shhs !shhs-*"
     env:
       TRIAL_FLAGS: "-j 2"
     plugins:
@@ -69,7 +69,6 @@ steps:
       - "python -m pip install tox"
       - "tox -e py35,codecov"
     label: ":python: 3.5 / SQLite"
-    branches: "!shhs !shhs-*"
     env:
       TRIAL_FLAGS: "-j 2"
     plugins:
@@ -87,7 +86,6 @@ steps:
       - "python -m pip install tox"
       - "tox -e py36,codecov"
     label: ":python: 3.6 / SQLite"
-    branches: "!shhs !shhs-*"
     env:
       TRIAL_FLAGS: "-j 2"
     plugins:
@@ -119,7 +117,6 @@ steps:
           limit: 2
 
   - label: ":python: 3.5 / :postgres: 9.5"
-    branches: "!shhs !shhs-*"
     env:
       TRIAL_FLAGS: "-j 4"
     command:
@@ -137,7 +134,6 @@ steps:
           limit: 2
 
   - label: ":python: 3.7 / :postgres: 9.5"
-    branches: "!shhs !shhs-*"
     env:
       TRIAL_FLAGS: "-j 4"
     command:
@@ -173,7 +169,6 @@ steps:
 
 
   - label: "SyTest - :python: 3.5 / SQLite / Monolith"
-    branches: "!shhs !shhs-*"
     agents:
       queue: "medium"
     command:
@@ -214,7 +209,6 @@ steps:
           limit: 2
 
   - label: "SyTest - :python: 3.5 / :postgres: 9.6 / Workers"
-    branches: "!shhs !shhs-*"
     agents:
       queue: "medium"
     env:
@@ -236,15 +230,3 @@ steps:
           limit: 2
         - exit_status: 2
           limit: 2
-
-  - wait
-
-  - label: ":docker: x86_64"
-    agents:
-      queue: "release"
-    branches: "shhs-*"
-    command:
-      - "docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.7.4 . -t matrixdotorg/synapse:${BUILDKITE_TAG}"
-      - "docker save matrixdotorg/synapse:${BUILDKITE_TAG} | gzip -9 > docker.tar.gz"
-    artifact_paths:
-      - "docker.tar.gz"

.circleci/config.yml

@@ -0,0 +1,33 @@
+version: 2
+jobs:
+  dockerhubuploadrelease:
+    machine: true
+    steps:
+      - checkout
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 .
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
+  dockerhubuploadlatest:
+    machine: true
+    steps:
+      - checkout
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest -t matrixdotorg/synapse:latest-py3 .
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      - run: docker push matrixdotorg/synapse:latest
+      - run: docker push matrixdotorg/synapse:latest-py3
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - dockerhubuploadrelease:
+          filters:
+            tags:
+              only: /v[0-9].[0-9]+.[0-9]+.*/
+            branches:
+              ignore: /.*/
+      - dockerhubuploadlatest:
+          filters:
+            branches:
+              only: master

.gitignore

@@ -19,7 +19,6 @@ _trial_temp*/
 /*.signing.key
 /env/
 /homeserver*.yaml
-/logs
 /media_store/
 /uploads
 
@@ -38,3 +37,4 @@ _trial_temp*/
 /docs/build/
 /htmlcov
 /pip-wheel-metadata/
+

CHANGES.md

@@ -1,125 +1,3 @@
-Synapse 1.2.1 (2019-07-26)
-==========================
-
-Security update
----------------
-
-This release includes *four* security fixes:
-
-- Prevent an attack where a federated server could send redactions for arbitrary events in v1 and v2 rooms. ([\#5767](https://github.com/matrix-org/synapse/issues/5767))
-- Prevent a denial-of-service attack where cycles of redaction events would make Synapse spin infinitely. Thanks to `@lrizika:matrix.org` for identifying and responsibly disclosing this issue. ([0f2ecb961](https://github.com/matrix-org/synapse/commit/0f2ecb961))
-- Prevent an attack where users could be joined or parted from public rooms without their consent. Thanks to @Dylanger for identifying and responsibly disclosing this issue. ([\#5744](https://github.com/matrix-org/synapse/issues/5744))
-- Fix a vulnerability where a federated server could spoof read-receipts from
-  users on other servers. Thanks to @Dylanger for identifying this issue too. ([\#5743](https://github.com/matrix-org/synapse/issues/5743))
-
-Additionally, the following fix was in Synapse **1.2.0**, but was not correctly
-identified during the original release:
-
-- It was possible for a room moderator to send a redaction for an `m.room.create` event, which would downgrade the room to version 1. Thanks to `/dev/ponies` for identifying and responsibly disclosing this issue! ([\#5701](https://github.com/matrix-org/synapse/issues/5701))
-
-Synapse 1.2.0 (2019-07-25)
-==========================
-
-No significant changes.
-
-
-Synapse 1.2.0rc2 (2019-07-24)
-=============================
-
-Bugfixes
---------
-
-- Fix a regression introduced in v1.2.0rc1 which led to incorrect labels on some prometheus metrics. ([\#5734](https://github.com/matrix-org/synapse/issues/5734))
-
-
-Synapse 1.2.0rc1 (2019-07-22)
-=============================
-
-Security fixes
---------------
-
-This update included a security fix which was initially incorrectly flagged as
-a regular bug fix.
-
-- It was possible for a room moderator to send a redaction for an `m.room.create` event, which would downgrade the room to version 1. Thanks to `/dev/ponies` for identifying and responsibly disclosing this issue! ([\#5701](https://github.com/matrix-org/synapse/issues/5701))
-
-Features
---------
-
-- Add support for opentracing. ([\#5544](https://github.com/matrix-org/synapse/issues/5544), [\#5712](https://github.com/matrix-org/synapse/issues/5712))
-- Add ability to pull all locally stored events out of synapse that a particular user can see. ([\#5589](https://github.com/matrix-org/synapse/issues/5589))
-- Add a basic admin command app to allow server operators to run Synapse admin commands separately from the main production instance. ([\#5597](https://github.com/matrix-org/synapse/issues/5597))
-- Add `sender` and `origin_server_ts` fields to `m.replace`. ([\#5613](https://github.com/matrix-org/synapse/issues/5613))
-- Add default push rule to ignore reactions. ([\#5623](https://github.com/matrix-org/synapse/issues/5623))
-- Include the original event when asking for its relations. ([\#5626](https://github.com/matrix-org/synapse/issues/5626))
-- Implement `session_lifetime` configuration option, after which access tokens will expire. ([\#5660](https://github.com/matrix-org/synapse/issues/5660))
-- Return "This account has been deactivated" when a deactivated user tries to login. ([\#5674](https://github.com/matrix-org/synapse/issues/5674))
-- Enable aggregations support by default ([\#5714](https://github.com/matrix-org/synapse/issues/5714))
-
-
-Bugfixes
---------
-
-- Fix 'utime went backwards' errors on daemonization. ([\#5609](https://github.com/matrix-org/synapse/issues/5609))
-- Various minor fixes to the federation request rate limiter. ([\#5621](https://github.com/matrix-org/synapse/issues/5621))
-- Forbid viewing relations on an event once it has been redacted. ([\#5629](https://github.com/matrix-org/synapse/issues/5629))
-- Fix requests to the `/store_invite` endpoint of identity servers being sent in the wrong format. ([\#5638](https://github.com/matrix-org/synapse/issues/5638))
-- Fix newly-registered users not being able to lookup their own profile without joining a room. ([\#5644](https://github.com/matrix-org/synapse/issues/5644))
-- Fix bug in #5626 that prevented the original_event field from actually having the contents of the original event in a call to `/relations`. ([\#5654](https://github.com/matrix-org/synapse/issues/5654))
-- Fix 3PID bind requests being sent to identity servers as `application/x-form-www-urlencoded` data, which is deprecated. ([\#5658](https://github.com/matrix-org/synapse/issues/5658))
-- Fix some problems with authenticating redactions in recent room versions. ([\#5699](https://github.com/matrix-org/synapse/issues/5699), [\#5700](https://github.com/matrix-org/synapse/issues/5700), [\#5707](https://github.com/matrix-org/synapse/issues/5707))
-
-
-Updates to the Docker image
----------------------------
-
-- Base Docker image on a newer Alpine Linux version (3.8 -> 3.10). ([\#5619](https://github.com/matrix-org/synapse/issues/5619))
-- Add missing space in default logging file format generated by the Docker image. ([\#5620](https://github.com/matrix-org/synapse/issues/5620))
-
-
-Improved Documentation
-----------------------
-
-- Add information about nginx normalisation to reverse_proxy.rst. Contributed by @skalarproduktraum - thanks! ([\#5397](https://github.com/matrix-org/synapse/issues/5397))
-- --no-pep517 should be --no-use-pep517 in the documentation to setup the development environment. ([\#5651](https://github.com/matrix-org/synapse/issues/5651))
-- Improvements to Postgres setup instructions. Contributed by @Lrizika - thanks! ([\#5661](https://github.com/matrix-org/synapse/issues/5661))
-- Minor tweaks to postgres documentation. ([\#5675](https://github.com/matrix-org/synapse/issues/5675))
-
-
-Deprecations and Removals
--------------------------
-
-- Remove support for the `invite_3pid_guest` configuration setting. ([\#5625](https://github.com/matrix-org/synapse/issues/5625))
-
-
-Internal Changes
-----------------
-
-- Move logging code out of `synapse.util` and into `synapse.logging`. ([\#5606](https://github.com/matrix-org/synapse/issues/5606), [\#5617](https://github.com/matrix-org/synapse/issues/5617))
-- Add a blacklist file to the repo to blacklist certain sytests from failing CI. ([\#5611](https://github.com/matrix-org/synapse/issues/5611))
-- Make runtime errors surrounding password reset emails much clearer. ([\#5616](https://github.com/matrix-org/synapse/issues/5616))
-- Remove dead code for persiting outgoing federation transactions. ([\#5622](https://github.com/matrix-org/synapse/issues/5622))
-- Add `lint.sh` to the scripts-dev folder which will run all linting steps required by CI. ([\#5627](https://github.com/matrix-org/synapse/issues/5627))
-- Move RegistrationHandler.get_or_create_user to test code. ([\#5628](https://github.com/matrix-org/synapse/issues/5628))
-- Add some more common python virtual-environment paths to the black exclusion list. ([\#5630](https://github.com/matrix-org/synapse/issues/5630))
-- Some counter metrics exposed over Prometheus have been renamed, with the old names preserved for backwards compatibility and deprecated. See `docs/metrics-howto.rst` for details. ([\#5636](https://github.com/matrix-org/synapse/issues/5636))
-- Unblacklist some user_directory sytests. ([\#5637](https://github.com/matrix-org/synapse/issues/5637))
-- Factor out some redundant code in the login implementation. ([\#5639](https://github.com/matrix-org/synapse/issues/5639))
-- Update ModuleApi to avoid register(generate_token=True). ([\#5640](https://github.com/matrix-org/synapse/issues/5640))
-- Remove access-token support from `RegistrationHandler.register`, and rename it. ([\#5641](https://github.com/matrix-org/synapse/issues/5641))
-- Remove access-token support from `RegistrationStore.register`, and rename it. ([\#5642](https://github.com/matrix-org/synapse/issues/5642))
-- Improve logging for auto-join when a new user is created. ([\#5643](https://github.com/matrix-org/synapse/issues/5643))
-- Remove unused and unnecessary check for FederationDeniedError in _exception_to_failure. ([\#5645](https://github.com/matrix-org/synapse/issues/5645))
-- Fix a small typo in a code comment. ([\#5655](https://github.com/matrix-org/synapse/issues/5655))
-- Clean up exception handling around client access tokens. ([\#5656](https://github.com/matrix-org/synapse/issues/5656))
-- Add a mechanism for per-test homeserver configuration in the unit tests. ([\#5657](https://github.com/matrix-org/synapse/issues/5657))
-- Inline issue_access_token. ([\#5659](https://github.com/matrix-org/synapse/issues/5659))
-- Update the sytest BuildKite configuration to checkout Synapse in `/src`. ([\#5664](https://github.com/matrix-org/synapse/issues/5664))
-- Add a `docker` type to the towncrier configuration. ([\#5673](https://github.com/matrix-org/synapse/issues/5673))
-- Convert `synapse.federation.transport.server` to `async`. Might improve some stack traces. ([\#5689](https://github.com/matrix-org/synapse/issues/5689))
-- Documentation for opentracing. ([\#5703](https://github.com/matrix-org/synapse/issues/5703))
-
-
 Synapse 1.1.0 (2019-07-04)
 ==========================
 

UPGRADE.rst

@@ -49,13 +49,6 @@ returned by the Client-Server API:
     # configured on port 443.
     curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"
 
-Upgrading to v1.2.0
-===================
-
-Some counter metrics have been renamed, with the old names deprecated. See
-`the metrics documentation <docs/metrics-howto.rst#renaming-of-metrics--deprecation-of-old-names-in-12>`_
-for details.
-
 Upgrading to v1.1.0
 ===================
 

changelog.d/5072.feature

@@ -1 +0,0 @@
-Synapse can now be configured to not join remote rooms of a given "complexity" (currently, state events). This option can be used to prevent adverse performance on resource-constrained homeservers.

changelog.d/5099.misc

@@ -1 +0,0 @@
-Python 2 has been removed from the CI.

changelog.d/5397.doc

@@ -0,0 +1 @@
+Add information about nginx normalisation to reverse_proxy.rst. Contributed by @skalarproduktraum - thanks!

changelog.d/5544.misc

@@ -0,0 +1 @@
+Added opentracing and configuration options.

changelog.d/5589.feature

@@ -0,0 +1 @@
+Add ability to pull all locally stored events out of synapse that a particular user can see.

changelog.d/5606.misc

@@ -0,0 +1 @@
+Move logging code out of `synapse.util` and into `synapse.logging`.

changelog.d/5609.bugfix

@@ -0,0 +1 @@
+Fix 'utime went backwards' errors on daemonization.

changelog.d/5611.misc

@@ -0,0 +1 @@
+Add a blacklist file to the repo to blacklist certain sytests from failing CI.

changelog.d/5613.feature

@@ -0,0 +1 @@
+Add `sender` and `origin_server_ts` fields to `m.replace`.

changelog.d/5616.misc

@@ -0,0 +1 @@
+Make runtime errors surrounding password reset emails much clearer.

changelog.d/5617.misc

@@ -0,0 +1 @@
+Move logging code out of `synapse.util` and into `synapse.logging`.

changelog.d/5619.docker

@@ -0,0 +1 @@
+Base Docker image on a newer Alpine Linux version (3.8 -> 3.10).

changelog.d/5620.docker

@@ -0,0 +1 @@
+Add missing space in default logging file format generated by the Docker image.

changelog.d/5621.bugfix

@@ -0,0 +1 @@
+Various minor fixes to the federation request rate limiter.

changelog.d/5622.misc

@@ -0,0 +1 @@
+Remove dead code for persiting outgoing federation transactions.

changelog.d/5623.feature

@@ -0,0 +1 @@
+Add default push rule to ignore reactions.

changelog.d/5625.removal

@@ -0,0 +1 @@
+Remove support for the `invite_3pid_guest` configuration setting.

changelog.d/5626.feature

@@ -0,0 +1 @@
+Include the original event when asking for its relations.

changelog.d/5627.misc

@@ -0,0 +1 @@
+Add `lint.sh` to the scripts-dev folder which will run all linting steps required by CI.

changelog.d/5628.misc

@@ -0,0 +1 @@
+Move RegistrationHandler.get_or_create_user to test code.

changelog.d/5630.misc

@@ -0,0 +1 @@
+Add some more common python virtual-environment paths to the black exclusion list.

changelog.d/5637.misc

@@ -0,0 +1 @@
+Unblacklist some user_directory sytests.

changelog.d/5638.bugfix

@@ -0,0 +1 @@
+Fix requests to the `/store_invite` endpoint of identity servers being sent in the wrong format.

changelog.d/5639.misc

@@ -0,0 +1 @@
+Factor out some redundant code in the login implementation.

changelog.d/5640.misc

@@ -0,0 +1 @@
+Update ModuleApi to avoid register(generate_token=True).

changelog.d/5641.misc

@@ -0,0 +1 @@
+Remove access-token support from RegistrationHandler.register, and rename it.

changelog.d/5642.misc

@@ -0,0 +1 @@
+Remove access-token support from `RegistrationStore.register`, and rename it.

changelog.d/5643.misc

@@ -0,0 +1 @@
+Improve logging for auto-join when a new user is created.

changelog.d/5644.bugfix

@@ -0,0 +1 @@
+Fix newly-registered users not being able to lookup their own profile without joining a room.

changelog.d/5645.misc

@@ -0,0 +1 @@
+Remove unused and unnecessary check for FederationDeniedError in _exception_to_failure.

changelog.d/5651.doc

@@ -0,0 +1 @@
+--no-pep517 should be --no-use-pep517 in the documentation to setup the development environment.

changelog.d/5654.bugfix

@@ -0,0 +1 @@
+Fix bug in #5626 that prevented the original_event field from actually having the contents of the original event in a call to `/relations`.

changelog.d/5655.misc

@@ -0,0 +1 @@
+Fix a small typo in a code comment.

changelog.d/5656.misc

@@ -0,0 +1 @@
+Clean up exception handling around client access tokens.

changelog.d/5657.misc

@@ -0,0 +1 @@
+Add a mechanism for per-test homeserver configuration in the unit tests.

changelog.d/5658.bugfix

@@ -0,0 +1 @@
+Fix 3PID bind requests being sent to identity servers as `application/x-form-www-urlencoded` data, which is deprecated.

changelog.d/5659.misc

@@ -0,0 +1 @@
+Inline issue_access_token.

changelog.d/5660.feature

@@ -0,0 +1 @@
+Implement `session_lifetime` configuration option, after which access tokens will expire.

changelog.d/5661.doc

@@ -0,0 +1 @@
+Improvements to Postgres setup instructions. Contributed by @Lrizika - thanks!

changelog.d/5664.misc

@@ -0,0 +1 @@
+Update the sytest BuildKite configuration to checkout Synapse in `/src`.

changelog.d/5673.misc

@@ -0,0 +1 @@
+Add a `docker` type to the towncrier configuration.

changelog.d/5674.feature

@@ -0,0 +1 @@
+Return "This account has been deactivated" when a deactivated user tries to login.

changelog.d/5686.feature

@@ -0,0 +1 @@
+Use `M_USER_DEACTIVATED` instead of `M_UNKNOWN` for errcode when a deactivated user attempts to login.

debian/changelog

@@ -1,21 +1,9 @@
-matrix-synapse-py3 (1.2.1) stable; urgency=medium
-
-  * New synapse release 1.2.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 26 Jul 2019 11:32:47 +0100
-
-matrix-synapse-py3 (1.2.0) stable; urgency=medium
+matrix-synapse-py3 (1.1.0-1) UNRELEASED; urgency=medium
 
   [ Amber Brown ]
   * Update logging config defaults to match API changes in Synapse.
 
-  [ Richard van der Hoff ]
-  * Add Recommends and Depends for some libraries which you probably want.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.2.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 25 Jul 2019 14:10:07 +0100
+ -- Erik Johnston <erikj@rae>  Thu, 04 Jul 2019 13:59:02 +0100
 
 matrix-synapse-py3 (1.1.0) stable; urgency=medium
 

debian/control

@@ -2,20 +2,16 @@ Source: matrix-synapse-py3
 Section: contrib/python
 Priority: extra
 Maintainer: Synapse Packaging team <packages@matrix.org>
-# keep this list in sync with the build dependencies in docker/Dockerfile-dhvirtualenv.
 Build-Depends:
  debhelper (>= 9),
  dh-systemd,
  dh-virtualenv (>= 1.1),
- libsystemd-dev,
- libpq-dev,
  lsb-release,
  python3-dev,
  python3,
  python3-setuptools,
  python3-pip,
  python3-venv,
- libsqlite3-dev,
  tar,
 Standards-Version: 3.9.8
 Homepage: https://github.com/matrix-org/synapse
@@ -32,12 +28,9 @@ Depends:
  debconf,
  python3-distutils|libpython3-stdlib (<< 3.6),
  ${misc:Depends},
- ${shlibs:Depends},
  ${synapse:pydepends},
 # some of our scripts use perl, but none of them are important,
 # so we put perl:Depends in Suggests rather than Depends.
-Recommends:
- ${shlibs1:Recommends},
 Suggests:
  sqlite3,
  ${perl:Depends},

debian/rules

@@ -3,29 +3,15 @@
 # Build Debian package using https://github.com/spotify/dh-virtualenv
 #
 
-# assume we only have one package
-PACKAGE_NAME:=`dh_listpackages`
-
 override_dh_systemd_enable:
 	dh_systemd_enable --name=matrix-synapse
 
 override_dh_installinit:
 	dh_installinit --name=matrix-synapse
 
-# we don't really want to strip the symbols from our object files.
 override_dh_strip:
 
 override_dh_shlibdeps:
-        # make the postgres package's dependencies a recommendation
-        # rather than a hard dependency.
-	find debian/$(PACKAGE_NAME)/ -path '*/site-packages/psycopg2/*.so' | \
-	    xargs dpkg-shlibdeps -Tdebian/$(PACKAGE_NAME).substvars \
-	        -pshlibs1 -dRecommends
-
-        # all the other dependencies can be normal 'Depends' requirements,
-        # except for PIL's, which is self-contained and which confuses
-        # dpkg-shlibdeps.
-	dh_shlibdeps -X site-packages/PIL/.libs -X site-packages/psycopg2
 
 override_dh_virtualenv:
 	./debian/build_virtualenv

docker/Dockerfile-dhvirtualenv

@@ -43,9 +43,6 @@ RUN cd dh-virtualenv-1.1 && dpkg-buildpackage -us -uc -b
 FROM ${distro}
 
 # Install the build dependencies
-#
-# NB: keep this list in sync with the list of build-deps in debian/control
-# TODO: it would be nice to do that automatically.
 RUN apt-get update -qq -o Acquire::Languages=none \
     && env DEBIAN_FRONTEND=noninteractive apt-get install \
         -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \

docs/metrics-howto.rst

@@ -59,108 +59,6 @@ How to monitor Synapse metrics using Prometheus
    Restart Prometheus.
 
 
-Renaming of metrics & deprecation of old names in 1.2
------------------------------------------------------
-
-Synapse 1.2 updates the Prometheus metrics to match the naming convention of the
-upstream ``prometheus_client``. The old names are considered deprecated and will
-be removed in a future version of Synapse.
-
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-|                                  New Name                                   |                               Old Name                                |
-+=============================================================================+=======================================================================+
-| python_gc_objects_collected_total                                           | python_gc_objects_collected                                           |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| python_gc_objects_uncollectable_total                                       | python_gc_objects_uncollectable                                       |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| python_gc_collections_total                                                 | python_gc_collections                                                 |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| process_cpu_seconds_total                                                   | process_cpu_seconds                                                   |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_federation_client_sent_transactions_total                           | synapse_federation_client_sent_transactions                           |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_federation_client_events_processed_total                            | synapse_federation_client_events_processed                            |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_event_processing_loop_count_total                                   | synapse_event_processing_loop_count                                   |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_event_processing_loop_room_count_total                              | synapse_event_processing_loop_room_count                              |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_util_metrics_block_count_total                                      | synapse_util_metrics_block_count                                      |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_util_metrics_block_time_seconds_total                               | synapse_util_metrics_block_time_seconds                               |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_util_metrics_block_ru_utime_seconds_total                           | synapse_util_metrics_block_ru_utime_seconds                           |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_util_metrics_block_ru_stime_seconds_total                           | synapse_util_metrics_block_ru_stime_seconds                           |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_util_metrics_block_db_txn_count_total                               | synapse_util_metrics_block_db_txn_count                               |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_util_metrics_block_db_txn_duration_seconds_total                    | synapse_util_metrics_block_db_txn_duration_seconds                    |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_util_metrics_block_db_sched_duration_seconds_total                  | synapse_util_metrics_block_db_sched_duration_seconds                  |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_background_process_start_count_total                                | synapse_background_process_start_count                                |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_background_process_ru_utime_seconds_total                           | synapse_background_process_ru_utime_seconds                           |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_background_process_ru_stime_seconds_total                           | synapse_background_process_ru_stime_seconds                           |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_background_process_db_txn_count_total                               | synapse_background_process_db_txn_count                               |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_background_process_db_txn_duration_seconds_total                    | synapse_background_process_db_txn_duration_seconds                    |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_background_process_db_sched_duration_seconds_total                  | synapse_background_process_db_sched_duration_seconds                  |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_storage_events_persisted_events_total                               | synapse_storage_events_persisted_events                               |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_storage_events_persisted_events_sep_total                           | synapse_storage_events_persisted_events_sep                           |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_storage_events_state_delta_total                                    | synapse_storage_events_state_delta                                    |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_storage_events_state_delta_single_event_total                       | synapse_storage_events_state_delta_single_event                       |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_storage_events_state_delta_reuse_delta_total                        | synapse_storage_events_state_delta_reuse_delta                        |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_federation_server_received_pdus_total                               | synapse_federation_server_received_pdus                               |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_federation_server_received_edus_total                               | synapse_federation_server_received_edus                               |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_handler_presence_notified_presence_total                            | synapse_handler_presence_notified_presence                            |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_handler_presence_federation_presence_out_total                      | synapse_handler_presence_federation_presence_out                      |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_handler_presence_presence_updates_total                             | synapse_handler_presence_presence_updates                             |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_handler_presence_timers_fired_total                                 | synapse_handler_presence_timers_fired                                 |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_handler_presence_federation_presence_total                          | synapse_handler_presence_federation_presence                          |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_handler_presence_bump_active_time_total                             | synapse_handler_presence_bump_active_time                             |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_federation_client_sent_edus_total                                   | synapse_federation_client_sent_edus                                   |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_federation_client_sent_pdu_destinations_count_total                 | synapse_federation_client_sent_pdu_destinations:count                 |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_federation_client_sent_pdu_destinations_total                       | synapse_federation_client_sent_pdu_destinations:total                 |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_handlers_appservice_events_processed_total                          | synapse_handlers_appservice_events_processed                          |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_notifier_notified_events_total                                      | synapse_notifier_notified_events                                      |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total | synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total   | synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter   |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_http_httppusher_http_pushes_processed_total                         | synapse_http_httppusher_http_pushes_processed                         |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_http_httppusher_http_pushes_failed_total                            | synapse_http_httppusher_http_pushes_failed                            |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_http_httppusher_badge_updates_processed_total                       | synapse_http_httppusher_badge_updates_processed                       |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_http_httppusher_badge_updates_failed_total                          | synapse_http_httppusher_badge_updates_failed                          |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-
-
 Removal of deprecated metrics & time based counters becoming histograms in 0.31.0
 ---------------------------------------------------------------------------------
 

docs/opentracing.rst

@@ -1,100 +0,0 @@
-===========
-OpenTracing
-===========
-
-Background
-----------
-
-OpenTracing is a semi-standard being adopted by a number of distributed tracing
-platforms. It is a common api for facilitating vendor-agnostic tracing
-instrumentation. That is, we can use the OpenTracing api and select one of a
-number of tracer implementations to do the heavy lifting in the background.
-Our current selected implementation is Jaeger.
-
-OpenTracing is a tool which gives an insight into the causal relationship of
-work done in and between servers. The servers each track events and report them
-to a centralised server - in Synapse's case: Jaeger. The basic unit used to
-represent events is the span. The span roughly represents a single piece of work
-that was done and the time at which it occurred. A span can have child spans,
-meaning that the work of the child had to be completed for the parent span to
-complete, or it can have follow-on spans which represent work that is undertaken
-as a result of the parent but is not depended on by the parent to in order to
-finish.
-
-Since this is undertaken in a distributed environment a request to another
-server, such as an RPC or a simple GET, can be considered a span (a unit or
-work) for the local server. This causal link is what OpenTracing aims to
-capture and visualise. In order to do this metadata about the local server's
-span, i.e the 'span context', needs to be included with the request to the
-remote.
-
-It is up to the remote server to decide what it does with the spans
-it creates. This is called the sampling policy and it can be configured
-through Jaeger's settings.
-
-For OpenTracing concepts see 
-https://opentracing.io/docs/overview/what-is-tracing/.
-
-For more information about Jaeger's implementation see
-https://www.jaegertracing.io/docs/
-
-=====================
-Seting up OpenTracing
-=====================
-
-To receive OpenTracing spans, start up a Jaeger server. This can be done
-using docker like so:
-
-.. code-block:: bash
-
-   docker run -d --name jaeger
-     -p 6831:6831/udp \
-     -p 6832:6832/udp \
-     -p 5778:5778 \
-     -p 16686:16686 \
-     -p 14268:14268 \
-     jaegertracing/all-in-one:1.13
-
-Latest documentation is probably at
-https://www.jaegertracing.io/docs/1.13/getting-started/
-
-
-Enable OpenTracing in Synapse
------------------------------
-
-OpenTracing is not enabled by default. It must be enabled in the homeserver
-config by uncommenting the config options under ``opentracing`` as shown in
-the `sample config <./sample_config.yaml>`_. For example:
-
-.. code-block:: yaml
-
-  opentracing:
-    tracer_enabled: true
-    homeserver_whitelist:
-      - "mytrustedhomeserver.org"
-      - "*.myotherhomeservers.com"
-
-Homeserver whitelisting
------------------------
-
-The homeserver whitelist is configured using regular expressions. A list of regular
-expressions can be given and their union will be compared when propagating any
-spans contexts to another homeserver. 
-
-Though it's mostly safe to send and receive span contexts to and from
-untrusted users since span contexts are usually opaque ids it can lead to
-two problems, namely:
-
-- If the span context is marked as sampled by the sending homeserver the receiver will
-  sample it. Therefore two homeservers with wildly different sampling policies
-  could incur higher sampling counts than intended.
-- Sending servers can attach arbitrary data to spans, known as 'baggage'. For safety this has been disabled in Synapse
-  but that doesn't prevent another server sending you baggage which will be logged
-  to OpenTracing's logs.
-
-==================
-Configuring Jaeger
-==================
-
-Sampling strategies can be set as in this document:
-https://www.jaegertracing.io/docs/1.13/sampling/

docs/postgres.rst

@@ -11,9 +11,7 @@ a postgres database.
 
 * If you are using the `matrix.org debian/ubuntu
   packages <../INSTALL.md#matrixorg-packages>`_,
-  the necessary python library will already be installed, but you will need to
-  ensure the low-level postgres library is installed, which you can do with
-  ``apt install libpq5``.
+  the necessary libraries will already be installed.
 
 * For other pre-built packages, please consult the documentation from the
   relevant package.
@@ -36,7 +34,7 @@ Assuming your PostgreSQL database user is called ``postgres``, create a user
    su - postgres
    createuser --pwprompt synapse_user
 
-Before you can authenticate with the ``synapse_user``, you must create a
+Before you can authenticate with the ``synapse_user``, you must create a 
 database that it can access. To create a database, first connect to the database
 with your database user::
 
@@ -55,7 +53,7 @@ and then run::
 This would create an appropriate database named ``synapse`` owned by the
 ``synapse_user`` user (which must already have been created as above).
 
-Note that the PostgreSQL database *must* have the correct encoding set (as
+Note that the PostgreSQL database *must* have the correct encoding set (as 
 shown above), otherwise it will not be able to store UTF8 strings.
 
 You may need to enable password authentication so ``synapse_user`` can connect

docs/sample_config.yaml

@@ -278,17 +278,6 @@ listeners:
 # Used by phonehome stats to group together related servers.
 #server_context: context
 
-# Resource-constrained Homeserver Settings
-#
-# If limit_large_remote_room_joins is True, the room complexity will be
-# checked before a user joins a new remote room. If it is above
-# limit_large_remote_room_complexity, it will disallow joining or
-# instantly leave.
-#
-# Uncomment the below lines to enable:
-#limit_large_remote_room_joins: True
-#limit_large_remote_room_complexity: 1.0
-
 # Whether to require a user to be in the room to add an alias to it.
 # Defaults to 'true'.
 #
@@ -1420,24 +1409,17 @@ password_config:
 
 
 ## Opentracing ##
+# These settings enable opentracing which implements distributed tracing
+# This allows you to observe the causal chain of events across servers
+# including requests, key lookups etc. across any server running
+# synapse or any other other services which supports opentracing.
+# (specifically those implemented with jaeger)
 
-# These settings enable opentracing, which implements distributed tracing.
-# This allows you to observe the causal chains of events across servers
-# including requests, key lookups etc., across any server running
-# synapse or any other other services which supports opentracing
-# (specifically those implemented with Jaeger).
-#
-opentracing:
-    # tracing is disabled by default. Uncomment the following line to enable it.
-    #
-    #enabled: true
-
-    # The list of homeservers we wish to send and receive span contexts and span baggage.
-    # See docs/opentracing.rst
-    # This is a list of regexes which are matched against the server_name of the
-    # homeserver.
-    #
-    # By defult, it is empty, so no servers are matched.
-    #
-    #homeserver_whitelist:
-    #  - ".*"
+#opentracing:
+#  # Enable / disable tracer
+#  tracer_enabled: false
+#  # The list of homeservers we wish to expose our current traces to.
+#  # The list is a list of regexes which are matched against the
+#  # servername of the homeserver
+#  homeserver_whitelist:
+#    - ".*"

synapse/__init__.py

@@ -35,4 +35,4 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.2.1"
+__version__ = "1.1.0"

synapse/api/auth.py

@@ -606,6 +606,21 @@ class Auth(object):
 
         defer.returnValue(auth_ids)
 
+    def check_redaction(self, room_version, event, auth_events):
+        """Check whether the event sender is allowed to redact the target event.
+
+        Returns:
+            True if the the sender is allowed to redact the target event if the
+                target event was created by them.
+            False if the sender is allowed to redact the target event with no
+                further checks.
+
+        Raises:
+            AuthError if the event sender is definitely not allowed to redact
+                the target event.
+        """
+        return event_auth.check_redaction(room_version, event, auth_events)
+
     @defer.inlineCallbacks
     def check_can_change_room_list(self, room_id, user):
         """Check if the user is allowed to edit the room's entry in the

synapse/api/errors.py

@@ -61,6 +61,7 @@ class Codes(object):
     INCOMPATIBLE_ROOM_VERSION = "M_INCOMPATIBLE_ROOM_VERSION"
     WRONG_ROOM_KEYS_VERSION = "M_WRONG_ROOM_KEYS_VERSION"
     EXPIRED_ACCOUNT = "ORG_MATRIX_EXPIRED_ACCOUNT"
+    USER_DEACTIVATED = "M_USER_DEACTIVATED"
 
 
 class CodeMessageException(RuntimeError):
@@ -151,7 +152,7 @@ class UserDeactivatedError(SynapseError):
             msg (str): The human-readable error message
         """
         super(UserDeactivatedError, self).__init__(
-            code=http_client.FORBIDDEN, msg=msg, errcode=Codes.UNKNOWN
+            code=http_client.FORBIDDEN, msg=msg, errcode=Codes.USER_DEACTIVATED
         )
 
 

synapse/app/_base.py

@@ -48,7 +48,7 @@ def register_sighup(func):
     _sighup_callbacks.append(func)
 
 
-def start_worker_reactor(appname, config, run_command=reactor.run):
+def start_worker_reactor(appname, config):
     """ Run the reactor in the main process
 
     Daemonizes if necessary, and then configures some resources, before starting
@@ -57,7 +57,6 @@ def start_worker_reactor(appname, config, run_command=reactor.run):
     Args:
         appname (str): application name which will be sent to syslog
         config (synapse.config.Config): config object
-        run_command (Callable[]): callable that actually runs the reactor
     """
 
     logger = logging.getLogger(config.worker_app)
@@ -70,19 +69,11 @@ def start_worker_reactor(appname, config, run_command=reactor.run):
         daemonize=config.worker_daemonize,
         print_pidfile=config.print_pidfile,
         logger=logger,
-        run_command=run_command,
     )
 
 
 def start_reactor(
-    appname,
-    soft_file_limit,
-    gc_thresholds,
-    pid_file,
-    daemonize,
-    print_pidfile,
-    logger,
-    run_command=reactor.run,
+    appname, soft_file_limit, gc_thresholds, pid_file, daemonize, print_pidfile, logger
 ):
     """ Run the reactor in the main process
 
@@ -97,7 +88,6 @@ def start_reactor(
         daemonize (bool): true to run the reactor in a background process
         print_pidfile (bool): whether to print the pid file, if daemonize is True
         logger (logging.Logger): logger instance to pass to Daemonize
-        run_command (Callable[]): callable that actually runs the reactor
     """
 
     install_dns_limiter(reactor)
@@ -107,7 +97,7 @@ def start_reactor(
         change_resource_limit(soft_file_limit)
         if gc_thresholds:
             gc.set_threshold(*gc_thresholds)
-        run_command()
+        reactor.run()
 
     # make sure that we run the reactor with the sentinel log context,
     # otherwise other PreserveLoggingContext instances will get confused
@@ -149,7 +139,8 @@ def listen_metrics(bind_addresses, port):
     """
     Start Prometheus metrics server.
     """
-    from synapse.metrics import RegistryProxy, start_http_server
+    from synapse.metrics import RegistryProxy
+    from prometheus_client import start_http_server
 
     for host in bind_addresses:
         logger.info("Starting metrics listener on %s:%d", host, port)

synapse/app/admin_cmd.py

@@ -1,264 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright 2019 Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import argparse
-import logging
-import os
-import sys
-import tempfile
-
-from canonicaljson import json
-
-from twisted.internet import defer, task
-
-import synapse
-from synapse.app import _base
-from synapse.config._base import ConfigError
-from synapse.config.homeserver import HomeServerConfig
-from synapse.config.logger import setup_logging
-from synapse.handlers.admin import ExfiltrationWriter
-from synapse.replication.slave.storage._base import BaseSlavedStore
-from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
-from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
-from synapse.replication.slave.storage.client_ips import SlavedClientIpStore
-from synapse.replication.slave.storage.deviceinbox import SlavedDeviceInboxStore
-from synapse.replication.slave.storage.devices import SlavedDeviceStore
-from synapse.replication.slave.storage.events import SlavedEventStore
-from synapse.replication.slave.storage.filtering import SlavedFilteringStore
-from synapse.replication.slave.storage.groups import SlavedGroupServerStore
-from synapse.replication.slave.storage.presence import SlavedPresenceStore
-from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
-from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
-from synapse.replication.slave.storage.registration import SlavedRegistrationStore
-from synapse.replication.slave.storage.room import RoomStore
-from synapse.replication.tcp.client import ReplicationClientHandler
-from synapse.server import HomeServer
-from synapse.storage.engines import create_engine
-from synapse.util.logcontext import LoggingContext
-from synapse.util.versionstring import get_version_string
-
-logger = logging.getLogger("synapse.app.admin_cmd")
-
-
-class AdminCmdSlavedStore(
-    SlavedReceiptsStore,
-    SlavedAccountDataStore,
-    SlavedApplicationServiceStore,
-    SlavedRegistrationStore,
-    SlavedFilteringStore,
-    SlavedPresenceStore,
-    SlavedGroupServerStore,
-    SlavedDeviceInboxStore,
-    SlavedDeviceStore,
-    SlavedPushRuleStore,
-    SlavedEventStore,
-    SlavedClientIpStore,
-    RoomStore,
-    BaseSlavedStore,
-):
-    pass
-
-
-class AdminCmdServer(HomeServer):
-    DATASTORE_CLASS = AdminCmdSlavedStore
-
-    def _listen_http(self, listener_config):
-        pass
-
-    def start_listening(self, listeners):
-        pass
-
-    def build_tcp_replication(self):
-        return AdminCmdReplicationHandler(self)
-
-
-class AdminCmdReplicationHandler(ReplicationClientHandler):
-    @defer.inlineCallbacks
-    def on_rdata(self, stream_name, token, rows):
-        pass
-
-    def get_streams_to_replicate(self):
-        return {}
-
-
-@defer.inlineCallbacks
-def export_data_command(hs, args):
-    """Export data for a user.
-
-    Args:
-        hs (HomeServer)
-        args (argparse.Namespace)
-    """
-
-    user_id = args.user_id
-    directory = args.output_directory
-
-    res = yield hs.get_handlers().admin_handler.export_user_data(
-        user_id, FileExfiltrationWriter(user_id, directory=directory)
-    )
-    print(res)
-
-
-class FileExfiltrationWriter(ExfiltrationWriter):
-    """An ExfiltrationWriter that writes the users data to a directory.
-    Returns the directory location on completion.
-
-    Note: This writes to disk on the main reactor thread.
-
-    Args:
-        user_id (str): The user whose data is being exfiltrated.
-        directory (str|None): The directory to write the data to, if None then
-            will write to a temporary directory.
-    """
-
-    def __init__(self, user_id, directory=None):
-        self.user_id = user_id
-
-        if directory:
-            self.base_directory = directory
-        else:
-            self.base_directory = tempfile.mkdtemp(
-                prefix="synapse-exfiltrate__%s__" % (user_id,)
-            )
-
-        os.makedirs(self.base_directory, exist_ok=True)
-        if list(os.listdir(self.base_directory)):
-            raise Exception("Directory must be empty")
-
-    def write_events(self, room_id, events):
-        room_directory = os.path.join(self.base_directory, "rooms", room_id)
-        os.makedirs(room_directory, exist_ok=True)
-        events_file = os.path.join(room_directory, "events")
-
-        with open(events_file, "a") as f:
-            for event in events:
-                print(json.dumps(event.get_pdu_json()), file=f)
-
-    def write_state(self, room_id, event_id, state):
-        room_directory = os.path.join(self.base_directory, "rooms", room_id)
-        state_directory = os.path.join(room_directory, "state")
-        os.makedirs(state_directory, exist_ok=True)
-
-        event_file = os.path.join(state_directory, event_id)
-
-        with open(event_file, "a") as f:
-            for event in state.values():
-                print(json.dumps(event.get_pdu_json()), file=f)
-
-    def write_invite(self, room_id, event, state):
-        self.write_events(room_id, [event])
-
-        # We write the invite state somewhere else as they aren't full events
-        # and are only a subset of the state at the event.
-        room_directory = os.path.join(self.base_directory, "rooms", room_id)
-        os.makedirs(room_directory, exist_ok=True)
-
-        invite_state = os.path.join(room_directory, "invite_state")
-
-        with open(invite_state, "a") as f:
-            for event in state.values():
-                print(json.dumps(event), file=f)
-
-    def finished(self):
-        return self.base_directory
-
-
-def start(config_options):
-    parser = argparse.ArgumentParser(description="Synapse Admin Command")
-    HomeServerConfig.add_arguments_to_parser(parser)
-
-    subparser = parser.add_subparsers(
-        title="Admin Commands",
-        required=True,
-        dest="command",
-        metavar="<admin_command>",
-        help="The admin command to perform.",
-    )
-    export_data_parser = subparser.add_parser(
-        "export-data", help="Export all data for a user"
-    )
-    export_data_parser.add_argument("user_id", help="User to extra data from")
-    export_data_parser.add_argument(
-        "--output-directory",
-        action="store",
-        metavar="DIRECTORY",
-        required=False,
-        help="The directory to store the exported data in. Must be empty. Defaults"
-        " to creating a temp directory.",
-    )
-    export_data_parser.set_defaults(func=export_data_command)
-
-    try:
-        config, args = HomeServerConfig.load_config_with_parser(parser, config_options)
-    except ConfigError as e:
-        sys.stderr.write("\n" + str(e) + "\n")
-        sys.exit(1)
-
-    if config.worker_app is not None:
-        assert config.worker_app == "synapse.app.admin_cmd"
-
-    # Update the config with some basic overrides so that don't have to specify
-    # a full worker config.
-    config.worker_app = "synapse.app.admin_cmd"
-
-    if (
-        not config.worker_daemonize
-        and not config.worker_log_file
-        and not config.worker_log_config
-    ):
-        # Since we're meant to be run as a "command" let's not redirect stdio
-        # unless we've actually set log config.
-        config.no_redirect_stdio = True
-
-    # Explicitly disable background processes
-    config.update_user_directory = False
-    config.start_pushers = False
-    config.send_federation = False
-
-    setup_logging(config, use_worker_options=True)
-
-    synapse.events.USE_FROZEN_DICTS = config.use_frozen_dicts
-
-    database_engine = create_engine(config.database_config)
-
-    ss = AdminCmdServer(
-        config.server_name,
-        db_config=config.database_config,
-        config=config,
-        version_string="Synapse/" + get_version_string(synapse),
-        database_engine=database_engine,
-    )
-
-    ss.setup()
-
-    # We use task.react as the basic run command as it correctly handles tearing
-    # down the reactor when the deferreds resolve and setting the return value.
-    # We also make sure that `_base.start` gets run before we actually run the
-    # command.
-
-    @defer.inlineCallbacks
-    def run(_reactor):
-        with LoggingContext("command"):
-            yield _base.start(ss, [])
-            yield args.func(ss, args)
-
-    _base.start_worker_reactor(
-        "synapse-admin-cmd", config, run_command=lambda: task.react(run)
-    )
-
-
-if __name__ == "__main__":
-    with LoggingContext("main"):
-        start(sys.argv[1:])

synapse/app/appservice.py

@@ -27,7 +27,8 @@ from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext, run_in_background
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
 from synapse.replication.slave.storage.directory import DirectoryStore
 from synapse.replication.slave.storage.events import SlavedEventStore

synapse/app/client_reader.py

@@ -28,7 +28,8 @@ from synapse.config.logger import setup_logging
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore

synapse/app/event_creator.py

@@ -28,7 +28,8 @@ from synapse.config.logger import setup_logging
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore

synapse/app/federation_reader.py

@@ -29,7 +29,8 @@ from synapse.config.logger import setup_logging
 from synapse.federation.transport.server import TransportLayerServer
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore

synapse/app/federation_sender.py

@@ -28,8 +28,9 @@ from synapse.config.logger import setup_logging
 from synapse.federation import send_queue
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext, run_in_background
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
 from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage.deviceinbox import SlavedDeviceInboxStore
 from synapse.replication.slave.storage.devices import SlavedDeviceStore
 from synapse.replication.slave.storage.events import SlavedEventStore

synapse/app/frontend_proxy.py

@@ -30,7 +30,8 @@ from synapse.http.server import JsonResource
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
 from synapse.replication.slave.storage.client_ips import SlavedClientIpStore

synapse/app/homeserver.py

@@ -55,8 +55,9 @@ from synapse.http.additional_resource import AdditionalResource
 from synapse.http.server import RootRedirect
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
 from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.module_api import ModuleApi
 from synapse.python_dependencies import check_requirements
 from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource

synapse/app/media_repository.py

@@ -28,7 +28,8 @@ from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
 from synapse.replication.slave.storage.client_ips import SlavedClientIpStore

synapse/app/pusher.py

@@ -27,7 +27,8 @@ from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext, run_in_background
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import __func__
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.events import SlavedEventStore

synapse/app/synchrotron.py

@@ -32,7 +32,8 @@ from synapse.handlers.presence import PresenceHandler, get_interested_parties
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext, run_in_background
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore, __func__
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore

synapse/app/user_dir.py

@@ -29,7 +29,8 @@ from synapse.config.logger import setup_logging
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext, run_in_background
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
 from synapse.replication.slave.storage.client_ips import SlavedClientIpStore

synapse/config/_base.py

@@ -137,42 +137,12 @@ class Config(object):
             return file_stream.read()
 
     def invoke_all(self, name, *args, **kargs):
-        """Invoke all instance methods with the given name and arguments in the
-        class's MRO.
-
-        Args:
-            name (str): Name of function to invoke
-            *args
-            **kwargs
-
-        Returns:
-            list: The list of the return values from each method called
-        """
         results = []
         for cls in type(self).mro():
             if name in cls.__dict__:
                 results.append(getattr(cls, name)(self, *args, **kargs))
         return results
 
-    @classmethod
-    def invoke_all_static(cls, name, *args, **kargs):
-        """Invoke all static methods with the given name and arguments in the
-        class's MRO.
-
-        Args:
-            name (str): Name of function to invoke
-            *args
-            **kwargs
-
-        Returns:
-            list: The list of the return values from each method called
-        """
-        results = []
-        for c in cls.mro():
-            if name in c.__dict__:
-                results.append(getattr(c, name)(*args, **kargs))
-        return results
-
     def generate_config(
         self,
         config_dir_path,
@@ -232,23 +202,6 @@ class Config(object):
         Returns: Config object.
         """
         config_parser = argparse.ArgumentParser(description=description)
-        cls.add_arguments_to_parser(config_parser)
-        obj, _ = cls.load_config_with_parser(config_parser, argv)
-
-        return obj
-
-    @classmethod
-    def add_arguments_to_parser(cls, config_parser):
-        """Adds all the config flags to an ArgumentParser.
-
-        Doesn't support config-file-generation: used by the worker apps.
-
-        Used for workers where we want to add extra flags/subcommands.
-
-        Args:
-            config_parser (ArgumentParser): App description
-        """
-
         config_parser.add_argument(
             "-c",
             "--config-path",
@@ -266,34 +219,16 @@ class Config(object):
             " Defaults to the directory containing the last config file",
         )
 
-        cls.invoke_all_static("add_arguments", config_parser)
-
-    @classmethod
-    def load_config_with_parser(cls, parser, argv):
-        """Parse the commandline and config files with the given parser
-
-        Doesn't support config-file-generation: used by the worker apps.
-
-        Used for workers where we want to add extra flags/subcommands.
-
-        Args:
-            parser (ArgumentParser)
-            argv (list[str])
-
-        Returns:
-            tuple[HomeServerConfig, argparse.Namespace]: Returns the parsed
-            config object and the parsed argparse.Namespace object from
-            `parser.parse_args(..)`
-        """
-
         obj = cls()
 
-        config_args = parser.parse_args(argv)
+        obj.invoke_all("add_arguments", config_parser)
+
+        config_args = config_parser.parse_args(argv)
 
         config_files = find_config_files(search_paths=config_args.config_path)
 
         if not config_files:
-            parser.error("Must supply a config file.")
+            config_parser.error("Must supply a config file.")
 
         if config_args.keys_directory:
             config_dir_path = config_args.keys_directory
@@ -309,7 +244,7 @@ class Config(object):
 
         obj.invoke_all("read_arguments", config_args)
 
-        return obj, config_args
+        return obj
 
     @classmethod
     def load_or_generate_config(cls, description, argv):
@@ -466,7 +401,7 @@ class Config(object):
             formatter_class=argparse.RawDescriptionHelpFormatter,
         )
 
-        obj.invoke_all_static("add_arguments", parser)
+        obj.invoke_all("add_arguments", parser)
         args = parser.parse_args(remaining_args)
 
         config_dict = read_config_files(config_files)

synapse/config/database.py

@@ -69,8 +69,7 @@ class DatabaseConfig(Config):
             if database_path is not None:
                 self.database_config["args"]["database"] = database_path
 
-    @staticmethod
-    def add_arguments(parser):
+    def add_arguments(self, parser):
         db_group = parser.add_argument_group("database")
         db_group.add_argument(
             "-d",

synapse/config/logger.py

@@ -103,8 +103,7 @@ class LoggingConfig(Config):
         if args.log_file is not None:
             self.log_file = args.log_file
 
-    @staticmethod
-    def add_arguments(parser):
+    def add_arguments(cls, parser):
         logging_group = parser.add_argument_group("logging")
         logging_group.add_argument(
             "-v",

synapse/config/registration.py

@@ -237,8 +237,7 @@ class RegistrationConfig(Config):
             % locals()
         )
 
-    @staticmethod
-    def add_arguments(parser):
+    def add_arguments(self, parser):
         reg_group = parser.add_argument_group("registration")
         reg_group.add_argument(
             "--enable-registration",

synapse/config/repository.py

@@ -87,22 +87,13 @@ def parse_thumbnail_requirements(thumbnail_sizes):
 
 class ContentRepositoryConfig(Config):
     def read_config(self, config, **kwargs):
-        self.enable_media_repo = config.get("enable_media_repo", True)
-
         self.max_upload_size = self.parse_size(config.get("max_upload_size", "10M"))
         self.max_image_pixels = self.parse_size(config.get("max_image_pixels", "32M"))
         self.max_spider_size = self.parse_size(config.get("max_spider_size", "10M"))
 
-        if self.enable_media_repo:
-            self.media_store_path = self.ensure_directory(
-                config.get("media_store_path", "media_store")
-            )
-            self.uploads_path = self.ensure_directory(
-                config.get("uploads_path", "uploads")
-            )
-        else:
-            self.media_store_path = None
-            self.uploads_path = None
+        self.media_store_path = self.ensure_directory(
+            config.get("media_store_path", "media_store")
+        )
 
         backup_media_store_path = config.get("backup_media_store_path")
 
@@ -159,6 +150,7 @@ class ContentRepositoryConfig(Config):
                 (provider_class, parsed_config, wrapper_config)
             )
 
+        self.uploads_path = self.ensure_directory(config.get("uploads_path", "uploads"))
         self.dynamic_thumbnails = config.get("dynamic_thumbnails", False)
         self.thumbnail_requirements = parse_thumbnail_requirements(
             config.get("thumbnail_sizes", DEFAULT_THUMBNAIL_SIZES)

synapse/config/server.py

@@ -136,7 +136,7 @@ class ServerConfig(Config):
 
         # Whether to enable experimental MSC1849 (aka relations) support
         self.experimental_msc1849_support_enabled = config.get(
-            "experimental_msc1849_support_enabled", True
+            "experimental_msc1849_support_enabled", False
         )
 
         # Options to control access by tracking MAU
@@ -247,12 +247,6 @@ class ServerConfig(Config):
 
         self.gc_thresholds = read_gc_thresholds(config.get("gc_thresholds", None))
 
-        # Resource-constrained Homeserver Configuration
-        self.limit_large_room_joins = config.get("limit_large_remote_room_joins", False)
-        self.limit_large_room_complexity = config.get(
-            "limit_large_remote_room_complexity", 1.0
-        )
-
         bind_port = config.get("bind_port")
         if bind_port:
             if config.get("no_tls", False):
@@ -623,17 +617,6 @@ class ServerConfig(Config):
         # Used by phonehome stats to group together related servers.
         #server_context: context
 
-        # Resource-constrained Homeserver Settings
-        #
-        # If limit_large_remote_room_joins is True, the room complexity will be
-        # checked before a user joins a new remote room. If it is above
-        # limit_large_remote_room_complexity, it will disallow joining or
-        # instantly leave.
-        #
-        # Uncomment the below lines to enable:
-        #limit_large_remote_room_joins: True
-        #limit_large_remote_room_complexity: 1.0
-
         # Whether to require a user to be in the room to add an alias to it.
         # Defaults to 'true'.
         #
@@ -656,8 +639,7 @@ class ServerConfig(Config):
         if args.print_pidfile is not None:
             self.print_pidfile = args.print_pidfile
 
-    @staticmethod
-    def add_arguments(parser):
+    def add_arguments(self, parser):
         server_group = parser.add_argument_group("server")
         server_group.add_argument(
             "-D",

synapse/config/tracer.py

@@ -18,42 +18,33 @@ from ._base import Config, ConfigError
 
 class TracerConfig(Config):
     def read_config(self, config, **kwargs):
-        opentracing_config = config.get("opentracing")
-        if opentracing_config is None:
-            opentracing_config = {}
+        self.tracer_config = config.get("opentracing")
 
-        self.opentracer_enabled = opentracing_config.get("enabled", False)
-        if not self.opentracer_enabled:
-            return
+        self.tracer_config = config.get("opentracing", {"tracer_enabled": False})
 
-        # The tracer is enabled so sanitize the config
+        if self.tracer_config.get("tracer_enabled", False):
+            # The tracer is enabled so sanitize the config
+            # If no whitelists are given
+            self.tracer_config.setdefault("homeserver_whitelist", [])
 
-        self.opentracer_whitelist = opentracing_config.get("homeserver_whitelist", [])
-        if not isinstance(self.opentracer_whitelist, list):
-            raise ConfigError("Tracer homeserver_whitelist config is malformed")
+            if not isinstance(self.tracer_config.get("homeserver_whitelist"), list):
+                raise ConfigError("Tracer homesererver_whitelist config is malformed")
 
     def generate_config_section(cls, **kwargs):
         return """\
         ## Opentracing ##
+        # These settings enable opentracing which implements distributed tracing
+        # This allows you to observe the causal chain of events across servers
+        # including requests, key lookups etc. across any server running
+        # synapse or any other other services which supports opentracing.
+        # (specifically those implemented with jaeger)
 
-        # These settings enable opentracing, which implements distributed tracing.
-        # This allows you to observe the causal chains of events across servers
-        # including requests, key lookups etc., across any server running
-        # synapse or any other other services which supports opentracing
-        # (specifically those implemented with Jaeger).
-        #
-        opentracing:
-            # tracing is disabled by default. Uncomment the following line to enable it.
-            #
-            #enabled: true
-
-            # The list of homeservers we wish to send and receive span contexts and span baggage.
-            # See docs/opentracing.rst
-            # This is a list of regexes which are matched against the server_name of the
-            # homeserver.
-            #
-            # By defult, it is empty, so no servers are matched.
-            #
-            #homeserver_whitelist:
-            #  - ".*"
+        #opentracing:
+        #  # Enable / disable tracer
+        #  tracer_enabled: false
+        #  # The list of homeservers we wish to expose our current traces to.
+        #  # The list is a list of regexes which are matched against the
+        #  # servername of the homeserver
+        #  homeserver_whitelist:
+        #    - ".*"
         """

synapse/events/__init__.py

@@ -104,17 +104,6 @@ class _EventInternalMetadata(object):
         """
         return getattr(self, "proactively_send", True)
 
-    def is_redacted(self):
-        """Whether the event has been redacted.
-
-        This is used for efficiently checking whether an event has been
-        marked as redacted without needing to make another database call.
-
-        Returns:
-            bool
-        """
-        return getattr(self, "redacted", False)
-
 
 def _event_dict_property(key):
     # We want to be able to use hasattr with the event dict properties.

synapse/events/utils.py

@@ -52,15 +52,10 @@ def prune_event(event):
 
     from . import event_type_from_format_version
 
-    pruned_event = event_type_from_format_version(event.format_version)(
+    return event_type_from_format_version(event.format_version)(
         pruned_event_dict, event.internal_metadata.get_dict()
     )
 
-    # Mark the event as redacted
-    pruned_event.internal_metadata.redacted = True
-
-    return pruned_event
-
 
 def prune_event_dict(event_dict):
     """Redacts the event_dict in the same way as `prune_event`, except it
@@ -365,12 +360,9 @@ class EventClientSerializer(object):
         event_id = event.event_id
         serialized_event = serialize_event(event, time_now, **kwargs)
 
-        # If MSC1849 is enabled then we need to look if there are any relations
-        # we need to bundle in with the event.
-        # Do not bundle relations if the event has been redacted
-        if not event.internal_metadata.is_redacted() and (
-            self.experimental_msc1849_support_enabled and bundle_aggregations
-        ):
+        # If MSC1849 is enabled then we need to look if thre are any relations
+        # we need to bundle in with the event
+        if self.experimental_msc1849_support_enabled and bundle_aggregations:
             annotations = yield self.store.get_aggregation_groups_for_event(event_id)
             references = yield self.store.get_relations_for_event(
                 event_id, RelationTypes.REFERENCE, direction="f"

synapse/federation/federation_client.py

@@ -995,39 +995,3 @@ class FederationClient(FederationBase):
                 )
 
         raise RuntimeError("Failed to send to any server.")
-
-    @defer.inlineCallbacks
-    def get_room_complexity(self, destination, room_id):
-        """
-        Fetch the complexity of a remote room from another server.
-
-        Args:
-            destination (str): The remote server
-            room_id (str): The room ID to ask about.
-
-        Returns:
-            Deferred[dict] or Deferred[None]: Dict contains the complexity
-            metric versions, while None means we could not fetch the complexity.
-        """
-        try:
-            complexity = yield self.transport_layer.get_room_complexity(
-                destination=destination, room_id=room_id
-            )
-            defer.returnValue(complexity)
-        except CodeMessageException as e:
-            # We didn't manage to get it -- probably a 404. We are okay if other
-            # servers don't give it to us.
-            logger.debug(
-                "Failed to fetch room complexity via %s for %s, got a %d",
-                destination,
-                room_id,
-                e.code,
-            )
-        except Exception:
-            logger.exception(
-                "Failed to fetch room complexity via %s for %s", destination, room_id
-            )
-
-        # If we don't manage to find it, return None. It's not an error if a
-        # server doesn't give it to us.
-        defer.returnValue(None)

synapse/federation/federation_server.py

@@ -369,7 +369,7 @@ class FederationServer(FederationBase):
             logger.warn("Room version %s not in %s", room_version, supported_versions)
             raise IncompatibleRoomVersionError(room_version=room_version)
 
-        pdu = yield self.handler.on_make_join_request(origin, room_id, user_id)
+        pdu = yield self.handler.on_make_join_request(room_id, user_id)
         time_now = self._clock.time_msec()
         defer.returnValue(
             {"event": pdu.get_pdu_json(time_now), "room_version": room_version}
@@ -423,7 +423,7 @@ class FederationServer(FederationBase):
     def on_make_leave_request(self, origin, room_id, user_id):
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, room_id)
-        pdu = yield self.handler.on_make_leave_request(origin, room_id, user_id)
+        pdu = yield self.handler.on_make_leave_request(room_id, user_id)
 
         room_version = yield self.store.get_room_version(room_id)
 

synapse/federation/transport/client.py

@@ -21,11 +21,7 @@ from six.moves import urllib
 from twisted.internet import defer
 
 from synapse.api.constants import Membership
-from synapse.api.urls import (
-    FEDERATION_UNSTABLE_PREFIX,
-    FEDERATION_V1_PREFIX,
-    FEDERATION_V2_PREFIX,
-)
+from synapse.api.urls import FEDERATION_V1_PREFIX, FEDERATION_V2_PREFIX
 from synapse.logging.utils import log_function
 
 logger = logging.getLogger(__name__)
@@ -939,23 +935,6 @@ class TransportLayerClient(object):
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
-    def get_room_complexity(self, destination, room_id):
-        """
-        Args:
-            destination (str): The remote server
-            room_id (str): The room ID to ask about.
-        """
-        path = _create_path(FEDERATION_UNSTABLE_PREFIX, "/rooms/%s/complexity", room_id)
-
-        return self.client.get_json(destination=destination, path=path)
-
-
-def _create_path(federation_prefix, path, *args):
-    """
-    Ensures that all args are url encoded.
-    """
-    return federation_prefix + path % tuple(urllib.parse.quote(arg, "") for arg in args)
-
 
 def _create_v1_path(path, *args):
     """Creates a path against V1 federation API from the path template and
@@ -972,7 +951,9 @@ def _create_v1_path(path, *args):
     Returns:
         str
     """
-    return _create_path(FEDERATION_V1_PREFIX, path, *args)
+    return FEDERATION_V1_PREFIX + path % tuple(
+        urllib.parse.quote(arg, "") for arg in args
+    )
 
 
 def _create_v2_path(path, *args):
@@ -990,4 +971,6 @@ def _create_v2_path(path, *args):
     Returns:
         str
     """
-    return _create_path(FEDERATION_V2_PREFIX, path, *args)
+    return FEDERATION_V2_PREFIX + path % tuple(
+        urllib.parse.quote(arg, "") for arg in args
+    )

synapse/handlers/federation.py

@@ -1204,28 +1204,11 @@ class FederationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     @log_function
-    def on_make_join_request(self, origin, room_id, user_id):
+    def on_make_join_request(self, room_id, user_id):
         """ We've received a /make_join/ request, so we create a partial
         join event for the room and return that. We do *not* persist or
         process it until the other server has signed it and sent it back.
-
-        Args:
-            origin (str): The (verified) server name of the requesting server.
-            room_id (str): Room to create join event in
-            user_id (str): The user to create the join for
-
-        Returns:
-            Deferred[FrozenEvent]
         """
-
-        if get_domain_from_id(user_id) != origin:
-            logger.info(
-                "Got /make_join request for user %r from different origin %s, ignoring",
-                user_id,
-                origin,
-            )
-            raise SynapseError(403, "User not from origin", Codes.FORBIDDEN)
-
         event_content = {"membership": Membership.JOIN}
 
         room_version = yield self.store.get_room_version(room_id)
@@ -1428,27 +1411,11 @@ class FederationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     @log_function
-    def on_make_leave_request(self, origin, room_id, user_id):
+    def on_make_leave_request(self, room_id, user_id):
         """ We've received a /make_leave/ request, so we create a partial
         leave event for the room and return that. We do *not* persist or
         process it until the other server has signed it and sent it back.
-
-        Args:
-            origin (str): The (verified) server name of the requesting server.
-            room_id (str): Room to create leave event in
-            user_id (str): The user to create the leave for
-
-        Returns:
-            Deferred[FrozenEvent]
         """
-        if get_domain_from_id(user_id) != origin:
-            logger.info(
-                "Got /make_leave request for user %r from different origin %s, ignoring",
-                user_id,
-                origin,
-            )
-            raise SynapseError(403, "User not from origin", Codes.FORBIDDEN)
-
         room_version = yield self.store.get_room_version(room_id)
         builder = self.event_builder_factory.new(
             room_version,
@@ -2798,28 +2765,3 @@ class FederationHandler(BaseHandler):
             )
         else:
             return user_joined_room(self.distributor, user, room_id)
-
-    @defer.inlineCallbacks
-    def get_room_complexity(self, remote_room_hosts, room_id):
-        """
-        Fetch the complexity of a remote room over federation.
-
-        Args:
-            remote_room_hosts (list[str]): The remote servers to ask.
-            room_id (str): The room ID to ask about.
-
-        Returns:
-            Deferred[dict] or Deferred[None]: Dict contains the complexity
-            metric versions, while None means we could not fetch the complexity.
-        """
-
-        for host in remote_room_hosts:
-            res = yield self.federation_client.get_room_complexity(host, room_id)
-
-            # We got a result, return it.
-            if res:
-                defer.returnValue(res)
-
-        # We fell off the bottom, couldn't get the complexity from anyone. Oh
-        # well.
-        defer.returnValue(None)

synapse/handlers/message.py

@@ -23,7 +23,6 @@ from canonicaljson import encode_canonical_json, json
 from twisted.internet import defer
 from twisted.internet.defer import succeed
 
-from synapse import event_auth
 from synapse.api.constants import EventTypes, Membership, RelationTypes
 from synapse.api.errors import (
     AuthError,
@@ -785,20 +784,6 @@ class EventCreationHandler(object):
                     event.signatures.update(returned_invite.signatures)
 
         if event.type == EventTypes.Redaction:
-            original_event = yield self.store.get_event(
-                event.redacts,
-                check_redacted=False,
-                get_prev_content=False,
-                allow_rejected=False,
-                allow_none=True,
-                check_room_id=event.room_id,
-            )
-
-            # we can make some additional checks now if we have the original event.
-            if original_event:
-                if original_event.type == EventTypes.Create:
-                    raise AuthError(403, "Redacting create events is not permitted")
-
             prev_state_ids = yield context.get_prev_state_ids(self.store)
             auth_events_ids = yield self.auth.compute_auth_events(
                 event, prev_state_ids, for_verification=True
@@ -806,18 +791,18 @@ class EventCreationHandler(object):
             auth_events = yield self.store.get_events(auth_events_ids)
             auth_events = {(e.type, e.state_key): e for e in auth_events.values()}
             room_version = yield self.store.get_room_version(event.room_id)
-
-            if event_auth.check_redaction(room_version, event, auth_events=auth_events):
-                # this user doesn't have 'redact' rights, so we need to do some more
-                # checks on the original event. Let's start by checking the original
-                # event exists.
-                if not original_event:
-                    raise NotFoundError("Could not find event %s" % (event.redacts,))
-
+            if self.auth.check_redaction(room_version, event, auth_events=auth_events):
+                original_event = yield self.store.get_event(
+                    event.redacts,
+                    check_redacted=False,
+                    get_prev_content=False,
+                    allow_rejected=False,
+                    allow_none=False,
+                )
                 if event.user_id != original_event.user_id:
                     raise AuthError(403, "You don't have permission to redact events")
 
-                # all the checks are done.
+                # We've already checked.
                 event.internal_metadata.recheck_redaction = False
 
         if event.type == EventTypes.Create:

synapse/handlers/receipts.py

@@ -17,7 +17,7 @@ import logging
 from twisted.internet import defer
 
 from synapse.handlers._base import BaseHandler
-from synapse.types import ReadReceipt, get_domain_from_id
+from synapse.types import ReadReceipt
 
 logger = logging.getLogger(__name__)
 
@@ -40,27 +40,18 @@ class ReceiptsHandler(BaseHandler):
     def _received_remote_receipt(self, origin, content):
         """Called when we receive an EDU of type m.receipt from a remote HS.
         """
-        receipts = []
-        for room_id, room_values in content.items():
-            for receipt_type, users in room_values.items():
-                for user_id, user_values in users.items():
-                    if get_domain_from_id(user_id) != origin:
-                        logger.info(
-                            "Received receipt for user %r from server %s, ignoring",
-                            user_id,
-                            origin,
-                        )
-                        continue
-
-                    receipts.append(
-                        ReadReceipt(
-                            room_id=room_id,
-                            receipt_type=receipt_type,
-                            user_id=user_id,
-                            event_ids=user_values["event_ids"],
-                            data=user_values.get("data", {}),
-                        )
-                    )
+        receipts = [
+            ReadReceipt(
+                room_id=room_id,
+                receipt_type=receipt_type,
+                user_id=user_id,
+                event_ids=user_values["event_ids"],
+                data=user_values.get("data", {}),
+            )
+            for room_id, room_values in content.items()
+            for receipt_type, users in room_values.items()
+            for user_id, user_values in users.items()
+        ]
 
         yield self._handle_new_receipts(receipts)
 

synapse/handlers/room_member.py

@@ -26,7 +26,8 @@ from unpaddedbase64 import decode_base64
 
 from twisted.internet import defer
 
-from synapse import types
+import synapse.server
+import synapse.types
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import AuthError, Codes, HttpResponseException, SynapseError
 from synapse.types import RoomID, UserID
@@ -38,11 +39,6 @@ from ._base import BaseHandler
 logger = logging.getLogger(__name__)
 
 id_server_scheme = "https://"
-ROOM_COMPLEXITY_TOO_GREAT = (
-    "Your homeserver is unable to join rooms this large or complex. "
-    "Please speak to your server administrator, or upgrade your instance "
-    "to join this room."
-)
 
 
 class RoomMemberHandler(object):
@@ -547,7 +543,7 @@ class RoomMemberHandler(object):
             ), "Sender (%s) must be same as requester (%s)" % (sender, requester.user)
             assert self.hs.is_mine(sender), "Sender must be our own: %s" % (sender,)
         else:
-            requester = types.create_requester(target_user)
+            requester = synapse.types.create_requester(target_user)
 
         prev_event = yield self.event_creation_handler.deduplicate_state_event(
             event, context
@@ -949,47 +945,6 @@ class RoomMemberMasterHandler(RoomMemberHandler):
         self.distributor.declare("user_joined_room")
         self.distributor.declare("user_left_room")
 
-    @defer.inlineCallbacks
-    def _is_remote_room_too_complex(self, room_id, remote_room_hosts):
-        """
-        Check if complexity of a remote room is too great.
-
-        Args:
-            room_id (str)
-            remote_room_hosts (list[str])
-
-        Returns: bool of whether the complexity is too great, or None
-            if unable to be fetched
-        """
-        max_complexity = self.hs.config.limit_large_room_complexity
-        complexity = yield self.federation_handler.get_room_complexity(
-            remote_room_hosts, room_id
-        )
-
-        if complexity:
-            if complexity["v1"] > max_complexity:
-                return True
-            return False
-        return None
-
-    @defer.inlineCallbacks
-    def _is_local_room_too_complex(self, room_id):
-        """
-        Check if the complexity of a local room is too great.
-
-        Args:
-            room_id (str)
-
-        Returns: bool
-        """
-        max_complexity = self.hs.config.limit_large_room_complexity
-        complexity = yield self.store.get_room_complexity(room_id)
-
-        if complexity["v1"] > max_complexity:
-            return True
-
-        return False
-
     @defer.inlineCallbacks
     def _remote_join(self, requester, remote_room_hosts, room_id, user, content):
         """Implements RoomMemberHandler._remote_join
@@ -997,6 +952,7 @@ class RoomMemberMasterHandler(RoomMemberHandler):
         # filter ourselves out of remote_room_hosts: do_invite_join ignores it
         # and if it is the only entry we'd like to return a 404 rather than a
         # 500.
+
         remote_room_hosts = [
             host for host in remote_room_hosts if host != self.hs.hostname
         ]
@@ -1004,18 +960,6 @@ class RoomMemberMasterHandler(RoomMemberHandler):
         if len(remote_room_hosts) == 0:
             raise SynapseError(404, "No known servers")
 
-        if self.hs.config.limit_large_room_joins:
-            # Fetch the room complexity
-            too_complex = yield self._is_remote_room_too_complex(
-                room_id, remote_room_hosts
-            )
-            if too_complex is True:
-                raise SynapseError(
-                    code=400,
-                    msg=ROOM_COMPLEXITY_TOO_GREAT,
-                    errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
-                )
-
         # We don't do an auth check if we are doing an invite
         # join dance for now, since we're kinda implicitly checking
         # that we are allowed to join when we decide whether or not we
@@ -1025,31 +969,6 @@ class RoomMemberMasterHandler(RoomMemberHandler):
         )
         yield self._user_joined_room(user, room_id)
 
-        # Check the room we just joined wasn't too large, if we didn't fetch the
-        # complexity of it before.
-        if self.hs.config.limit_large_room_joins:
-            if too_complex is False:
-                # We checked, and we're under the limit.
-                return
-
-            # Check again, but with the local state events
-            too_complex = yield self._is_local_room_too_complex(room_id)
-
-            if too_complex is False:
-                # We're under the limit.
-                return
-
-            # The room is too large. Leave.
-            requester = types.create_requester(user, None, False, None)
-            yield self.update_membership(
-                requester=requester, target=user, room_id=room_id, action="leave"
-            )
-            raise SynapseError(
-                code=400,
-                msg=ROOM_COMPLEXITY_TOO_GREAT,
-                errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
-            )
-
     @defer.inlineCallbacks
     def _remote_reject_invite(self, requester, remote_room_hosts, room_id, target):
         """Implements RoomMemberHandler._remote_reject_invite

synapse/http/server.py

@@ -245,9 +245,7 @@ class JsonResource(HttpServer, resource.Resource):
 
     isLeaf = True
 
-    _PathEntry = collections.namedtuple(
-        "_PathEntry", ["pattern", "callback", "servlet_classname"]
-    )
+    _PathEntry = collections.namedtuple("_PathEntry", ["pattern", "callback"])
 
     def __init__(self, hs, canonical_json=True):
         resource.Resource.__init__(self)
@@ -257,28 +255,12 @@ class JsonResource(HttpServer, resource.Resource):
         self.path_regexs = {}
         self.hs = hs
 
-    def register_paths(self, method, path_patterns, callback, servlet_classname):
-        """
-        Registers a request handler against a regular expression. Later request URLs are
-        checked against these regular expressions in order to identify an appropriate
-        handler for that request.
-
-        Args:
-            method (str): GET, POST etc
-
-            path_patterns (Iterable[str]): A list of regular expressions to which
-                the request URLs are compared.
-
-            callback (function): The handler for the request. Usually a Servlet
-
-            servlet_classname (str): The name of the handler to be used in prometheus
-                and opentracing logs.
-        """
+    def register_paths(self, method, path_patterns, callback):
         method = method.encode("utf-8")  # method is bytes on py3
         for path_pattern in path_patterns:
             logger.debug("Registering for %s %s", method, path_pattern.pattern)
             self.path_regexs.setdefault(method, []).append(
-                self._PathEntry(path_pattern, callback, servlet_classname)
+                self._PathEntry(path_pattern, callback)
             )
 
     def render(self, request):
@@ -293,9 +275,13 @@ class JsonResource(HttpServer, resource.Resource):
             This checks if anyone has registered a callback for that method and
             path.
         """
-        callback, servlet_classname, group_dict = self._get_handler_for_request(request)
+        callback, group_dict = self._get_handler_for_request(request)
 
-        # Make sure we have a name for this handler in prometheus.
+        servlet_instance = getattr(callback, "__self__", None)
+        if servlet_instance is not None:
+            servlet_classname = servlet_instance.__class__.__name__
+        else:
+            servlet_classname = "%r" % callback
         request.request_metrics.name = servlet_classname
 
         # Now trigger the callback. If it returns a response, we send it
@@ -325,8 +311,7 @@ class JsonResource(HttpServer, resource.Resource):
             request (twisted.web.http.Request):
 
         Returns:
-            Tuple[Callable, str, dict[unicode, unicode]]: callback method, the
-                label to use for that method in prometheus metrics, and the
+            Tuple[Callable, dict[unicode, unicode]]: callback method, and the
                 dict mapping keys to path components as specified in the
                 handler's path match regexp.
 
@@ -335,7 +320,7 @@ class JsonResource(HttpServer, resource.Resource):
                 None, or a tuple of (http code, response body).
         """
         if request.method == b"OPTIONS":
-            return _options_handler, "options_request_handler", {}
+            return _options_handler, {}
 
         # Loop through all the registered callbacks to check if the method
         # and path regex match
@@ -343,10 +328,10 @@ class JsonResource(HttpServer, resource.Resource):
             m = path_entry.pattern.match(request.path.decode("ascii"))
             if m:
                 # We found a match!
-                return path_entry.callback, path_entry.servlet_classname, m.groupdict()
+                return path_entry.callback, m.groupdict()
 
         # Huh. No one wanted to handle that? Fiiiiiine. Send 400.
-        return _unrecognised_request_handler, "unrecognised_request_handler", {}
+        return _unrecognised_request_handler, {}
 
     def _send_response(
         self, request, code, response_json_object, response_code_message=None

synapse/http/servlet.py

@@ -290,13 +290,11 @@ class RestServlet(object):
 
             for method in ("GET", "PUT", "POST", "OPTIONS", "DELETE"):
                 if hasattr(self, "on_%s" % (method,)):
-                    servlet_classname = self.__class__.__name__
                     method_handler = getattr(self, "on_%s" % (method,))
                     http_server.register_paths(
                         method,
                         patterns,
-                        trace_servlet(servlet_classname, method_handler),
-                        servlet_classname,
+                        trace_servlet(self.__class__.__name__, method_handler),
                     )
 
         else:

synapse/logging/opentracing.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2019 The Matrix.org Foundation C.I.C.
+# Copyright 2019 The Matrix.org Foundation C.I.C.d
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -24,140 +24,6 @@
 # this move the methods have work very similarly to opentracing's and it should only
 # be a matter of few regexes to move over to opentracing's access patterns proper.
 
-"""
-============================
-Using OpenTracing in Synapse
-============================
-
-Python-specific tracing concepts are at https://opentracing.io/guides/python/.
-Note that Synapse wraps OpenTracing in a small module (this one) in order to make the
-OpenTracing dependency optional. That means that the access patterns are
-different to those demonstrated in the OpenTracing guides. However, it is
-still useful to know, especially if OpenTracing is included as a full dependency
-in the future or if you are modifying this module.
-
-
-OpenTracing is encapsulated so that
-no span objects from OpenTracing are exposed in Synapse's code. This allows
-OpenTracing to be easily disabled in Synapse and thereby have OpenTracing as
-an optional dependency. This does however limit the number of modifiable spans
-at any point in the code to one. From here out references to `opentracing`
-in the code snippets refer to the Synapses module.
-
-Tracing
--------
-
-In Synapse it is not possible to start a non-active span. Spans can be started
-using the ``start_active_span`` method. This returns a scope (see
-OpenTracing docs) which is a context manager that needs to be entered and
-exited. This is usually done by using ``with``.
-
-.. code-block:: python
-
-   from synapse.logging.opentracing import start_active_span
-
-   with start_active_span("operation name"):
-       # Do something we want to tracer
-
-Forgetting to enter or exit a scope will result in some mysterious and grievous log
-context errors.
-
-At anytime where there is an active span ``opentracing.set_tag`` can be used to
-set a tag on the current active span.
-
-Tracing functions
------------------
-
-Functions can be easily traced using decorators. There is a decorator for
-'normal' function and for functions which are actually deferreds. The name of
-the function becomes the operation name for the span.
-
-.. code-block:: python
-
-   from synapse.logging.opentracing import trace, trace_deferred
-
-   # Start a span using 'normal_function' as the operation name
-   @trace
-   def normal_function(*args, **kwargs):
-       # Does all kinds of cool and expected things
-       return something_usual_and_useful
-
-   # Start a span using 'deferred_function' as the operation name
-   @trace_deferred
-   @defer.inlineCallbacks
-   def deferred_function(*args, **kwargs):
-       # We start
-       yield we_wait
-       # we finish
-       defer.returnValue(something_usual_and_useful)
-
-Operation names can be explicitly set for functions by using
-``trace_using_operation_name`` and
-``trace_deferred_using_operation_name``
-
-.. code-block:: python
-
-   from synapse.logging.opentracing import (
-       trace_using_operation_name,
-       trace_deferred_using_operation_name
-   )
-
-   @trace_using_operation_name("A *much* better operation name")
-   def normal_function(*args, **kwargs):
-       # Does all kinds of cool and expected things
-       return something_usual_and_useful
-
-   @trace_deferred_using_operation_name("Another exciting operation name!")
-   @defer.inlineCallbacks
-   def deferred_function(*args, **kwargs):
-       # We start
-       yield we_wait
-       # we finish
-       defer.returnValue(something_usual_and_useful)
-
-Contexts and carriers
----------------------
-
-There are a selection of wrappers for injecting and extracting contexts from
-carriers provided. Unfortunately OpenTracing's three context injection
-techniques are not adequate for our inject of OpenTracing span-contexts into
-Twisted's http headers, EDU contents and our database tables. Also note that
-the binary encoding format mandated by OpenTracing is not actually implemented
-by jaeger_client v4.0.0 - it will silently noop.
-Please refer to the end of ``logging/opentracing.py`` for the available
-injection and extraction methods.
-
-Homeserver whitelisting
------------------------
-
-Most of the whitelist checks are encapsulated in the modules's injection
-and extraction method but be aware that using custom carriers or crossing
-unchartered waters will require the enforcement of the whitelist.
-``logging/opentracing.py`` has a ``whitelisted_homeserver`` method which takes
-in a destination and compares it to the whitelist.
-
-=======
-Gotchas
-=======
-
-- Checking whitelists on span propagation
-- Inserting pii
-- Forgetting to enter or exit a scope
-- Span source: make sure that the span you expect to be active across a
-  function call really will be that one. Does the current function have more
-  than one caller? Will all of those calling functions have be in a context
-  with an active span?
-"""
-
-import contextlib
-import logging
-import re
-from functools import wraps
-
-from twisted.internet import defer
-
-from synapse.config import ConfigError
-
 try:
     import opentracing
 except ImportError:
@@ -169,6 +35,12 @@ except ImportError:
     JaegerConfig = None
     LogContextScopeManager = None
 
+import contextlib
+import logging
+import re
+from functools import wraps
+
+from twisted.internet import defer
 
 logger = logging.getLogger(__name__)
 
@@ -219,8 +91,7 @@ def only_if_tracing(func):
     return _only_if_tracing_inner
 
 
-# A regex which matches the server_names to expose traces for.
-# None means 'block everything'.
+# Block everything by default
 _homeserver_whitelist = None
 
 tags = _DumTagNames
@@ -230,24 +101,31 @@ def init_tracer(config):
     """Set the whitelists and initialise the JaegerClient tracer
 
     Args:
-        config (HomeserverConfig): The config used by the homeserver
+        config (Config)
+        The config used by the homeserver. Here it's used to set the service
+        name to the homeserver's.
     """
     global opentracing
-    if not config.opentracer_enabled:
+    if not config.tracer_config.get("tracer_enabled", False):
         # We don't have a tracer
         opentracing = None
         return
 
-    if not opentracing or not JaegerConfig:
-        raise ConfigError(
-            "The server has been configured to use opentracing but opentracing is not "
-            "installed."
+    if not opentracing:
+        logger.error(
+            "The server has been configure to use opentracing but opentracing is not installed."
+        )
+        raise ModuleNotFoundError("opentracing")
+
+    if not JaegerConfig:
+        logger.error(
+            "The server has been configure to use opentracing but opentracing is not installed."
         )
 
     # Include the worker name
     name = config.worker_name if config.worker_name else "master"
 
-    set_homeserver_whitelist(config.opentracer_whitelist)
+    set_homeserver_whitelist(config.tracer_config["homeserver_whitelist"])
     jaeger_config = JaegerConfig(
         config={"sampler": {"type": "const", "param": 1}, "logging": True},
         service_name="{} {}".format(config.server_name, name),
@@ -354,6 +232,7 @@ def whitelisted_homeserver(destination):
     """Checks if a destination matches the whitelist
     Args:
         destination (String)"""
+    global _homeserver_whitelist
     if _homeserver_whitelist:
         return _homeserver_whitelist.match(destination)
     return False

synapse/logging/scopecontextmanager.py

@@ -34,7 +34,9 @@ class LogContextScopeManager(ScopeManager):
     """
 
     def __init__(self, config):
-        pass
+        # Set the whitelists
+        logger.info(config.tracer_config)
+        self._homeserver_whitelist = config.tracer_config["homeserver_whitelist"]
 
     @property
     def active(self):

synapse/metrics/__init__.py

@@ -29,16 +29,8 @@ from prometheus_client.core import REGISTRY, GaugeMetricFamily, HistogramMetricF
 
 from twisted.internet import reactor
 
-from synapse.metrics._exposition import (
-    MetricsResource,
-    generate_latest,
-    start_http_server,
-)
-
 logger = logging.getLogger(__name__)
 
-METRICS_PREFIX = "/_synapse/metrics"
-
 running_on_pypy = platform.python_implementation() == "PyPy"
 all_metrics = []
 all_collectors = []
@@ -478,12 +470,3 @@ try:
         gc.disable()
 except AttributeError:
     pass
-
-__all__ = [
-    "MetricsResource",
-    "generate_latest",
-    "start_http_server",
-    "LaterGauge",
-    "InFlightGauge",
-    "BucketCollector",
-]

synapse/metrics/_exposition.py

@@ -1,258 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2015-2019 Prometheus Python Client Developers
-# Copyright 2019 Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-This code is based off `prometheus_client/exposition.py` from version 0.7.1.
-
-Due to the renaming of metrics in prometheus_client 0.4.0, this customised
-vendoring of the code will emit both the old versions that Synapse dashboards
-expect, and the newer "best practice" version of the up-to-date official client.
-"""
-
-import math
-import threading
-from collections import namedtuple
-from http.server import BaseHTTPRequestHandler, HTTPServer
-from socketserver import ThreadingMixIn
-from urllib.parse import parse_qs, urlparse
-
-from prometheus_client import REGISTRY
-
-from twisted.web.resource import Resource
-
-try:
-    from prometheus_client.samples import Sample
-except ImportError:
-    Sample = namedtuple("Sample", ["name", "labels", "value", "timestamp", "exemplar"])
-
-
-CONTENT_TYPE_LATEST = str("text/plain; version=0.0.4; charset=utf-8")
-
-
-INF = float("inf")
-MINUS_INF = float("-inf")
-
-
-def floatToGoString(d):
-    d = float(d)
-    if d == INF:
-        return "+Inf"
-    elif d == MINUS_INF:
-        return "-Inf"
-    elif math.isnan(d):
-        return "NaN"
-    else:
-        s = repr(d)
-        dot = s.find(".")
-        # Go switches to exponents sooner than Python.
-        # We only need to care about positive values for le/quantile.
-        if d > 0 and dot > 6:
-            mantissa = "{0}.{1}{2}".format(s[0], s[1:dot], s[dot + 1 :]).rstrip("0.")
-            return "{0}e+0{1}".format(mantissa, dot - 1)
-        return s
-
-
-def sample_line(line, name):
-    if line.labels:
-        labelstr = "{{{0}}}".format(
-            ",".join(
-                [
-                    '{0}="{1}"'.format(
-                        k,
-                        v.replace("\\", r"\\").replace("\n", r"\n").replace('"', r"\""),
-                    )
-                    for k, v in sorted(line.labels.items())
-                ]
-            )
-        )
-    else:
-        labelstr = ""
-    timestamp = ""
-    if line.timestamp is not None:
-        # Convert to milliseconds.
-        timestamp = " {0:d}".format(int(float(line.timestamp) * 1000))
-    return "{0}{1} {2}{3}\n".format(
-        name, labelstr, floatToGoString(line.value), timestamp
-    )
-
-
-def nameify_sample(sample):
-    """
-    If we get a prometheus_client<0.4.0 sample as a tuple, transform it into a
-    namedtuple which has the names we expect.
-    """
-    if not isinstance(sample, Sample):
-        sample = Sample(*sample, None, None)
-
-    return sample
-
-
-def generate_latest(registry, emit_help=False):
-    output = []
-
-    for metric in registry.collect():
-
-        if metric.name.startswith("__unused"):
-            continue
-
-        if not metric.samples:
-            # No samples, don't bother.
-            continue
-
-        mname = metric.name
-        mnewname = metric.name
-        mtype = metric.type
-
-        # OpenMetrics -> Prometheus
-        if mtype == "counter":
-            mnewname = mnewname + "_total"
-        elif mtype == "info":
-            mtype = "gauge"
-            mnewname = mnewname + "_info"
-        elif mtype == "stateset":
-            mtype = "gauge"
-        elif mtype == "gaugehistogram":
-            mtype = "histogram"
-        elif mtype == "unknown":
-            mtype = "untyped"
-
-        # Output in the old format for compatibility.
-        if emit_help:
-            output.append(
-                "# HELP {0} {1}\n".format(
-                    mname,
-                    metric.documentation.replace("\\", r"\\").replace("\n", r"\n"),
-                )
-            )
-        output.append("# TYPE {0} {1}\n".format(mname, mtype))
-        for sample in map(nameify_sample, metric.samples):
-            # Get rid of the OpenMetrics specific samples
-            for suffix in ["_created", "_gsum", "_gcount"]:
-                if sample.name.endswith(suffix):
-                    break
-            else:
-                newname = sample.name.replace(mnewname, mname)
-                if ":" in newname and newname.endswith("_total"):
-                    newname = newname[: -len("_total")]
-                output.append(sample_line(sample, newname))
-
-        # Get rid of the weird colon things while we're at it
-        if mtype == "counter":
-            mnewname = mnewname.replace(":total", "")
-        mnewname = mnewname.replace(":", "_")
-
-        if mname == mnewname:
-            continue
-
-        # Also output in the new format, if it's different.
-        if emit_help:
-            output.append(
-                "# HELP {0} {1}\n".format(
-                    mnewname,
-                    metric.documentation.replace("\\", r"\\").replace("\n", r"\n"),
-                )
-            )
-        output.append("# TYPE {0} {1}\n".format(mnewname, mtype))
-        for sample in map(nameify_sample, metric.samples):
-            # Get rid of the OpenMetrics specific samples
-            for suffix in ["_created", "_gsum", "_gcount"]:
-                if sample.name.endswith(suffix):
-                    break
-            else:
-                output.append(
-                    sample_line(
-                        sample, sample.name.replace(":total", "").replace(":", "_")
-                    )
-                )
-
-    return "".join(output).encode("utf-8")
-
-
-class MetricsHandler(BaseHTTPRequestHandler):
-    """HTTP handler that gives metrics from ``REGISTRY``."""
-
-    registry = REGISTRY
-
-    def do_GET(self):
-        registry = self.registry
-        params = parse_qs(urlparse(self.path).query)
-
-        if "help" in params:
-            emit_help = True
-        else:
-            emit_help = False
-
-        try:
-            output = generate_latest(registry, emit_help=emit_help)
-        except Exception:
-            self.send_error(500, "error generating metric output")
-            raise
-        self.send_response(200)
-        self.send_header("Content-Type", CONTENT_TYPE_LATEST)
-        self.end_headers()
-        self.wfile.write(output)
-
-    def log_message(self, format, *args):
-        """Log nothing."""
-
-    @classmethod
-    def factory(cls, registry):
-        """Returns a dynamic MetricsHandler class tied
-           to the passed registry.
-        """
-        # This implementation relies on MetricsHandler.registry
-        #  (defined above and defaulted to REGISTRY).
-
-        # As we have unicode_literals, we need to create a str()
-        #  object for type().
-        cls_name = str(cls.__name__)
-        MyMetricsHandler = type(cls_name, (cls, object), {"registry": registry})
-        return MyMetricsHandler
-
-
-class _ThreadingSimpleServer(ThreadingMixIn, HTTPServer):
-    """Thread per request HTTP server."""
-
-    # Make worker threads "fire and forget". Beginning with Python 3.7 this
-    # prevents a memory leak because ``ThreadingMixIn`` starts to gather all
-    # non-daemon threads in a list in order to join on them at server close.
-    # Enabling daemon threads virtually makes ``_ThreadingSimpleServer`` the
-    # same as Python 3.7's ``ThreadingHTTPServer``.
-    daemon_threads = True
-
-
-def start_http_server(port, addr="", registry=REGISTRY):
-    """Starts an HTTP server for prometheus metrics as a daemon thread"""
-    CustomMetricsHandler = MetricsHandler.factory(registry)
-    httpd = _ThreadingSimpleServer((addr, port), CustomMetricsHandler)
-    t = threading.Thread(target=httpd.serve_forever)
-    t.daemon = True
-    t.start()
-
-
-class MetricsResource(Resource):
-    """
-    Twisted ``Resource`` that serves prometheus metrics.
-    """
-
-    isLeaf = True
-
-    def __init__(self, registry=REGISTRY):
-        self.registry = registry
-
-    def render_GET(self, request):
-        request.setHeader(b"Content-Type", CONTENT_TYPE_LATEST.encode("ascii"))
-        return generate_latest(self.registry)

synapse/metrics/resource.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015, 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from prometheus_client.twisted import MetricsResource
+
+METRICS_PREFIX = "/_synapse/metrics"
+
+__all__ = ["MetricsResource", "METRICS_PREFIX"]

synapse/python_dependencies.py

@@ -65,7 +65,9 @@ REQUIREMENTS = [
     "msgpack>=0.5.2",
     "phonenumbers>=8.2.0",
     "six>=1.10",
-    "prometheus_client>=0.0.18,<0.8.0",
+    # prometheus_client 0.4.0 changed the format of counter metrics
+    # (cf https://github.com/matrix-org/synapse/issues/4001)
+    "prometheus_client>=0.0.18,<0.4.0",
     # we use attr.s(slots), which arrived in 16.0.0
     # Twisted 18.7.0 requires attrs>=17.4.0
     "attrs>=17.4.0",

synapse/replication/http/_base.py

@@ -205,7 +205,7 @@ class ReplicationEndpoint(object):
         args = "/".join("(?P<%s>[^/]+)" % (arg,) for arg in url_args)
         pattern = re.compile("^/_synapse/replication/%s/%s$" % (self.NAME, args))
 
-        http_server.register_paths(method, [pattern], handler, self.__class__.__name__)
+        http_server.register_paths(method, [pattern], handler)
 
     def _cached_handler(self, request, txn_id, **kwargs):
         """Called on new incoming requests when caching is enabled. Checks

synapse/rest/admin/server_notice_servlet.py

@@ -59,14 +59,9 @@ class SendServerNoticeServlet(RestServlet):
 
     def register(self, json_resource):
         PATTERN = "^/_synapse/admin/v1/send_server_notice"
+        json_resource.register_paths("POST", (re.compile(PATTERN + "$"),), self.on_POST)
         json_resource.register_paths(
-            "POST", (re.compile(PATTERN + "$"),), self.on_POST, self.__class__.__name__
-        )
-        json_resource.register_paths(
-            "PUT",
-            (re.compile(PATTERN + "/(?P<txn_id>[^/]*)$"),),
-            self.on_PUT,
-            self.__class__.__name__,
+            "PUT", (re.compile(PATTERN + "/(?P<txn_id>[^/]*)$"),), self.on_PUT
         )
 
     @defer.inlineCallbacks