Merge branch 'develop' into matrix-org-hotfixes

`@cached` doesn't work on decorated functions, because it uses inspection on
the target to calculate the number of arguments.

.circleci/config.yml

@@ -1,48 +0,0 @@
-version: 2
-jobs:
-  sytestpy2:
-    machine: true
-    steps:
-      - checkout
-      - run: docker pull matrixdotorg/sytest-synapsepy2
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2
-      - store_artifacts:
-          path: ~/project/logs
-          destination: logs
-  sytestpy2postgres:
-    machine: true
-    steps:
-      - checkout
-      - run: docker pull matrixdotorg/sytest-synapsepy2
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2
-      - store_artifacts:
-          path: ~/project/logs
-          destination: logs
-  sytestpy3:
-    machine: true
-    steps:
-      - checkout
-      - run: docker pull matrixdotorg/sytest-synapsepy3
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs hawkowl/sytestpy3
-      - store_artifacts:
-          path: ~/project/logs
-          destination: logs
-  sytestpy3postgres:
-    machine: true
-    steps:
-      - checkout
-      - run: docker pull matrixdotorg/sytest-synapsepy3
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3
-      - store_artifacts:
-          path: ~/project/logs
-          destination: logs
-
-workflows:
-  version: 2
-  build:
-    jobs:
-      - sytestpy2
-      - sytestpy2postgres
-# Currently broken while the Python 3 port is incomplete
-#      - sytestpy3
-#      - sytestpy3postgres

.dockerignore

@@ -1,8 +0,0 @@
-Dockerfile
-.travis.yml
-.gitignore
-demo/etc
-tox.ini
-synctl
-.git/*
-.tox/*

.github/ISSUE_TEMPLATE.md

@@ -27,9 +27,8 @@ Describe here the problem that you are experiencing, or the feature you are requ
 
 Describe how what happens differs from what you expected.
 
-<!-- If you can identify any relevant log snippets from _homeserver.log_, please include
-those (please be careful to remove any personal or private data). Please surround them with
-``` (three backticks, on a line on their own), so that they are formatted legibly. -->
+If you can identify any relevant log snippets from _homeserver.log_, please include
+those here (please be careful to remove any personal or private data):
 
 ### Version information
 

.gitignore

@@ -1,6 +1,5 @@
 *.pyc
 .*.swp
-*~
 
 .DS_Store
 _trial_temp/
@@ -14,7 +13,6 @@ docs/build/
 cmdclient_config.json
 homeserver*.db
 homeserver*.log
-homeserver*.log.*
 homeserver*.pid
 homeserver*.yaml
 
@@ -34,7 +32,6 @@ demo/media_store.*
 demo/etc
 
 uploads
-cache
 
 .idea/
 media_store/
@@ -42,8 +39,6 @@ media_store/
 *.tac
 
 build/
-venv/
-venv*/
 
 localhost-800*/
 static/client/register/register_config.js
@@ -53,4 +48,3 @@ env/
 *.config
 
 .vscode/
-.ropeproject/

.travis.yml

@@ -1,43 +1,14 @@
 sudo: false
 language: python
+python: 2.7
 
 # tell travis to cache ~/.cache/pip
 cache: pip
 
-before_script:
-  - git remote set-branches --add origin develop
-  - git fetch origin develop
-
-services:
-  - postgresql
-
-matrix:
-  fast_finish: true
-  include:
-  - python: 2.7
-    env: TOX_ENV=packaging
-
-  - python: 2.7
-    env: TOX_ENV=pep8
-
-  - python: 2.7
-    env: TOX_ENV=py27
-
-  - python: 2.7
-    env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
-
-  - python: 3.6
-    env: TOX_ENV=py36
-
-  - python: 3.6
-    env: TOX_ENV=check_isort
-
-  - python: 3.6
-    env: TOX_ENV=check-newsfragment
-
-  allow_failures:
-  - python: 2.7
-    env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
+env:
+  - TOX_ENV=packaging
+  - TOX_ENV=pep8
+  - TOX_ENV=py27
 
 install:
   - pip install tox

AUTHORS.rst

@@ -60,9 +60,3 @@ Niklas Riekenbrauck <nikriek at gmail dot.com>
 
 Christoph Witzany <christoph at web.crofting.com>
  * Add LDAP support for authentication
-
-Pierre Jaury <pierre at jaury.eu>
-* Docker packaging
-
-Serban Constantin <serban.constantin at gmail dot com>
- * Small bug fix

CONTRIBUTING.rst

@@ -30,11 +30,11 @@ use github's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use `Jenkins <http://matrix.org/jenkins>`_ and
+We use `Jenkins <http://matrix.org/jenkins>`_ and 
 `Travis <https://travis-ci.org/matrix-org/synapse>`_ for continuous
-integration. All pull requests to synapse get automatically tested by Travis;
-the Jenkins builds require an adminstrator to start them. If your change
-breaks the build, this will be shown in github, so please keep an eye on the
+integration. All pull requests to synapse get automatically tested by Travis; 
+the Jenkins builds require an adminstrator to start them. If your change 
+breaks the build, this will be shown in github, so please keep an eye on the 
 pull request for feedback.
 
 Code style
@@ -48,26 +48,6 @@ Please ensure your changes match the cosmetic style of the existing project,
 and **never** mix cosmetic and functional changes in the same commit, as it
 makes it horribly hard to review otherwise.
 
-Changelog
-~~~~~~~~~
-
-All changes, even minor ones, need a corresponding changelog / newsfragment
-entry. These are managed by Towncrier
-(https://github.com/hawkowl/towncrier).
-
-To create a changelog entry, make a new file in the ``changelog.d``
-file named in the format of ``PRnumber.type``. The type can be
-one of ``feature``, ``bugfix``, ``removal`` (also used for
-deprecations), or ``misc`` (for internal-only changes). The content of
-the file is your changelog entry, which can contain RestructuredText
-formatting. A note of contributors is welcomed in changelogs for
-non-misc changes (the content of misc changes is not displayed).
-
-For example, a fix in PR #1234 would have its changelog entry in
-``changelog.d/1234.bugfix``, and contain content like "The security levels of
-Florbs are now validated when recieved over federation. Contributed by Jane
-Matrix".
-
 Attribution
 ~~~~~~~~~~~
 
@@ -125,20 +105,16 @@ the contribution or otherwise have the right to contribute it to Matrix::
         personal information I submit with it, including my sign-off) is
         maintained indefinitely and may be redistributed consistent with
         this project or the open source license(s) involved.
-
+        
 If you agree to this for your contribution, then all that's needed is to
 include the line in your commit or pull request comment::
 
     Signed-off-by: Your Name <your@email.example.org>
-
-We accept contributions under a legally identifiable name, such as
-your name on government documentation or common-law names (names
-claimed by legitimate usage or repute). Unfortunately, we cannot
-accept anonymous contributions at this time.
-
-Git allows you to add this signoff automatically when using the ``-s``
-flag to ``git commit``, which uses the name and email set in your
-``user.name`` and ``user.email`` git configs.
+    
+...using your real name; unfortunately pseudonyms and anonymous contributions
+can't be accepted. Git makes this trivial - just use the -s flag when you do
+``git commit``, having first set ``user.name`` and ``user.email`` git configs
+(which you should have done anyway :)
 
 Conclusion
 ~~~~~~~~~~

MANIFEST.in

@@ -2,7 +2,6 @@ include synctl
 include LICENSE
 include VERSION
 include *.rst
-include *.md
 include demo/README
 include demo/demo.tls.dh
 include demo/*.py
@@ -26,14 +25,7 @@ recursive-include synapse/static *.js
 exclude jenkins.sh
 exclude jenkins*.sh
 exclude jenkins*
-exclude Dockerfile
-exclude .dockerignore
 recursive-exclude jenkins *.sh
 
-include pyproject.toml
-recursive-include changelog.d *
-
 prune .github
 prune demo/etc
-prune docker
-prune .circleci

README.rst

@@ -71,7 +71,7 @@ We'd like to invite you to join #matrix:matrix.org (via
 https://matrix.org/docs/projects/try-matrix-now.html), run a homeserver, take a look
 at the `Matrix spec <https://matrix.org/docs/spec>`_, and experiment with the
 `APIs <https://matrix.org/docs/api>`_ and `Client SDKs
-<https://matrix.org/docs/projects/try-matrix-now.html#client-sdks>`_.
+<http://matrix.org/docs/projects/try-matrix-now.html#client-sdks>`_.
 
 Thanks for using Matrix!
 
@@ -157,19 +157,11 @@ if you prefer.
 
 In case of problems, please see the _`Troubleshooting` section below.
 
-There is an offical synapse image available at 
-https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with
-the docker-compose file available at `contrib/docker <contrib/docker>`_. Further information on
-this including configuration options is available in the README on
-hub.docker.com.
-
-Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
-Dockerfile to automate a synapse server in a single Docker image, at
-https://hub.docker.com/r/avhost/docker-matrix/tags/
+Alternatively, Silvio Fricke has contributed a Dockerfile to automate the
+above in Docker at https://registry.hub.docker.com/u/silviof/docker-matrix/.
 
 Also, Martin Giess has created an auto-deployment process with vagrant/ansible,
-tested with VirtualBox/AWS/DigitalOcean - see 
-https://github.com/EMnify/matrix-synapse-auto-deploy
+tested with VirtualBox/AWS/DigitalOcean - see https://github.com/EMnify/matrix-synapse-auto-deploy
 for details.
 
 Configuring synapse
@@ -290,7 +282,7 @@ Connecting to Synapse from a client
 
 The easiest way to try out your new Synapse installation is by connecting to it
 from a web client. The easiest option is probably the one at
-https://riot.im/app. You will need to specify a "Custom server" when you log on
+http://riot.im/app. You will need to specify a "Custom server" when you log on
 or register: set this to ``https://domain.tld`` if you setup a reverse proxy
 following the recommended setup, or ``https://localhost:8448`` - remember to specify the
 port (``:8448``) if not ``:443`` unless you changed the configuration. (Leave the identity
@@ -336,7 +328,7 @@ Security Note
 =============
 
 Matrix serves raw user generated data in some APIs - specifically the `content
-repository endpoints <https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid>`_.
+repository endpoints <http://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid>`_.
 
 Whilst we have tried to mitigate against possible XSS attacks (e.g.
 https://github.com/matrix-org/synapse/pull/1021) we recommend running
@@ -355,7 +347,7 @@ Platform-Specific Instructions
 Debian
 ------
 
-Matrix provides official Debian packages via apt from https://matrix.org/packages/debian/.
+Matrix provides official Debian packages via apt from http://matrix.org/packages/debian/.
 Note that these packages do not include a client - choose one from
 https://matrix.org/docs/projects/try-matrix-now.html (or build your own with one of our SDKs :)
 
@@ -369,19 +361,6 @@ Synapse is in the Fedora repositories as ``matrix-synapse``::
 Oleg Girko provides Fedora RPMs at
 https://obs.infoserver.lv/project/monitor/matrix-synapse
 
-OpenSUSE
---------
-
-Synapse is in the OpenSUSE repositories as ``matrix-synapse``::
-
-    sudo zypper install matrix-synapse
-
-SUSE Linux Enterprise Server
-----------------------------
-
-Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 repository at
-https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/
-
 ArchLinux
 ---------
 
@@ -544,7 +523,7 @@ Troubleshooting Running
 -----------------------
 
 If synapse fails with ``missing "sodium.h"`` crypto errors, you may need
-to manually upgrade PyNaCL, as synapse uses NaCl (https://nacl.cr.yp.to/) for
+to manually upgrade PyNaCL, as synapse uses NaCl (http://nacl.cr.yp.to/) for
 encryption and digital signatures.
 Unfortunately PyNACL currently has a few issues
 (https://github.com/pyca/pynacl/issues/53) and
@@ -635,9 +614,6 @@ should have the format ``_matrix._tcp.<yourdomain.com> <ttl> IN SRV 10 0 <port>
     $ dig -t srv _matrix._tcp.example.com
     _matrix._tcp.example.com. 3600    IN      SRV     10 0 8448 synapse.example.com.
 
-Note that the server hostname cannot be an alias (CNAME record): it has to point
-directly to the server hosting the synapse instance.
-
 You can then configure your homeserver to use ``<yourdomain.com>`` as the domain in
 its user-ids, by setting ``server_name``::
 
@@ -692,8 +668,8 @@ useful just for development purposes. See `<demo/README>`_.
 Using PostgreSQL
 ================
 
-As of Synapse 0.9, `PostgreSQL <https://www.postgresql.org>`_ is supported as an
-alternative to the `SQLite <https://sqlite.org/>`_ database that Synapse has
+As of Synapse 0.9, `PostgreSQL <http://www.postgresql.org>`_ is supported as an
+alternative to the `SQLite <http://sqlite.org/>`_ database that Synapse has
 traditionally used for convenience and simplicity.
 
 The advantages of Postgres include:
@@ -717,7 +693,7 @@ Using a reverse proxy with Synapse
 It is recommended to put a reverse proxy such as
 `nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
 `Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_ or
-`HAProxy <https://www.haproxy.org/>`_ in front of Synapse. One advantage of
+`HAProxy <http://www.haproxy.org/>`_ in front of Synapse. One advantage of
 doing so is that it means that you can expose the default https port (443) to
 Matrix clients without needing to run Synapse with root privileges.
 

changelog.d/.gitignore

@@ -1 +0,0 @@
-!.gitignore

changelog.d/1491.feature

@@ -1 +0,0 @@
-Add support for the SNI extension to federation TLS connections

changelog.d/3184.feature

@@ -1 +0,0 @@
-Add /_media/r0/config

changelog.d/3423.misc

@@ -1 +0,0 @@
-The test suite now can run under PostgreSQL.

changelog.d/3568.feature

@@ -1 +0,0 @@
-speed up /members API and add `at` and `membership` params as per MSC1227

changelog.d/3574.feature

@@ -1 +0,0 @@
-implement `summary` block in /sync response as per MSC688

changelog.d/3632.misc

@@ -1 +0,0 @@
-Refactor HTTP replication endpoints to reduce code duplication

changelog.d/3633.feature

@@ -1 +0,0 @@
-Add ability to limit number of monthly active users on the server

changelog.d/3647.misc

@@ -1 +0,0 @@
-Tests now correctly execute on Python 3.

changelog.d/3653.feature

@@ -1 +0,0 @@
-Support more federation endpoints on workers

changelog.d/3654.feature

@@ -1 +0,0 @@
-Basic support for room versioning

changelog.d/3655.feature

@@ -1 +0,0 @@
-Ability to disable client/server Synapse via conf toggle

changelog.d/3658.bugfix

@@ -1 +0,0 @@
-Fix occasional glitches in the synapse_event_persisted_position metric

changelog.d/3660.misc

@@ -1 +0,0 @@
-Sytests can now be run inside a Docker container.

changelog.d/3661.bugfix

@@ -1 +0,0 @@
-Fix bug on deleting 3pid when using identity servers that don't support unbind API

changelog.d/3662.feature

@@ -1 +0,0 @@
-Ability to whitelist specific threepids against monthly active user limiting

changelog.d/3664.feature

@@ -1 +0,0 @@
-Add some metrics for the appservice and federation event sending loops

changelog.d/3669.misc

@@ -1 +0,0 @@
-Update docker base image from alpine 3.7 to 3.8.

changelog.d/3670.feature

@@ -1 +0,0 @@
-Where server is disabled, block ability for locked out users to read new messages

changelog.d/3676.bugfix

@@ -1 +0,0 @@
-Make the tests pass on Twisted < 18.7.0

changelog.d/3677.bugfix

@@ -1 +0,0 @@
-Don’t ship recaptcha_ajax.js, use it directly from Google

changelog.d/3678.misc

@@ -1 +0,0 @@
-Rename synapse.util.async to synapse.util.async_helpers to mitigate async becoming a keyword on Python 3.7.

changelog.d/3679.misc

@@ -1 +0,0 @@
-Synapse's tests are now formatted with the black autoformatter.

changelog.d/3681.bugfix

@@ -1 +0,0 @@
-Fixes test_reap_monthly_active_users so it passes under postgres

changelog.d/3684.misc

@@ -1 +0,0 @@
-Implemented a new testing base class to reduce test boilerplate.

changelog.d/3687.feature

@@ -1 +0,0 @@
-set admin uri via config, to be used in error messages where the user should contact the administrator

changelog.d/3689.bugfix

@@ -1 +0,0 @@
-Fix mau blocking calulation bug on login

changelog.d/3690.misc

@@ -1 +0,0 @@
-Rename MAU prometheus metrics

changelog.d/3692.bugfix

@@ -1 +0,0 @@
-Fix missing yield in synapse.storage.monthly_active_users.initialise_reserved_users

contrib/README.rst

@@ -1,10 +0,0 @@
-Community Contributions
-=======================
-
-Everything in this directory are projects submitted by the community that may be useful
-to others. As such, the project maintainers cannot guarantee support, stability
-or backwards compatibility of these projects. 
-
-Files in this directory should *not* be relied on directly, as they may not
-continue to work or exist in future. If you wish to use any of these files then
-they should be copied to avoid them breaking from underneath you.

contrib/docker/README.md

@@ -1,41 +0,0 @@
-# Synapse Docker
-
-### Automated configuration
-
-It is recommended that you use Docker Compose to run your containers, including
-this image and a Postgres server. A sample ``docker-compose.yml`` is provided,
-including example labels for reverse proxying and other artifacts.
-
-Read the section about environment variables and set at least mandatory variables,
-then run the server:
-
-```
-docker-compose up -d
-```
-
-If secrets are not specified in the environment variables, they will be generated
-as part of the startup. Please ensure these secrets are kept between launches of the
-Docker container, as their loss may require users to log in again.
-
-### Manual configuration
-
-A sample ``docker-compose.yml`` is provided, including example labels for
-reverse proxying and other artifacts. The docker-compose file is an example,
-please comment/uncomment sections that are not suitable for your usecase.
-
-Specify a ``SYNAPSE_CONFIG_PATH``, preferably to a persistent path,
-to use manual configuration. To generate a fresh ``homeserver.yaml``, simply run:
-
-```
-docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host synapse generate
-```
-
-Then, customize your configuration and run the server:
-
-```
-docker-compose up -d
-```
-
-### More information
-
-For more information on required environment variables and mounts, see the main docker documentation at [/docker/README.md](../../docker/README.md)

contrib/docker/docker-compose.yml

@@ -1,50 +0,0 @@
-# This compose file is compatible with Compose itself, it might need some
-# adjustments to run properly with stack.
-
-version: '3'
-
-services:
-
-  synapse:
-    build: ../..
-    image: docker.io/matrixdotorg/synapse:latest
-    # Since snyapse does not retry to connect to the database, restart upon
-    # failure
-    restart: unless-stopped
-    # See the readme for a full documentation of the environment settings
-    environment:
-      - SYNAPSE_SERVER_NAME=my.matrix.host
-      - SYNAPSE_REPORT_STATS=no
-      - SYNAPSE_ENABLE_REGISTRATION=yes
-      - SYNAPSE_LOG_LEVEL=INFO
-      - POSTGRES_PASSWORD=changeme
-    volumes:
-      # You may either store all the files in a local folder
-      - ./files:/data
-      # .. or you may split this between different storage points
-      # - ./files:/data
-      # - /path/to/ssd:/data/uploads
-      # - /path/to/large_hdd:/data/media
-    depends_on:
-      - db
-    # In order to expose Synapse, remove one of the following, you might for
-    # instance expose the TLS port directly:
-    ports:
-      - 8448:8448/tcp
-    # ... or use a reverse proxy, here is an example for traefik:
-    labels:
-      - traefik.enable=true
-      - traefik.frontend.rule=Host:my.matrix.Host
-      - traefik.port=8448
-
-  db:
-    image: docker.io/postgres:10-alpine
-    # Change that password, of course!
-    environment:
-      - POSTGRES_USER=synapse
-      - POSTGRES_PASSWORD=changeme
-    volumes:
-      # You may store the database tables in a local folder..
-      - ./schemas:/var/lib/postgresql/data
-      # .. or store them on some high performance storage for better results
-      # - /path/to/ssd/storage:/var/lib/postfesql/data

contrib/grafana/README.md

@@ -1,6 +0,0 @@
-# Using the Synapse Grafana dashboard
-
-0. Set up Prometheus and Grafana. Out of scope for this readme. Useful documentation about using Grafana with Prometheus: http://docs.grafana.org/features/datasources/prometheus/
-1. Have your Prometheus scrape your Synapse. https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.rst
-2. Import dashboard into Grafana. Download `synapse.json`. Import it to Grafana and select the correct Prometheus datasource. http://docs.grafana.org/reference/export_import/
-3. Set up additional recording rules

contrib/systemd/synapse.service

@@ -2,9 +2,6 @@
 # (e.g. https://www.archlinux.org/packages/community/any/matrix-synapse/ for ArchLinux)
 # rather than in a user home directory or similar under virtualenv.
 
-# **NOTE:** This is an example service file that may change in the future. If you
-# wish to use this please copy rather than symlink it.
-
 [Unit]
 Description=Synapse Matrix homeserver
 
@@ -15,7 +12,6 @@ Group=synapse
 WorkingDirectory=/var/lib/synapse
 ExecStart=/usr/bin/python2.7 -m synapse.app.homeserver --config-path=/etc/synapse/homeserver.yaml
 ExecStop=/usr/bin/synctl stop /etc/synapse/homeserver.yaml
-# EnvironmentFile=-/etc/sysconfig/synapse  # Can be used to e.g. set SYNAPSE_CACHE_FACTOR 
 
 [Install]
 WantedBy=multi-user.target

docker/Dockerfile

@@ -1,35 +0,0 @@
-FROM docker.io/python:2-alpine3.8
-
-RUN apk add --no-cache --virtual .nacl_deps \
-        build-base \
-        libffi-dev \
-        libjpeg-turbo-dev \
-        libressl-dev \
-        libxslt-dev \
-        linux-headers \
-        postgresql-dev \
-        su-exec \
-        zlib-dev
-
-COPY . /synapse
-
-# A wheel cache may be provided in ./cache for faster build
-RUN cd /synapse \
- && pip install --upgrade \
-        lxml \
-        pip \
-        psycopg2 \
-        setuptools \
- && mkdir -p /synapse/cache \
- && pip install -f /synapse/cache --upgrade --process-dependency-links . \
- && mv /synapse/docker/start.py /synapse/docker/conf / \
- && rm -rf \
-        setup.cfg \
-        setup.py \
-        synapse
-
-VOLUME ["/data"]
-
-EXPOSE 8008/tcp 8448/tcp
-
-ENTRYPOINT ["/start.py"]

docker/README.md

@@ -1,124 +0,0 @@
-# Synapse Docker
-
-This Docker image will run Synapse as a single process. It does not provide a database
-server or a TURN server, you should run these separately.
-
-## Run
-
-We do not currently offer a `latest` image, as this has somewhat undefined semantics.
-We instead release only tagged versions so upgrading between releases is entirely
-within your control.
-
-### Using docker-compose (easier)
-
-This image is designed to run either with an automatically generated configuration
-file or with a custom configuration that requires manual editing.
-
-An easy way to make use of this image is via docker-compose. See the
-[contrib/docker](../contrib/docker)
-section of the synapse project for examples.
-
-### Without Compose (harder)
-
-If you do not wish to use Compose, you may still run this image using plain
-Docker commands. Note that the following is just a guideline and you may need
-to add parameters to the docker run command to account for the network situation
-with your postgres database.
-
-```
-docker run \
-    -d \
-    --name synapse \
-    -v ${DATA_PATH}:/data \
-    -e SYNAPSE_SERVER_NAME=my.matrix.host \
-    -e SYNAPSE_REPORT_STATS=yes \
-    docker.io/matrixdotorg/synapse:latest
-```
-
-## Volumes
-
-The image expects a single volume, located at ``/data``, that will hold:
-
-* temporary files during uploads;
-* uploaded media and thumbnails;
-* the SQLite database if you do not configure postgres;
-* the appservices configuration.
-
-You are free to use separate volumes depending on storage endpoints at your
-disposal. For instance, ``/data/media`` coud be stored on a large but low
-performance hdd storage while other files could be stored on high performance
-endpoints.
-
-In order to setup an application service, simply create an ``appservices``
-directory in the data volume and write the application service Yaml
-configuration file there. Multiple application services are supported.
-
-## Environment
-
-Unless you specify a custom path for the configuration file, a very generic
-file will be generated, based on the following environment settings.
-These are a good starting point for setting up your own deployment.
-
-Global settings:
-
-* ``UID``, the user id Synapse will run as [default 991]
-* ``GID``, the group id Synapse will run as [default 991]
-* ``SYNAPSE_CONFIG_PATH``, path to a custom config file
-
-If ``SYNAPSE_CONFIG_PATH`` is set, you should generate a configuration file
-then customize it manually. No other environment variable is required.
-
-Otherwise, a dynamic configuration file will be used. The following environment
-variables are available for configuration:
-
-* ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname.
-* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous
-  statistics reporting back to the Matrix project which helps us to get funding.
-* ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if
-  you run your own TLS-capable reverse proxy).
-* ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on
-  the Synapse instance.
-* ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server.
-* ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`].
-* ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`].
-* ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public
-  key in order to enable recaptcha upon registration.
-* ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private
-  key in order to enable recaptcha upon registration.
-* ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN
-  uris to enable TURN for this homeserver.
-* ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required.
-
-Shared secrets, that will be initialized to random values if not set:
-
-* ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if
-  registration is disable.
-* ``SYNAPSE_MACAROON_SECRET_KEY`` secret for signing access tokens
-  to the server.
-
-Database specific values (will use SQLite if not set):
-
-* `POSTGRES_DB` - The database name for the synapse postgres database. [default: `synapse`]
-* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`]
-* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy.
-* `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`]
-
-Mail server specific values (will not send emails if not set):
-
-* ``SYNAPSE_SMTP_HOST``, hostname to the mail server.
-* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default ``25``].
-* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if any.
-* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail server if any.
-
-## Build
-
-Build the docker image with the `docker build` command from the root of the synapse repository.
-
-```
-docker build -t docker.io/matrixdotorg/synapse . -f docker/Dockerfile
-```
-
-The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:<version>` where `<version>` is the same as the release tag in the synapse git repository.
-
-You may have a local Python wheel cache available, in which case copy the relevant
-packages in the ``cache/`` directory at the root of the project.

docker/conf/homeserver.yaml

@@ -1,219 +0,0 @@
-# vim:ft=yaml
-
-## TLS ##
-
-tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt"
-tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key"
-tls_dh_params_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.dh"
-no_tls: {{ "True" if SYNAPSE_NO_TLS else "False" }}
-tls_fingerprints: []
-
-## Server ##
-
-server_name: "{{ SYNAPSE_SERVER_NAME }}"
-pid_file: /homeserver.pid
-web_client: False
-soft_file_limit: 0
-
-## Ports ##
-
-listeners:
-  {% if not SYNAPSE_NO_TLS %}
-  -
-    port: 8448
-    bind_addresses: ['0.0.0.0']
-    type: http
-    tls: true
-    x_forwarded: false
-    resources:
-      - names: [client]
-        compress: true
-      - names: [federation]  # Federation APIs
-        compress: false
-  {% endif %}
-
-  - port: 8008
-    tls: false
-    bind_addresses: ['0.0.0.0']
-    type: http
-    x_forwarded: false
-
-    resources:
-      - names: [client]
-        compress: true
-      - names: [federation]
-        compress: false
-
-## Database ##
-
-{% if POSTGRES_PASSWORD %}
-database:
-  name: "psycopg2"
-  args:
-    user: "{{ POSTGRES_USER or "synapse" }}"
-    password: "{{ POSTGRES_PASSWORD }}"
-    database: "{{ POSTGRES_DB or "synapse" }}"
-    host: "{{ POSTGRES_HOST or "db" }}"
-    port: "{{ POSTGRES_PORT or "5432" }}"
-    cp_min: 5
-    cp_max: 10
-{% else %}
-database:
-  name: "sqlite3"
-  args:
-    database: "/data/homeserver.db"
-{% endif %}
-
-## Performance ##
-
-event_cache_size: "{{ SYNAPSE_EVENT_CACHE_SIZE or "10K" }}"
-verbose: 0
-log_file: "/data/homeserver.log"
-log_config: "/compiled/log.config"
-
-## Ratelimiting ##
-
-rc_messages_per_second: 0.2
-rc_message_burst_count: 10.0
-federation_rc_window_size: 1000
-federation_rc_sleep_limit: 10
-federation_rc_sleep_delay: 500
-federation_rc_reject_limit: 50
-federation_rc_concurrent: 3
-
-## Files ##
-
-media_store_path: "/data/media"
-uploads_path: "/data/uploads"
-max_upload_size: "10M"
-max_image_pixels: "32M"
-dynamic_thumbnails: false
-
-# List of thumbnail to precalculate when an image is uploaded.
-thumbnail_sizes:
-- width: 32
-  height: 32
-  method: crop
-- width: 96
-  height: 96
-  method: crop
-- width: 320
-  height: 240
-  method: scale
-- width: 640
-  height: 480
-  method: scale
-- width: 800
-  height: 600
-  method: scale
-
-url_preview_enabled: False
-max_spider_size: "10M"
-
-## Captcha ##
-
-{% if SYNAPSE_RECAPTCHA_PUBLIC_KEY %}
-recaptcha_public_key: "{{ SYNAPSE_RECAPTCHA_PUBLIC_KEY }}"
-recaptcha_private_key: "{{ SYNAPSE_RECAPTCHA_PRIVATE_KEY }}"
-enable_registration_captcha: True
-recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify"
-{% else %}
-recaptcha_public_key: "YOUR_PUBLIC_KEY"
-recaptcha_private_key: "YOUR_PRIVATE_KEY"
-enable_registration_captcha: False
-recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify"
-{% endif %}
-
-## Turn ##
-
-{% if SYNAPSE_TURN_URIS %}
-turn_uris:
-{% for uri in SYNAPSE_TURN_URIS.split(',') %}    - "{{ uri }}"
-{% endfor %}
-turn_shared_secret: "{{ SYNAPSE_TURN_SECRET }}"
-turn_user_lifetime: "1h"
-turn_allow_guests: True
-{% else %}
-turn_uris: []
-turn_shared_secret: "YOUR_SHARED_SECRET"
-turn_user_lifetime: "1h"
-turn_allow_guests: True
-{% endif %}
-
-## Registration ##
-
-enable_registration: {{ "True" if SYNAPSE_ENABLE_REGISTRATION else "False" }}
-registration_shared_secret: "{{ SYNAPSE_REGISTRATION_SHARED_SECRET }}"
-bcrypt_rounds: 12
-allow_guest_access: {{ "True" if SYNAPSE_ALLOW_GUEST else "False" }}
-enable_group_creation: true
-
-# The list of identity servers trusted to verify third party
-# identifiers by this server.
-trusted_third_party_id_servers:
-    - matrix.org
-    - vector.im
-    - riot.im
-
-## Metrics ###
-
-{% if SYNAPSE_REPORT_STATS.lower() == "yes" %}
-enable_metrics: True
-report_stats: True
-{% else %}
-enable_metrics: False
-report_stats: False
-{% endif %}
-
-## API Configuration ##
-
-room_invite_state_types:
-    - "m.room.join_rules"
-    - "m.room.canonical_alias"
-    - "m.room.avatar"
-    - "m.room.name"
-
-{% if SYNAPSE_APPSERVICES %}
-app_service_config_files:
-{% for appservice in SYNAPSE_APPSERVICES %}    - "{{ appservice }}"
-{% endfor %}
-{% else %}
-app_service_config_files: []
-{% endif %}
-
-macaroon_secret_key: "{{ SYNAPSE_MACAROON_SECRET_KEY }}"
-expire_access_token: False
-
-## Signing Keys ##
-
-signing_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.signing.key"
-old_signing_keys: {}
-key_refresh_interval: "1d" # 1 Day.
-
-# The trusted servers to download signing keys from.
-perspectives:
-  servers:
-    "matrix.org":
-      verify_keys:
-        "ed25519:auto":
-          key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"
-
-password_config:
-   enabled: true
-
-{% if SYNAPSE_SMTP_HOST %}
-email:
-   enable_notifs: false
-   smtp_host: "{{ SYNAPSE_SMTP_HOST }}"
-   smtp_port: {{ SYNAPSE_SMTP_PORT or "25" }}
-   smtp_user: "{{ SYNAPSE_SMTP_USER }}"
-   smtp_pass: "{{ SYNAPSE_SMTP_PASSWORD }}"
-   require_transport_security: False
-   notif_from: "{{ SYNAPSE_SMTP_FROM or "hostmaster@" + SYNAPSE_SERVER_NAME }}"
-   app_name: Matrix
-   template_dir: res/templates
-   notif_template_html: notif_mail.html
-   notif_template_text: notif_mail.txt
-   notif_for_new_users: True
-   riot_base_url: "https://{{ SYNAPSE_SERVER_NAME }}"
-{% endif %}

docker/conf/log.config

@@ -1,29 +0,0 @@
-version: 1
-
-formatters:
-  precise:
-   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s'
-
-filters:
-  context:
-    (): synapse.util.logcontext.LoggingContextFilter
-    request: ""
-
-handlers:
-  console:
-    class: logging.StreamHandler
-    formatter: precise
-    filters: [context]
-
-loggers:
-    synapse:
-        level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }}
-
-    synapse.storage.SQL:
-        # beware: increasing this to DEBUG will make synapse log sensitive
-        # information such as access tokens.
-        level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }}
-
-root:
-    level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }}
-    handlers: [console]

docker/start.py

@@ -1,66 +0,0 @@
-#!/usr/local/bin/python
-
-import jinja2
-import os
-import sys
-import subprocess
-import glob
-
-# Utility functions
-convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ))
-
-def check_arguments(environ, args):
-    for argument in args:
-        if argument not in environ:
-            print("Environment variable %s is mandatory, exiting." % argument)
-            sys.exit(2)
-
-def generate_secrets(environ, secrets):
-    for name, secret in secrets.items():
-        if secret not in environ:
-            filename = "/data/%s.%s.key" % (environ["SYNAPSE_SERVER_NAME"], name)
-            if os.path.exists(filename):
-                with open(filename) as handle: value = handle.read()
-            else:
-                print("Generating a random secret for {}".format(name))
-                value = os.urandom(32).encode("hex")
-                with open(filename, "w") as handle: handle.write(value)
-            environ[secret] = value
-
-# Prepare the configuration
-mode = sys.argv[1] if len(sys.argv) > 1 else None
-environ = os.environ.copy()
-ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991))
-args = ["python", "-m", "synapse.app.homeserver"]
-
-# In generate mode, generate a configuration, missing keys, then exit
-if mode == "generate":
-    check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_CONFIG_PATH"))
-    args += [
-        "--server-name", environ["SYNAPSE_SERVER_NAME"],
-        "--report-stats", environ["SYNAPSE_REPORT_STATS"],
-        "--config-path", environ["SYNAPSE_CONFIG_PATH"],
-        "--generate-config"
-    ]
-    os.execv("/usr/local/bin/python", args)
-
-# In normal mode, generate missing keys if any, then run synapse
-else:
-    # Parse the configuration file
-    if "SYNAPSE_CONFIG_PATH" in environ:
-        args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]]
-    else:
-        check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"))
-        generate_secrets(environ, {
-            "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET",
-            "macaroon": "SYNAPSE_MACAROON_SECRET_KEY"
-        })
-        environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml")
-        if not os.path.exists("/compiled"): os.mkdir("/compiled")
-        convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ)
-        convert("/conf/log.config", "/compiled/log.config", environ)
-        subprocess.check_output(["chown", "-R", ownership, "/data"])
-        args += ["--config-path", "/compiled/homeserver.yaml"]
-    # Generate missing keys and start synapse
-    subprocess.check_output(args + ["--generate-keys"])
-    os.execv("/sbin/su-exec", ["su-exec", ownership] + args)

docs/admin_api/register_api.rst

@@ -1,63 +0,0 @@
-Shared-Secret Registration
-==========================
-
-This API allows for the creation of users in an administrative and
-non-interactive way. This is generally used for bootstrapping a Synapse
-instance with administrator accounts.
-
-To authenticate yourself to the server, you will need both the shared secret
-(``registration_shared_secret`` in the homeserver configuration), and a
-one-time nonce. If the registration shared secret is not configured, this API
-is not enabled.
-
-To fetch the nonce, you need to request one from the API::
-
-  > GET /_matrix/client/r0/admin/register
-
-  < {"nonce": "thisisanonce"}
-
-Once you have the nonce, you can make a ``POST`` to the same URL with a JSON
-body containing the nonce, username, password, whether they are an admin
-(optional, False by default), and a HMAC digest of the content.
-
-As an example::
-
-  > POST /_matrix/client/r0/admin/register
-  > {
-     "nonce": "thisisanonce",
-     "username": "pepper_roni",
-     "password": "pizza",
-     "admin": true,
-     "mac": "mac_digest_here"
-    }
-
-  < {
-     "access_token": "token_here",
-     "user_id": "@pepper_roni@test",
-     "home_server": "test",
-     "device_id": "device_id_here"
-    }
-
-The MAC is the hex digest output of the HMAC-SHA1 algorithm, with the key being
-the shared secret and the content being the nonce, user, password, and either
-the string "admin" or "notadmin", each separated by NULs. For an example of
-generation in Python::
-
-  import hmac, hashlib
-
-  def generate_mac(nonce, user, password, admin=False):
-
-      mac = hmac.new(
-        key=shared_secret,
-        digestmod=hashlib.sha1,
-      )
-
-      mac.update(nonce.encode('utf8'))
-      mac.update(b"\x00")
-      mac.update(user.encode('utf8'))
-      mac.update(b"\x00")
-      mac.update(password.encode('utf8'))
-      mac.update(b"\x00")
-      mac.update(b"admin" if admin else b"notadmin")
-
-      return mac.hexdigest()

docs/admin_api/user_admin_api.rst

@@ -44,26 +44,13 @@ Deactivate Account
 
 This API deactivates an account. It removes active access tokens, resets the
 password, and deletes third-party IDs (to prevent the user requesting a
-password reset). It can also mark the user as GDPR-erased (stopping their data
-from distributed further, and deleting it entirely if there are no other
-references to it).
+password reset).
 
 The api is::
 
     POST /_matrix/client/r0/admin/deactivate/<user_id>
 
-with a body of:
-
-.. code:: json
-
-    {
-        "erase": true
-    }
-
-including an ``access_token`` of a server admin.
-
-The erase parameter is optional and defaults to 'false'.
-An empty body may be passed for backwards compatibility.
+including an ``access_token`` of a server admin, and an empty request body.
 
 
 Reset password

docs/code_style.rst

@@ -16,7 +16,7 @@
       print("I am a fish %s" %
             "moo")
 
-    and this::
+      and this::
 
         print(
             "I am a fish %s" %

docs/consent_tracking.md

@@ -1,160 +0,0 @@
-Support in Synapse for tracking agreement to server terms and conditions
-========================================================================
-
-Synapse 0.30 introduces support for tracking whether users have agreed to the
-terms and conditions set by the administrator of a server - and blocking access
-to the server until they have.
-
-There are several parts to this functionality; each requires some specific
-configuration in `homeserver.yaml` to be enabled.
-
-Note that various parts of the configuation and this document refer to the
-"privacy policy": agreement with a privacy policy is one particular use of this
-feature, but of course adminstrators can specify other terms and conditions
-unrelated to "privacy" per se.
-
-Collecting policy agreement from a user
----------------------------------------
-
-Synapse can be configured to serve the user a simple policy form with an
-"accept" button. Clicking "Accept" records the user's acceptance in the
-database and shows a success page.
-
-To enable this, first create templates for the policy and success pages.
-These should be stored on the local filesystem.
-
-These templates use the [Jinja2](http://jinja.pocoo.org) templating language,
-and [docs/privacy_policy_templates](privacy_policy_templates) gives
-examples of the sort of thing that can be done.
-
-Note that the templates must be stored under a name giving the language of the
-template - currently this must always be `en` (for "English");
-internationalisation support is intended for the future.
-
-The template for the policy itself should be versioned and named according to 
-the version: for example `1.0.html`. The version of the policy which the user
-has agreed to is stored in the database.
-
-Once the templates are in place, make the following changes to `homeserver.yaml`:
-
- 1. Add a `user_consent` section, which should look like:
-
-    ```yaml
-    user_consent:
-      template_dir: privacy_policy_templates
-      version: 1.0
-    ```
-
-    `template_dir` points to the directory containing the policy
-    templates. `version` defines the version of the policy which will be served
-    to the user. In the example above, Synapse will serve
-    `privacy_policy_templates/en/1.0.html`.
-
-
- 2. Add a `form_secret` setting at the top level:
-
-
-    ```yaml
-    form_secret: "<unique secret>"
-    ```
-
-    This should be set to an arbitrary secret string (try `pwgen -y 30` to
-    generate suitable secrets).
-
-    More on what this is used for below.
-
- 3. Add `consent` wherever the `client` resource is currently enabled in the
-    `listeners` configuration. For example:
-
-    ```yaml
-    listeners:
-      - port: 8008
-        resources:
-          - names:
-            - client
-            - consent
-    ```
-
-
-Finally, ensure that `jinja2` is installed. If you are using a virtualenv, this
-should be a matter of `pip install Jinja2`. On debian, try `apt-get install
-python-jinja2`.
-
-Once this is complete, and the server has been restarted, try visiting
-`https://<server>/_matrix/consent`. If correctly configured, this should give
-an error "Missing string query parameter 'u'". It is now possible to manually
-construct URIs where users can give their consent.
-
-### Constructing the consent URI
-
-It may be useful to manually construct the "consent URI" for a given user - for
-instance, in order to send them an email asking them to consent. To do this,
-take the base `https://<server>/_matrix/consent` URL and add the following
-query parameters:
-
- * `u`: the user id of the user. This can either be a full MXID
-   (`@user:server.com`) or just the localpart (`user`).
-
- * `h`: hex-encoded HMAC-SHA256 of `u` using the `form_secret` as a key. It is
-   possible to calculate this on the commandline with something like:
-
-   ```bash
-   echo -n '<user>' | openssl sha256 -hmac '<form_secret>'
-   ```
-
-   This should result in a URI which looks something like:
-   `https://<server>/_matrix/consent?u=<user>&h=68a152465a4d...`.
-
-
-Sending users a server notice asking them to agree to the policy
-----------------------------------------------------------------
-
-It is possible to configure Synapse to send a [server
-notice](server_notices.md) to anybody who has not yet agreed to the current
-version of the policy. To do so:
-
- * ensure that the consent resource is configured, as in the previous section
-
- * ensure that server notices are configured, as in [server_notices.md](server_notices.md).
-
- * Add `server_notice_content` under `user_consent` in `homeserver.yaml`. For
-   example:
-
-   ```yaml
-   user_consent:
-     server_notice_content:
-       msgtype: m.text
-       body: >-
-         Please give your consent to the privacy policy at %(consent_uri)s.
-   ```
-
-   Synapse automatically replaces the placeholder `%(consent_uri)s` with the
-   consent uri for that user.
-
- * ensure that `public_baseurl` is set in `homeserver.yaml`, and gives the base
-   URI that clients use to connect to the server. (It is used to construct
-   `consent_uri` in the server notice.)
-
-
-Blocking users from using the server until they agree to the policy
--------------------------------------------------------------------
-
-Synapse can be configured to block any attempts to join rooms or send messages
-until the user has given their agreement to the policy. (Joining the server
-notices room is exempted from this).
-
-To enable this, add `block_events_error` under `user_consent`. For example:
-
-```yaml
-user_consent:
-  block_events_error: >-
-    You can't send any messages until you consent to the privacy policy at
-    %(consent_uri)s.
-```
-
-Synapse automatically replaces the placeholder `%(consent_uri)s` with the
-consent uri for that user.
-
-ensure that `public_baseurl` is set in `homeserver.yaml`, and gives the base
-URI that clients use to connect to the server. (It is used to construct
-`consent_uri` in the error.)

docs/manhole.md

@@ -1,43 +0,0 @@
-Using the synapse manhole
-=========================
-
-The "manhole" allows server administrators to access a Python shell on a running
-Synapse installation. This is a very powerful mechanism for administration and
-debugging.
-
-To enable it, first uncomment the `manhole` listener configuration in
-`homeserver.yaml`:
-
-```yaml
-listeners:
-  - port: 9000
-    bind_addresses: ['::1', '127.0.0.1']
-    type: manhole
-```
-
-(`bind_addresses` in the above is important: it ensures that access to the
-manhole is only possible for local users).
-
-Note that this will give administrative access to synapse to **all users** with
-shell access to the server. It should therefore **not** be enabled in
-environments where untrusted users have shell access.
-
-Then restart synapse, and point an ssh client at port 9000 on localhost, using
-the username `matrix`:
-
-```bash
-ssh -p9000 matrix@localhost
-```
-
-The password is `rabbithole`.
-
-This gives a Python REPL in which `hs` gives access to the
-`synapse.server.HomeServer` object - which in turn gives access to many other
-parts of the process.
-
-As a simple example, retrieving an event from the database:
-
-```
->>> hs.get_datastore().get_event('$1416420717069yeQaw:matrix.org')
-<Deferred at 0x7ff253fc6998 current result: <FrozenEvent event_id='$1416420717069yeQaw:matrix.org', type='m.room.create', state_key=''>>
-```

docs/metrics-howto.rst

@@ -1,47 +1,25 @@
 How to monitor Synapse metrics using Prometheus
 ===============================================
 
-1. Install Prometheus:
+1. Install prometheus:
 
    Follow instructions at http://prometheus.io/docs/introduction/install/
 
-2. Enable Synapse metrics:
+2. Enable synapse metrics:
 
-   There are two methods of enabling metrics in Synapse.
+   Simply setting a (local) port number will enable it. Pick a port.
+   prometheus itself defaults to 9090, so starting just above that for
+   locally monitored services seems reasonable. E.g. 9092:
 
-   The first serves the metrics as a part of the usual web server and can be
-   enabled by adding the "metrics" resource to the existing listener as such::
+   Add to homeserver.yaml::
 
-     resources:
-       - names:
-         - client
-         - metrics
+     metrics_port: 9092
 
-   This provides a simple way of adding metrics to your Synapse installation,
-   and serves under ``/_synapse/metrics``. If you do not wish your metrics be
-   publicly exposed, you will need to either filter it out at your load
-   balancer, or use the second method.
+   Also ensure that ``enable_metrics`` is set to ``True``.
 
-   The second method runs the metrics server on a different port, in a
-   different thread to Synapse. This can make it more resilient to heavy load
-   meaning metrics cannot be retrieved, and can be exposed to just internal
-   networks easier. The served metrics are available over HTTP only, and will
-   be available at ``/``.
+   Restart synapse.
 
-   Add a new listener to homeserver.yaml::
-
-     listeners:
-       - type: metrics
-         port: 9000
-         bind_addresses:
-           - '0.0.0.0'
-
-   For both options, you will need to ensure that ``enable_metrics`` is set to
-   ``True``.
-
-   Restart Synapse.
-
-3. Add a Prometheus target for Synapse.
+3. Add a prometheus target for synapse.
 
    It needs to set the ``metrics_path`` to a non-default value (under ``scrape_configs``)::
 
@@ -53,50 +31,7 @@ How to monitor Synapse metrics using Prometheus
    If your prometheus is older than 1.5.2, you will need to replace
    ``static_configs`` in the above with ``target_groups``.
 
-   Restart Prometheus.
-
-
-Removal of deprecated metrics & time based counters becoming histograms in 0.31.0
----------------------------------------------------------------------------------
-
-The duplicated metrics deprecated in Synapse 0.27.0 have been removed.
-
-All time duration-based metrics have been changed to be seconds. This affects:
-
-+----------------------------------+
-| msec -> sec metrics              |
-+==================================+
-| python_gc_time                   |
-+----------------------------------+
-| python_twisted_reactor_tick_time |
-+----------------------------------+
-| synapse_storage_query_time       |
-+----------------------------------+
-| synapse_storage_schedule_time    |
-+----------------------------------+
-| synapse_storage_transaction_time |
-+----------------------------------+
-
-Several metrics have been changed to be histograms, which sort entries into
-buckets and allow better analysis. The following metrics are now histograms:
-
-+-------------------------------------------+
-| Altered metrics                           |
-+===========================================+
-| python_gc_time                            |
-+-------------------------------------------+
-| python_twisted_reactor_pending_calls      |
-+-------------------------------------------+
-| python_twisted_reactor_tick_time          |
-+-------------------------------------------+
-| synapse_http_server_response_time_seconds |
-+-------------------------------------------+
-| synapse_storage_query_time                |
-+-------------------------------------------+
-| synapse_storage_schedule_time             |
-+-------------------------------------------+
-| synapse_storage_transaction_time          |
-+-------------------------------------------+
+   Restart prometheus.
 
 
 Block and response metrics renamed for 0.27.0

docs/postgres.rst

@@ -6,22 +6,16 @@ Postgres version 9.4 or later is known to work.
 Set up database
 ===============
 
-Assuming your PostgreSQL database user is called ``postgres``, create a user
-``synapse_user`` with::
-
-   su - postgres
-   createuser --pwprompt synapse_user
-
-The PostgreSQL database used *must* have the correct encoding set, otherwise it
+The PostgreSQL database used *must* have the correct encoding set, otherwise
 would not be able to store UTF8 strings. To create a database with the correct
 encoding use, e.g.::
 
-   CREATE DATABASE synapse
-    ENCODING 'UTF8'
-    LC_COLLATE='C'
-    LC_CTYPE='C'
-    template=template0
-    OWNER synapse_user;
+ CREATE DATABASE synapse
+  ENCODING 'UTF8'
+  LC_COLLATE='C'
+  LC_CTYPE='C'
+  template=template0
+  OWNER synapse_user;
 
 This would create an appropriate database named ``synapse`` owned by the
 ``synapse_user`` user (which must already exist).
@@ -52,8 +46,8 @@ As with Debian/Ubuntu, postgres support depends on the postgres python connector
 Synapse config
 ==============
 
-When you are ready to start using PostgreSQL, edit the ``database`` section in
-your config file to match the following lines::
+When you are ready to start using PostgreSQL, add the following line to your
+config file::
 
     database:
         name: psycopg2
@@ -102,12 +96,9 @@ complete, restart synapse.  For instance::
     cp homeserver.db homeserver.db.snapshot
     ./synctl start
 
-Copy the old config file into a new config file::
-
-    cp homeserver.yaml homeserver-postgres.yaml
-
-Edit the database section as described in the section *Synapse config* above
-and with the SQLite snapshot located at ``homeserver.db.snapshot`` simply run::
+Assuming your new config file (as described in the section *Synapse config*)
+is named ``homeserver-postgres.yaml`` and the SQLite snapshot is at
+``homeserver.db.snapshot`` then simply run::
 
     synapse_port_db --sqlite-database homeserver.db.snapshot \
         --postgres-config homeserver-postgres.yaml
@@ -126,11 +117,6 @@ run::
         --postgres-config homeserver-postgres.yaml
 
 Once that has completed, change the synapse config to point at the PostgreSQL
-database configuration file ``homeserver-postgres.yaml``::
-
-    ./synctl stop
-    mv homeserver.yaml homeserver-old-sqlite.yaml 
-    mv homeserver-postgres.yaml homeserver.yaml 
-    ./synctl start
-
-Synapse should now be running against PostgreSQL.
+database configuration file ``homeserver-postgres.yaml`` (i.e. rename it to 
+``homeserver.yaml``) and restart synapse. Synapse should now be running against
+PostgreSQL.

docs/privacy_policy_templates/en/1.0.html

@@ -1,23 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <title>Matrix.org Privacy policy</title>
-  </head>
-  <body>
-  {% if has_consented %}
-    <p>
-      Your base already belong to us.
-    </p>
-  {% else %}
-    <p>
-      All your base are belong to us.
-    </p>
-    <form method="post" action="consent">
-      <input type="hidden" name="v" value="{{version}}"/>
-      <input type="hidden" name="u" value="{{user}}"/>
-      <input type="hidden" name="h" value="{{userhmac}}"/>
-      <input type="submit" value="Sure thing!"/>
-    </form>
-  {% endif %}
-  </body>
-</html>

docs/privacy_policy_templates/en/success.html

@@ -1,11 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <title>Matrix.org Privacy policy</title>
-  </head>
-  <body>
-    <p>
-      Sweet.
-    </p>
-  </body>
-</html>

docs/server_notices.md

@@ -1,74 +0,0 @@
-Server Notices
-==============
-
-'Server Notices' are a new feature introduced in Synapse 0.30. They provide a
-channel whereby server administrators can send messages to users on the server.
-
-They are used as part of communication of the server polices(see
-[consent_tracking.md](consent_tracking.md)), however the intention is that
-they may also find a use for features such as "Message of the day".
-
-This is a feature specific to Synapse, but it uses standard Matrix
-communication mechanisms, so should work with any Matrix client.
-
-User experience
----------------
-
-When the user is first sent a server notice, they will get an invitation to a
-room (typically called 'Server Notices', though this is configurable in
-`homeserver.yaml`). They will be **unable to reject** this invitation -
-attempts to do so will receive an error.
-
-Once they accept the invitation, they will see the notice message in the room
-history; it will appear to have come from the 'server notices user' (see
-below).
-
-The user is prevented from sending any messages in this room by the power
-levels.
-
-Having joined the room, the user can leave the room if they want. Subsequent
-server notices will then cause a new room to be created.
-
-Synapse configuration
----------------------
-
-Server notices come from a specific user id on the server. Server
-administrators are free to choose the user id - something like `server` is
-suggested, meaning the notices will come from
-`@server:<your_server_name>`. Once the Server Notices user is configured, that
-user id becomes a special, privileged user, so administrators should ensure
-that **it is not already allocated**.
-
-In order to support server notices, it is necessary to add some configuration
-to the `homeserver.yaml` file. In particular, you should add a `server_notices`
-section, which should look like this:
-
-```yaml
-server_notices:
-   system_mxid_localpart: server
-   system_mxid_display_name: "Server Notices"
-   system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
-   room_name: "Server Notices"
-```
-
-The only compulsory setting is `system_mxid_localpart`, which defines the user
-id of the Server Notices user, as above. `room_name` defines the name of the
-room which will be created.
-
-`system_mxid_display_name` and `system_mxid_avatar_url` can be used to set the
-displayname and avatar of the Server Notices user.
-
-Sending notices
----------------
-
-As of the current version of synapse, there is no convenient interface for
-sending notices (other than the automated ones sent as part of consent
-tracking).
-
-In the meantime, it is possible to test this feature using the manhole. Having
-gone into the manhole as described in [manhole.md](manhole.md), a notice can be
-sent with something like:
-
-```
->>> hs.get_server_notices_manager().send_notice('@user:server.com', {'msgtype':'m.text', 'body':'foo'})
-```

docs/workers.rst

@@ -173,23 +173,10 @@ endpoints matching the following regular expressions::
     ^/_matrix/federation/v1/backfill/
     ^/_matrix/federation/v1/get_missing_events/
     ^/_matrix/federation/v1/publicRooms
-    ^/_matrix/federation/v1/query/
-    ^/_matrix/federation/v1/make_join/
-    ^/_matrix/federation/v1/make_leave/
-    ^/_matrix/federation/v1/send_join/
-    ^/_matrix/federation/v1/send_leave/
-    ^/_matrix/federation/v1/invite/
-    ^/_matrix/federation/v1/query_auth/
-    ^/_matrix/federation/v1/event_auth/
-    ^/_matrix/federation/v1/exchange_third_party_invite/
-    ^/_matrix/federation/v1/send/
 
 The above endpoints should all be routed to the federation_reader worker by the
 reverse-proxy configuration.
 
-The `^/_matrix/federation/v1/send/` endpoint must only be handled by a single
-instance.
-
 ``synapse.app.federation_sender``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -219,10 +206,6 @@ Handles client API endpoints. It can handle REST endpoints matching the
 following regular expressions::
 
     ^/_matrix/client/(api/v1|r0|unstable)/publicRooms$
-    ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/joined_members$
-    ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/context/.*$
-    ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$
-    ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$
 
 ``synapse.app.user_dir``
 ~~~~~~~~~~~~~~~~~~~~~~~~

jenkins/prepare_synapse.sh

@@ -1,7 +1,5 @@
 #! /bin/bash
 
-set -eux
-
 cd "`dirname $0`/.."
 
 TOX_DIR=$WORKSPACE/.tox
@@ -16,20 +14,7 @@ fi
 tox -e py27 --notest -v
 
 TOX_BIN=$TOX_DIR/py27/bin
-
-# cryptography 2.2 requires setuptools >= 18.5.
-#
-# older versions of virtualenv (?) give us a virtualenv with the same version
-# of setuptools as is installed on the system python (and tox runs virtualenv
-# under python3, so we get the version of setuptools that is installed on that).
-#
-# anyway, make sure that we have a recent enough setuptools.
-$TOX_BIN/pip install 'setuptools>=18.5'
-
-# we also need a semi-recent version of pip, because old ones fail to install
-# the "enum34" dependency of cryptography.
-$TOX_BIN/pip install 'pip>=10'
-
+$TOX_BIN/pip install setuptools
 { python synapse/python_dependencies.py
   echo lxml psycopg2
 } | xargs $TOX_BIN/pip install

pyproject.toml

@@ -1,30 +0,0 @@
-[tool.towncrier]
-    package = "synapse"
-    filename = "CHANGES.md"
-    directory = "changelog.d"
-    issue_format = "[\\#{issue}](https://github.com/matrix-org/synapse/issues/{issue})"
-
-    [[tool.towncrier.type]]
-        directory = "feature"
-        name = "Features"
-        showcontent = true
-
-    [[tool.towncrier.type]]
-        directory = "bugfix"
-        name = "Bugfixes"
-        showcontent = true
-
-    [[tool.towncrier.type]]
-        directory = "doc"
-        name = "Improved Documentation"
-        showcontent = true
-
-    [[tool.towncrier.type]]
-        directory = "removal"
-        name = "Deprecations and Removals"
-        showcontent = true
-
-    [[tool.towncrier.type]]
-        directory = "misc"
-        name = "Internal Changes"
-        showcontent = true

scripts-dev/federation_client.py

@@ -18,22 +18,14 @@
 from __future__ import print_function
 
 import argparse
-from urlparse import urlparse, urlunparse
-
 import nacl.signing
 import json
 import base64
 import requests
 import sys
-
-from requests.adapters import HTTPAdapter
 import srvlookup
 import yaml
 
-# uncomment the following to enable debug logging of http requests
-#from httplib import HTTPConnection
-#HTTPConnection.debuglevel = 1
-
 def encode_base64(input_bytes):
     """Encode bytes as a base64 string without any padding."""
 
@@ -121,6 +113,17 @@ def read_signing_keys(stream):
     return keys
 
 
+def lookup(destination, path):
+    if ":" in destination:
+        return "https://%s%s" % (destination, path)
+    else:
+        try:
+            srv = srvlookup.lookup("matrix", "tcp", destination)[0]
+            return "https://%s:%d%s" % (srv.host, srv.port, path)
+        except:
+            return "https://%s:%d%s" % (destination, 8448, path)
+
+
 def request_json(method, origin_name, origin_key, destination, path, content):
     if method is None:
         if content is None:
@@ -149,19 +152,13 @@ def request_json(method, origin_name, origin_key, destination, path, content):
         authorization_headers.append(bytes(header))
         print ("Authorization: %s" % header, file=sys.stderr)
 
-    dest = "matrix://%s%s" % (destination, path)
+    dest = lookup(destination, path)
     print ("Requesting %s" % dest, file=sys.stderr)
 
-    s = requests.Session()
-    s.mount("matrix://", MatrixConnectionAdapter())
-
-    result = s.request(
+    result = requests.request(
         method=method,
         url=dest,
-        headers={
-            "Host": destination,
-            "Authorization": authorization_headers[0]
-        },
+        headers={"Authorization": authorization_headers[0]},
         verify=False,
         data=content,
     )
@@ -245,39 +242,5 @@ def read_args_from_config(args):
             args.signing_key_path = config['signing_key_path']
 
 
-class MatrixConnectionAdapter(HTTPAdapter):
-    @staticmethod
-    def lookup(s):
-        if s[-1] == ']':
-            # ipv6 literal (with no port)
-            return s, 8448
-
-        if ":" in s:
-            out = s.rsplit(":",1)
-            try:
-                port = int(out[1])
-            except ValueError:
-                raise ValueError("Invalid host:port '%s'" % s)
-            return out[0], port
-
-        try:
-            srv = srvlookup.lookup("matrix", "tcp", s)[0]
-            return srv.host, srv.port
-        except:
-            return s, 8448
-
-    def get_connection(self, url, proxies=None):
-        parsed = urlparse(url)
-
-        (host, port) = self.lookup(parsed.netloc)
-        netloc = "%s:%d" % (host, port)
-        print("Connecting to %s" % (netloc,), file=sys.stderr)
-        url = urlunparse((
-            "https", netloc, parsed.path, parsed.params, parsed.query,
-            parsed.fragment,
-        ))
-        return super(MatrixConnectionAdapter, self).get_connection(url, proxies)
-
-
 if __name__ == "__main__":
     main()

scripts-dev/nuke-room-from-db.sh

@@ -6,19 +6,9 @@
 
 ## Do not run it lightly.
 
-set -e
-
-if [ "$1" == "-h" ] || [ "$1" == "" ]; then
-  echo "Call with ROOM_ID as first option and then pipe it into the database. So for instance you might run"
-  echo " nuke-room-from-db.sh <room_id> | sqlite3 homeserver.db"
-  echo "or"
-  echo " nuke-room-from-db.sh <room_id> | psql --dbname=synapse"
-  exit
-fi
-
 ROOMID="$1"
 
-cat <<EOF
+sqlite3 homeserver.db <<EOF
 DELETE FROM event_forward_extremities WHERE room_id = '$ROOMID';
 DELETE FROM event_backward_extremities WHERE room_id = '$ROOMID';
 DELETE FROM event_edges WHERE room_id = '$ROOMID';
@@ -39,7 +29,7 @@ DELETE FROM state_groups WHERE room_id = '$ROOMID';
 DELETE FROM state_groups_state WHERE room_id = '$ROOMID';
 DELETE FROM receipts_graph WHERE room_id = '$ROOMID';
 DELETE FROM receipts_linearized WHERE room_id = '$ROOMID';
-DELETE FROM event_search WHERE room_id = '$ROOMID'; 
+DELETE FROM event_search_content WHERE c1room_id = '$ROOMID';
 DELETE FROM guest_access WHERE room_id = '$ROOMID';
 DELETE FROM history_visibility WHERE room_id = '$ROOMID';
 DELETE FROM room_tags WHERE room_id = '$ROOMID';

scripts/register_new_matrix_user

@@ -26,37 +26,11 @@ import yaml
 
 
 def request_registration(user, password, server_location, shared_secret, admin=False):
-    req = urllib2.Request(
-        "%s/_matrix/client/r0/admin/register" % (server_location,),
-        headers={'Content-Type': 'application/json'}
-    )
-
-    try:
-        if sys.version_info[:3] >= (2, 7, 9):
-            # As of version 2.7.9, urllib2 now checks SSL certs
-            import ssl
-            f = urllib2.urlopen(req, context=ssl.SSLContext(ssl.PROTOCOL_SSLv23))
-        else:
-            f = urllib2.urlopen(req)
-        body = f.read()
-        f.close()
-        nonce = json.loads(body)["nonce"]
-    except urllib2.HTTPError as e:
-        print "ERROR! Received %d %s" % (e.code, e.reason,)
-        if 400 <= e.code < 500:
-            if e.info().type == "application/json":
-                resp = json.load(e)
-                if "error" in resp:
-                    print resp["error"]
-        sys.exit(1)
-
     mac = hmac.new(
         key=shared_secret,
         digestmod=hashlib.sha1,
     )
 
-    mac.update(nonce)
-    mac.update("\x00")
     mac.update(user)
     mac.update("\x00")
     mac.update(password)
@@ -66,10 +40,10 @@ def request_registration(user, password, server_location, shared_secret, admin=F
     mac = mac.hexdigest()
 
     data = {
-        "nonce": nonce,
-        "username": user,
+        "user": user,
         "password": password,
         "mac": mac,
+        "type": "org.matrix.login.shared_secret",
         "admin": admin,
     }
 
@@ -78,7 +52,7 @@ def request_registration(user, password, server_location, shared_secret, admin=F
     print "Sending registration request..."
 
     req = urllib2.Request(
-        "%s/_matrix/client/r0/admin/register" % (server_location,),
+        "%s/_matrix/client/api/v1/register" % (server_location,),
         data=json.dumps(data),
         headers={'Content-Type': 'application/json'}
     )

setup.cfg

@@ -14,26 +14,7 @@ ignore =
     pylint.cfg
     tox.ini
 
-[pep8]
-max-line-length = 90
-#  W503 requires that binary operators be at the end, not start, of lines. Erik
-#  doesn't like it.  E203 is contrary to PEP8.
-ignore = W503,E203
-
 [flake8]
-# note that flake8 inherits the "ignore" settings from "pep8" (because it uses
-# pep8 to do those checks), but not the "max-line-length" setting
 max-line-length = 90
-
-[isort]
-line_length = 89
-not_skip = __init__.py
-sections=FUTURE,STDLIB,COMPAT,THIRDPARTY,TWISTED,FIRSTPARTY,TESTS,LOCALFOLDER
-default_section=THIRDPARTY
-known_first_party = synapse
-known_tests=tests
-known_compat = mock,six
-known_twisted=twisted,OpenSSL
-multi_line_output=3
-include_trailing_comma=true
-combine_as_imports=true
+#  W503 requires that binary operators be at the end, not start, of lines. Erik doesn't like it.
+ignore = W503

synapse/__init__.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.33.2"
+__version__ = "0.27.3-rc2"

synapse/api/auth.py

@@ -15,19 +15,15 @@
 
 import logging
 
-from six import itervalues
-
 import pymacaroons
-from netaddr import IPAddress
-
 from twisted.internet import defer
 
 import synapse.types
 from synapse import event_auth
-from synapse.api.constants import EventTypes, JoinRules, Membership
+from synapse.api.constants import EventTypes, Membership, JoinRules
 from synapse.api.errors import AuthError, Codes
 from synapse.types import UserID
-from synapse.util.caches import CACHE_SIZE_FACTOR, register_cache
+from synapse.util.caches import register_cache, CACHE_SIZE_FACTOR
 from synapse.util.caches.lrucache import LruCache
 from synapse.util.metrics import Measure
 
@@ -61,17 +57,16 @@ class Auth(object):
         self.TOKEN_NOT_FOUND_HTTP_STATUS = 401
 
         self.token_cache = LruCache(CACHE_SIZE_FACTOR * 10000)
-        register_cache("cache", "token_cache", self.token_cache)
+        register_cache("token_cache", self.token_cache)
 
     @defer.inlineCallbacks
     def check_from_context(self, event, context, do_sig_check=True):
-        prev_state_ids = yield context.get_prev_state_ids(self.store)
         auth_events_ids = yield self.compute_auth_events(
-            event, prev_state_ids, for_verification=True,
+            event, context.prev_state_ids, for_verification=True,
         )
         auth_events = yield self.store.get_events(auth_events_ids)
         auth_events = {
-            (e.type, e.state_key): e for e in itervalues(auth_events)
+            (e.type, e.state_key): e for e in auth_events.values()
         }
         self.check(event, auth_events=auth_events, do_sig_check=do_sig_check)
 
@@ -194,7 +189,7 @@ class Auth(object):
                     synapse.types.create_requester(user_id, app_service=app_service)
                 )
 
-            access_token = self.get_access_token_from_request(
+            access_token = get_access_token_from_request(
                 request, self.TOKEN_NOT_FOUND_HTTP_STATUS
             )
 
@@ -213,7 +208,7 @@ class Auth(object):
                 default=[b""]
             )[0]
             if user and access_token and ip_addr:
-                yield self.store.insert_client_ip(
+                self.store.insert_client_ip(
                     user_id=user.to_string(),
                     access_token=access_token,
                     ip=ip_addr,
@@ -240,22 +235,17 @@ class Auth(object):
     @defer.inlineCallbacks
     def _get_appservice_user_id(self, request):
         app_service = self.store.get_app_service_by_token(
-            self.get_access_token_from_request(
+            get_access_token_from_request(
                 request, self.TOKEN_NOT_FOUND_HTTP_STATUS
             )
         )
         if app_service is None:
             defer.returnValue((None, None))
 
-        if app_service.ip_range_whitelist:
-            ip_address = IPAddress(self.hs.get_ip_from_request(request))
-            if ip_address not in app_service.ip_range_whitelist:
-                defer.returnValue((None, None))
-
-        if b"user_id" not in request.args:
+        if "user_id" not in request.args:
             defer.returnValue((app_service.sender, app_service))
 
-        user_id = request.args[b"user_id"][0].decode('utf8')
+        user_id = request.args["user_id"][0]
         if app_service.sender == user_id:
             defer.returnValue((app_service.sender, app_service))
 
@@ -496,7 +486,7 @@ class Auth(object):
     def _look_up_user_by_access_token(self, token):
         ret = yield self.store.get_user_by_access_token(token)
         if not ret:
-            logger.warn("Unrecognised access token - not in store.")
+            logger.warn("Unrecognised access token - not in store: %s" % (token,))
             raise AuthError(
                 self.TOKEN_NOT_FOUND_HTTP_STATUS, "Unrecognised access token.",
                 errcode=Codes.UNKNOWN_TOKEN
@@ -514,12 +504,12 @@ class Auth(object):
 
     def get_appservice_by_req(self, request):
         try:
-            token = self.get_access_token_from_request(
+            token = get_access_token_from_request(
                 request, self.TOKEN_NOT_FOUND_HTTP_STATUS
             )
             service = self.store.get_app_service_by_token(token)
             if not service:
-                logger.warn("Unrecognised appservice access token.")
+                logger.warn("Unrecognised appservice access token: %s" % (token,))
                 raise AuthError(
                     self.TOKEN_NOT_FOUND_HTTP_STATUS,
                     "Unrecognised access token.",
@@ -545,8 +535,7 @@ class Auth(object):
 
     @defer.inlineCallbacks
     def add_auth_events(self, builder, context):
-        prev_state_ids = yield context.get_prev_state_ids(self.store)
-        auth_ids = yield self.compute_auth_events(builder, prev_state_ids)
+        auth_ids = yield self.compute_auth_events(builder, context.prev_state_ids)
 
         auth_events_entries = yield self.store.add_event_hashes(
             auth_ids
@@ -664,7 +653,7 @@ class Auth(object):
             auth_events[(EventTypes.PowerLevels, "")] = power_level_event
 
         send_level = event_auth.get_send_level(
-            EventTypes.Aliases, "", power_level_event,
+            EventTypes.Aliases, "", auth_events
         )
         user_level = event_auth.get_user_power_level(user_id, auth_events)
 
@@ -675,131 +664,67 @@ class Auth(object):
                 " edit its room list entry"
             )
 
-    @staticmethod
-    def has_access_token(request):
-        """Checks if the request has an access_token.
 
-        Returns:
-            bool: False if no access_token was given, True otherwise.
-        """
-        query_params = request.args.get("access_token")
-        auth_headers = request.requestHeaders.getRawHeaders(b"Authorization")
-        return bool(query_params) or bool(auth_headers)
+def has_access_token(request):
+    """Checks if the request has an access_token.
 
-    @staticmethod
-    def get_access_token_from_request(request, token_not_found_http_status=401):
-        """Extracts the access_token from the request.
+    Returns:
+        bool: False if no access_token was given, True otherwise.
+    """
+    query_params = request.args.get("access_token")
+    auth_headers = request.requestHeaders.getRawHeaders(b"Authorization")
+    return bool(query_params) or bool(auth_headers)
 
-        Args:
-            request: The http request.
-            token_not_found_http_status(int): The HTTP status code to set in the
-                AuthError if the token isn't found. This is used in some of the
-                legacy APIs to change the status code to 403 from the default of
-                401 since some of the old clients depended on auth errors returning
-                403.
-        Returns:
-            str: The access_token
-        Raises:
-            AuthError: If there isn't an access_token in the request.
-        """
 
-        auth_headers = request.requestHeaders.getRawHeaders(b"Authorization")
-        query_params = request.args.get(b"access_token")
-        if auth_headers:
-            # Try the get the access_token from a "Authorization: Bearer"
-            # header
-            if query_params is not None:
-                raise AuthError(
-                    token_not_found_http_status,
-                    "Mixing Authorization headers and access_token query parameters.",
-                    errcode=Codes.MISSING_TOKEN,
-                )
-            if len(auth_headers) > 1:
-                raise AuthError(
-                    token_not_found_http_status,
-                    "Too many Authorization headers.",
-                    errcode=Codes.MISSING_TOKEN,
-                )
-            parts = auth_headers[0].split(" ")
-            if parts[0] == "Bearer" and len(parts) == 2:
-                return parts[1]
-            else:
-                raise AuthError(
-                    token_not_found_http_status,
-                    "Invalid Authorization header.",
-                    errcode=Codes.MISSING_TOKEN,
-                )
+def get_access_token_from_request(request, token_not_found_http_status=401):
+    """Extracts the access_token from the request.
+
+    Args:
+        request: The http request.
+        token_not_found_http_status(int): The HTTP status code to set in the
+            AuthError if the token isn't found. This is used in some of the
+            legacy APIs to change the status code to 403 from the default of
+            401 since some of the old clients depended on auth errors returning
+            403.
+    Returns:
+        str: The access_token
+    Raises:
+        AuthError: If there isn't an access_token in the request.
+    """
+
+    auth_headers = request.requestHeaders.getRawHeaders(b"Authorization")
+    query_params = request.args.get(b"access_token")
+    if auth_headers:
+        # Try the get the access_token from a "Authorization: Bearer"
+        # header
+        if query_params is not None:
+            raise AuthError(
+                token_not_found_http_status,
+                "Mixing Authorization headers and access_token query parameters.",
+                errcode=Codes.MISSING_TOKEN,
+            )
+        if len(auth_headers) > 1:
+            raise AuthError(
+                token_not_found_http_status,
+                "Too many Authorization headers.",
+                errcode=Codes.MISSING_TOKEN,
+            )
+        parts = auth_headers[0].split(" ")
+        if parts[0] == "Bearer" and len(parts) == 2:
+            return parts[1]
         else:
-            # Try to get the access_token from the query params.
-            if not query_params:
-                raise AuthError(
-                    token_not_found_http_status,
-                    "Missing access token.",
-                    errcode=Codes.MISSING_TOKEN
-                )
-
-            return query_params[0]
-
-    @defer.inlineCallbacks
-    def check_in_room_or_world_readable(self, room_id, user_id):
-        """Checks that the user is or was in the room or the room is world
-        readable. If it isn't then an exception is raised.
-
-        Returns:
-            Deferred[tuple[str, str|None]]: Resolves to the current membership of
-            the user in the room and the membership event ID of the user. If
-            the user is not in the room and never has been, then
-            `(Membership.JOIN, None)` is returned.
-        """
-
-        try:
-            # check_user_was_in_room will return the most recent membership
-            # event for the user if:
-            #  * The user is a non-guest user, and was ever in the room
-            #  * The user is a guest user, and has joined the room
-            # else it will throw.
-            member_event = yield self.check_user_was_in_room(room_id, user_id)
-            defer.returnValue((member_event.membership, member_event.event_id))
-        except AuthError:
-            visibility = yield self.state.get_current_state(
-                room_id, EventTypes.RoomHistoryVisibility, ""
-            )
-            if (
-                visibility and
-                visibility.content["history_visibility"] == "world_readable"
-            ):
-                defer.returnValue((Membership.JOIN, None))
-                return
             raise AuthError(
-                403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN
+                token_not_found_http_status,
+                "Invalid Authorization header.",
+                errcode=Codes.MISSING_TOKEN,
             )
-
-    @defer.inlineCallbacks
-    def check_auth_blocking(self, user_id=None):
-        """Checks if the user should be rejected for some external reason,
-        such as monthly active user limiting or global disable flag
-
-        Args:
-            user_id(str|None): If present, checks for presence against existing
-            MAU cohort
-        """
-        if self.hs.config.hs_disabled:
+    else:
+        # Try to get the access_token from the query params.
+        if not query_params:
             raise AuthError(
-                403, self.hs.config.hs_disabled_message,
-                errcode=Codes.RESOURCE_LIMIT_EXCEED,
-                admin_uri=self.hs.config.admin_uri,
+                token_not_found_http_status,
+                "Missing access token.",
+                errcode=Codes.MISSING_TOKEN
             )
-        if self.hs.config.limit_usage_by_mau is True:
-            # If the user is already part of the MAU cohort
-            if user_id:
-                timestamp = yield self.store.user_last_seen_monthly_active(user_id)
-                if timestamp:
-                    return
-            # Else if there is no room in the MAU bucket, bail
-            current_mau = yield self.store.get_monthly_active_count()
-            if current_mau >= self.hs.config.max_mau_value:
-                raise AuthError(
-                    403, "Monthly Active User Limits AU Limit Exceeded",
-                    admin_uri=self.hs.config.admin_uri,
-                    errcode=Codes.RESOURCE_LIMIT_EXCEED
-                )
+
+        return query_params[0]

synapse/api/constants.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
-# Copyright 2018 New Vector Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,9 +16,6 @@
 
 """Contains constants from the specification."""
 
-# the "depth" field on events is limited to 2**63 - 1
-MAX_DEPTH = 2**63 - 1
-
 
 class Membership(object):
 
@@ -77,8 +73,6 @@ class EventTypes(object):
     Topic = "m.room.topic"
     Name = "m.room.name"
 
-    ServerACL = "m.room.server_acl"
-
 
 class RejectedReason(object):
     AUTH_ERROR = "auth_error"
@@ -95,11 +89,3 @@ class RoomCreationPreset(object):
 class ThirdPartyEntityKind(object):
     USER = "user"
     LOCATION = "location"
-
-
-# the version we will give rooms which are created on this server
-DEFAULT_ROOM_VERSION = "1"
-
-# vdh-test-version is a placeholder to get room versioning support working and tested
-# until we have a working v2.
-KNOWN_ROOM_VERSIONS = {"1", "vdh-test-version"}

synapse/api/errors.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,10 +17,7 @@
 
 import logging
 
-from six import iteritems
-from six.moves import http_client
-
-from canonicaljson import json
+import simplejson as json
 
 logger = logging.getLogger(__name__)
 
@@ -54,11 +50,6 @@ class Codes(object):
     THREEPID_DENIED = "M_THREEPID_DENIED"
     INVALID_USERNAME = "M_INVALID_USERNAME"
     SERVER_NOT_TRUSTED = "M_SERVER_NOT_TRUSTED"
-    CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN"
-    CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM"
-    RESOURCE_LIMIT_EXCEED = "M_RESOURCE_LIMIT_EXCEED"
-    UNSUPPORTED_ROOM_VERSION = "M_UNSUPPORTED_ROOM_VERSION"
-    INCOMPATIBLE_ROOM_VERSION = "M_INCOMPATIBLE_ROOM_VERSION"
 
 
 class CodeMessageException(RuntimeError):
@@ -73,6 +64,20 @@ class CodeMessageException(RuntimeError):
         self.code = code
         self.msg = msg
 
+    def error_dict(self):
+        return cs_error(self.msg)
+
+
+class MatrixCodeMessageException(CodeMessageException):
+    """An error from a general matrix endpoint, eg. from a proxied Matrix API call.
+
+    Attributes:
+        errcode (str): Matrix error code e.g 'M_FORBIDDEN'
+    """
+    def __init__(self, code, msg, errcode=Codes.UNKNOWN):
+        super(MatrixCodeMessageException, self).__init__(code, msg)
+        self.errcode = errcode
+
 
 class SynapseError(CodeMessageException):
     """A base exception type for matrix errors which have an errcode and error
@@ -98,54 +103,38 @@ class SynapseError(CodeMessageException):
             self.errcode,
         )
 
+    @classmethod
+    def from_http_response_exception(cls, err):
+        """Make a SynapseError based on an HTTPResponseException
 
-class ProxiedRequestError(SynapseError):
-    """An error from a general matrix endpoint, eg. from a proxied Matrix API call.
+        This is useful when a proxied request has failed, and we need to
+        decide how to map the failure onto a matrix error to send back to the
+        client.
 
-    Attributes:
-        errcode (str): Matrix error code e.g 'M_FORBIDDEN'
-    """
-    def __init__(self, code, msg, errcode=Codes.UNKNOWN, additional_fields=None):
-        super(ProxiedRequestError, self).__init__(
-            code, msg, errcode
-        )
-        if additional_fields is None:
-            self._additional_fields = {}
-        else:
-            self._additional_fields = dict(additional_fields)
+        An attempt is made to parse the body of the http response as a matrix
+        error. If that succeeds, the errcode and error message from the body
+        are used as the errcode and error message in the new synapse error.
 
-    def error_dict(self):
-        return cs_error(
-            self.msg,
-            self.errcode,
-            **self._additional_fields
-        )
-
-
-class ConsentNotGivenError(SynapseError):
-    """The error returned to the client when the user has not consented to the
-    privacy policy.
-    """
-    def __init__(self, msg, consent_uri):
-        """Constructs a ConsentNotGivenError
+        Otherwise, the errcode is set to M_UNKNOWN, and the error message is
+        set to the reason code from the HTTP response.
 
         Args:
-            msg (str): The human-readable error message
-            consent_url (str): The URL where the user can give their consent
-        """
-        super(ConsentNotGivenError, self).__init__(
-            code=http_client.FORBIDDEN,
-            msg=msg,
-            errcode=Codes.CONSENT_NOT_GIVEN
-        )
-        self._consent_uri = consent_uri
+            err (HttpResponseException):
 
-    def error_dict(self):
-        return cs_error(
-            self.msg,
-            self.errcode,
-            consent_uri=self._consent_uri
-        )
+        Returns:
+            SynapseError:
+        """
+        # try to parse the body as json, to get better errcode/msg, but
+        # default to M_UNKNOWN with the HTTP status as the error text
+        try:
+            j = json.loads(err.response)
+        except ValueError:
+            j = {}
+        errcode = j.get('errcode', Codes.UNKNOWN)
+        errmsg = j.get('error', err.msg)
+
+        res = SynapseError(err.code, errmsg, errcode)
+        return res
 
 
 class RegistrationError(SynapseError):
@@ -224,16 +213,11 @@ class NotFoundError(SynapseError):
 
 class AuthError(SynapseError):
     """An error raised when there was a problem authorising an event."""
-    def __init__(self, code, msg, errcode=Codes.FORBIDDEN, admin_uri=None):
-        self.admin_uri = admin_uri
-        super(AuthError, self).__init__(code, msg, errcode=errcode)
 
-    def error_dict(self):
-        return cs_error(
-            self.msg,
-            self.errcode,
-            admin_uri=self.admin_uri,
-        )
+    def __init__(self, *args, **kwargs):
+        if "errcode" not in kwargs:
+            kwargs["errcode"] = Codes.FORBIDDEN
+        super(AuthError, self).__init__(*args, **kwargs)
 
 
 class EventSizeError(SynapseError):
@@ -293,25 +277,12 @@ class LimitExceededError(SynapseError):
         )
 
 
-class IncompatibleRoomVersionError(SynapseError):
-    """A server is trying to join a room whose version it does not support."""
-
-    def __init__(self, room_version):
-        super(IncompatibleRoomVersionError, self).__init__(
-            code=400,
-            msg="Your homeserver does not support the features required to "
-                "join this room",
-            errcode=Codes.INCOMPATIBLE_ROOM_VERSION,
-        )
-
-        self._room_version = room_version
-
-    def error_dict(self):
-        return cs_error(
-            self.msg,
-            self.errcode,
-            room_version=self._room_version,
-        )
+def cs_exception(exception):
+    if isinstance(exception, CodeMessageException):
+        return exception.error_dict()
+    else:
+        logger.error("Unknown exception type: %s", type(exception))
+        return {}
 
 
 def cs_error(msg, code=Codes.UNKNOWN, **kwargs):
@@ -320,13 +291,13 @@ def cs_error(msg, code=Codes.UNKNOWN, **kwargs):
 
     Args:
         msg (str): The error message.
-        code (str): The error code.
+        code (int): The error code.
         kwargs : Additional keys to add to the response.
     Returns:
         A dict representing the error response JSON.
     """
     err = {"error": msg, "errcode": code}
-    for key, value in iteritems(kwargs):
+    for key, value in kwargs.iteritems():
         err[key] = value
     return err
 
@@ -370,7 +341,7 @@ class HttpResponseException(CodeMessageException):
     Represents an HTTP-level failure of an outbound request
 
     Attributes:
-        response (bytes): body of response
+        response (str): body of response
     """
     def __init__(self, code, msg, response):
         """
@@ -378,39 +349,7 @@ class HttpResponseException(CodeMessageException):
         Args:
             code (int): HTTP status code
             msg (str): reason phrase from HTTP response status line
-            response (bytes): body of response
+            response (str): body of response
         """
         super(HttpResponseException, self).__init__(code, msg)
         self.response = response
-
-    def to_synapse_error(self):
-        """Make a SynapseError based on an HTTPResponseException
-
-        This is useful when a proxied request has failed, and we need to
-        decide how to map the failure onto a matrix error to send back to the
-        client.
-
-        An attempt is made to parse the body of the http response as a matrix
-        error. If that succeeds, the errcode and error message from the body
-        are used as the errcode and error message in the new synapse error.
-
-        Otherwise, the errcode is set to M_UNKNOWN, and the error message is
-        set to the reason code from the HTTP response.
-
-        Returns:
-            SynapseError:
-        """
-        # try to parse the body as json, to get better errcode/msg, but
-        # default to M_UNKNOWN with the HTTP status as the error text
-        try:
-            j = json.loads(self.response)
-        except ValueError:
-            j = {}
-
-        if not isinstance(j, dict):
-            j = {}
-
-        errcode = j.pop('errcode', Codes.UNKNOWN)
-        errmsg = j.pop('error', self.msg)
-
-        return ProxiedRequestError(self.code, errmsg, errcode, j)

synapse/api/filtering.py

@@ -12,15 +12,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import jsonschema
-from canonicaljson import json
-from jsonschema import FormatChecker
-
-from twisted.internet import defer
-
 from synapse.api.errors import SynapseError
 from synapse.storage.presence import UserPresenceState
-from synapse.types import RoomID, UserID
+from synapse.types import UserID, RoomID
+from twisted.internet import defer
+
+import simplejson as json
+import jsonschema
+from jsonschema import FormatChecker
 
 FILTER_SCHEMA = {
     "additionalProperties": False,
@@ -113,13 +112,7 @@ ROOM_EVENT_FILTER_SCHEMA = {
         },
         "contains_url": {
             "type": "boolean"
-        },
-        "lazy_load_members": {
-            "type": "boolean"
-        },
-        "include_redundant_members": {
-            "type": "boolean"
-        },
+        }
     }
 }
 
@@ -267,12 +260,6 @@ class FilterCollection(object):
     def ephemeral_limit(self):
         return self._room_ephemeral_filter.limit()
 
-    def lazy_load_members(self):
-        return self._room_state_filter.lazy_load_members()
-
-    def include_redundant_members(self):
-        return self._room_state_filter.include_redundant_members()
-
     def filter_presence(self, events):
         return self._presence_filter.filter(events)
 
@@ -424,17 +411,11 @@ class Filter(object):
         return room_ids
 
     def filter(self, events):
-        return list(filter(self.check, events))
+        return filter(self.check, events)
 
     def limit(self):
         return self.filter_json.get("limit", 10)
 
-    def lazy_load_members(self):
-        return self.filter_json.get("lazy_load_members", False)
-
-    def include_redundant_members(self):
-        return self.filter_json.get("include_redundant_members", False)
-
 
 def _matches_wildcard(actual_value, filter_value):
     if filter_value.endswith("*"):

synapse/api/urls.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,12 +14,6 @@
 # limitations under the License.
 
 """Contains the URL paths to prefix various aspects of the server with. """
-import hmac
-from hashlib import sha256
-
-from six.moves.urllib.parse import urlencode
-
-from synapse.config import ConfigError
 
 CLIENT_PREFIX = "/_matrix/client/api/v1"
 CLIENT_V2_ALPHA_PREFIX = "/_matrix/client/v2_alpha"
@@ -32,46 +25,3 @@ SERVER_KEY_PREFIX = "/_matrix/key/v1"
 SERVER_KEY_V2_PREFIX = "/_matrix/key/v2"
 MEDIA_PREFIX = "/_matrix/media/r0"
 LEGACY_MEDIA_PREFIX = "/_matrix/media/v1"
-
-
-class ConsentURIBuilder(object):
-    def __init__(self, hs_config):
-        """
-        Args:
-            hs_config (synapse.config.homeserver.HomeServerConfig):
-        """
-        if hs_config.form_secret is None:
-            raise ConfigError(
-                "form_secret not set in config",
-            )
-        if hs_config.public_baseurl is None:
-            raise ConfigError(
-                "public_baseurl not set in config",
-            )
-
-        self._hmac_secret = hs_config.form_secret.encode("utf-8")
-        self._public_baseurl = hs_config.public_baseurl
-
-    def build_user_consent_uri(self, user_id):
-        """Build a URI which we can give to the user to do their privacy
-        policy consent
-
-        Args:
-            user_id (str): mxid or username of user
-
-        Returns
-            (str) the URI where the user can do consent
-        """
-        mac = hmac.new(
-            key=self._hmac_secret,
-            msg=user_id,
-            digestmod=sha256,
-        ).hexdigest()
-        consent_uri = "%s_matrix/consent?%s" % (
-            self._public_baseurl,
-            urlencode({
-                "u": user_id,
-                "h": mac
-            }),
-        )
-        return consent_uri

synapse/app/__init__.py

@@ -14,11 +14,9 @@
 # limitations under the License.
 
 import sys
-
-from synapse import python_dependencies  # noqa: E402
-
 sys.dont_write_bytecode = True
 
+from synapse import python_dependencies   # noqa: E402
 
 try:
     python_dependencies.check_requirements()

synapse/app/_base.py

@@ -17,18 +17,15 @@ import gc
 import logging
 import sys
 
-from daemonize import Daemonize
-
-from twisted.internet import error, reactor
-
-from synapse.util import PreserveLoggingContext
-from synapse.util.rlimit import change_resource_limit
-
 try:
     import affinity
 except Exception:
     affinity = None
 
+from daemonize import Daemonize
+from synapse.util import PreserveLoggingContext
+from synapse.util.rlimit import change_resource_limit
+from twisted.internet import error, reactor
 
 logger = logging.getLogger(__name__)
 
@@ -127,19 +124,6 @@ def quit_with_error(error_string):
     sys.exit(1)
 
 
-def listen_metrics(bind_addresses, port):
-    """
-    Start Prometheus metrics server.
-    """
-    from synapse.metrics import RegistryProxy
-    from prometheus_client import start_http_server
-
-    for host in bind_addresses:
-        reactor.callInThread(start_http_server, int(port),
-                             addr=host, registry=RegistryProxy)
-        logger.info("Metrics now reporting on %s:%d", host, port)
-
-
 def listen_tcp(bind_addresses, port, factory, backlog=50):
     """
     Create a TCP socket for a port and several addresses

synapse/app/appservice.py

@@ -16,9 +16,6 @@
 import logging
 import sys
 
-from twisted.internet import defer, reactor
-from twisted.web.resource import NoResource
-
 import synapse
 from synapse import events
 from synapse.app import _base
@@ -26,7 +23,6 @@ from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
 from synapse.http.site import SynapseSite
-from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
 from synapse.replication.slave.storage.directory import DirectoryStore
@@ -36,9 +32,11 @@ from synapse.replication.tcp.client import ReplicationClientHandler
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
 from synapse.util.httpresourcetree import create_resource_tree
-from synapse.util.logcontext import LoggingContext, run_in_background
+from synapse.util.logcontext import LoggingContext, preserve_fn
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
+from twisted.internet import reactor
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.appservice")
 
@@ -64,7 +62,7 @@ class AppserviceServer(HomeServer):
         for res in listener_config["resources"]:
             for name in res["names"]:
                 if name == "metrics":
-                    resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
+                    resources[METRICS_PREFIX] = MetricsResource(self)
 
         root_resource = create_resource_tree(resources, NoResource())
 
@@ -76,7 +74,6 @@ class AppserviceServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
-                self.version_string,
             )
         )
 
@@ -96,13 +93,6 @@ class AppserviceServer(HomeServer):
                         globals={"hs": self},
                     )
                 )
-            elif listener["type"] == "metrics":
-                if not self.get_config().enable_metrics:
-                    logger.warn(("Metrics listener configured, but "
-                                 "enable_metrics is not True!"))
-                else:
-                    _base.listen_metrics(listener["bind_addresses"],
-                                         listener["port"])
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])
 
@@ -122,14 +112,9 @@ class ASReplicationHandler(ReplicationClientHandler):
 
         if stream_name == "events":
             max_stream_id = self.store.get_room_max_stream_ordering()
-            run_in_background(self._notify_app_services, max_stream_id)
-
-    @defer.inlineCallbacks
-    def _notify_app_services(self, room_stream_id):
-        try:
-            yield self.appservice_handler.notify_interested_services(room_stream_id)
-        except Exception:
-            logger.exception("Error notifying application services of event")
+            preserve_fn(
+                self.appservice_handler.notify_interested_services
+            )(max_stream_id)
 
 
 def start(config_options):

synapse/app/client_reader.py

@@ -16,9 +16,6 @@
 import logging
 import sys
 
-from twisted.internet import reactor
-from twisted.web.resource import NoResource
-
 import synapse
 from synapse import events
 from synapse.app import _base
@@ -28,10 +25,8 @@ from synapse.config.logger import setup_logging
 from synapse.crypto import context_factory
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
-from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
-from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
 from synapse.replication.slave.storage.client_ips import SlavedClientIpStore
 from synapse.replication.slave.storage.directory import DirectoryStore
@@ -39,34 +34,29 @@ from synapse.replication.slave.storage.events import SlavedEventStore
 from synapse.replication.slave.storage.keys import SlavedKeyStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.replication.slave.storage.room import RoomStore
-from synapse.replication.slave.storage.transactions import SlavedTransactionStore
+from synapse.replication.slave.storage.transactions import TransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
-from synapse.rest.client.v1.room import (
-    JoinedRoomMemberListRestServlet,
-    PublicRoomListRestServlet,
-    RoomEventContextServlet,
-    RoomMemberListRestServlet,
-    RoomStateRestServlet,
-)
+from synapse.rest.client.v1.room import PublicRoomListRestServlet
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
 from synapse.util.httpresourcetree import create_resource_tree
 from synapse.util.logcontext import LoggingContext
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
+from twisted.internet import reactor
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.client_reader")
 
 
 class ClientReaderSlavedStore(
-    SlavedAccountDataStore,
     SlavedEventStore,
     SlavedKeyStore,
     RoomStore,
     DirectoryStore,
     SlavedApplicationServiceStore,
     SlavedRegistrationStore,
-    SlavedTransactionStore,
+    TransactionStore,
     SlavedClientIpStore,
     BaseSlavedStore,
 ):
@@ -87,16 +77,10 @@ class ClientReaderServer(HomeServer):
         for res in listener_config["resources"]:
             for name in res["names"]:
                 if name == "metrics":
-                    resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
+                    resources[METRICS_PREFIX] = MetricsResource(self)
                 elif name == "client":
                     resource = JsonResource(self, canonical_json=False)
-
                     PublicRoomListRestServlet(self).register(resource)
-                    RoomMemberListRestServlet(self).register(resource)
-                    JoinedRoomMemberListRestServlet(self).register(resource)
-                    RoomStateRestServlet(self).register(resource)
-                    RoomEventContextServlet(self).register(resource)
-
                     resources.update({
                         "/_matrix/client/r0": resource,
                         "/_matrix/client/unstable": resource,
@@ -114,7 +98,6 @@ class ClientReaderServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
-                self.version_string,
             )
         )
 
@@ -134,13 +117,7 @@ class ClientReaderServer(HomeServer):
                         globals={"hs": self},
                     )
                 )
-            elif listener["type"] == "metrics":
-                if not self.get_config().enable_metrics:
-                    logger.warn(("Metrics listener configured, but "
-                                 "enable_metrics is not True!"))
-                else:
-                    _base.listen_metrics(listener["bind_addresses"],
-                                         listener["port"])
+
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])
 
@@ -168,13 +145,11 @@ def start(config_options):
     database_engine = create_engine(config.database_config)
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
 
     ss = ClientReaderServer(
         config.server_name,
         db_config=config.database_config,
         tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,

synapse/app/event_creator.py

@@ -16,9 +16,6 @@
 import logging
 import sys
 
-from twisted.internet import reactor
-from twisted.web.resource import NoResource
-
 import synapse
 from synapse import events
 from synapse.app import _base
@@ -28,7 +25,6 @@ from synapse.config.logger import setup_logging
 from synapse.crypto import context_factory
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
-from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
@@ -43,13 +39,11 @@ from synapse.replication.slave.storage.pushers import SlavedPusherStore
 from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.replication.slave.storage.room import RoomStore
-from synapse.replication.slave.storage.transactions import SlavedTransactionStore
+from synapse.replication.slave.storage.transactions import TransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
 from synapse.rest.client.v1.room import (
+    RoomSendEventRestServlet, RoomMembershipRestServlet, RoomStateEventRestServlet,
     JoinRoomAliasServlet,
-    RoomMembershipRestServlet,
-    RoomSendEventRestServlet,
-    RoomStateEventRestServlet,
 )
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
@@ -57,13 +51,15 @@ from synapse.util.httpresourcetree import create_resource_tree
 from synapse.util.logcontext import LoggingContext
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
+from twisted.internet import reactor
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.event_creator")
 
 
 class EventCreatorSlavedStore(
     DirectoryStore,
-    SlavedTransactionStore,
+    TransactionStore,
     SlavedProfileStore,
     SlavedAccountDataStore,
     SlavedPusherStore,
@@ -94,7 +90,7 @@ class EventCreatorServer(HomeServer):
         for res in listener_config["resources"]:
             for name in res["names"]:
                 if name == "metrics":
-                    resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
+                    resources[METRICS_PREFIX] = MetricsResource(self)
                 elif name == "client":
                     resource = JsonResource(self, canonical_json=False)
                     RoomSendEventRestServlet(self).register(resource)
@@ -118,7 +114,6 @@ class EventCreatorServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
-                self.version_string,
             )
         )
 
@@ -138,13 +133,6 @@ class EventCreatorServer(HomeServer):
                         globals={"hs": self},
                     )
                 )
-            elif listener["type"] == "metrics":
-                if not self.get_config().enable_metrics:
-                    logger.warn(("Metrics listener configured, but "
-                                 "enable_metrics is not True!"))
-                else:
-                    _base.listen_metrics(listener["bind_addresses"],
-                                         listener["port"])
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])
 
@@ -174,13 +162,11 @@ def start(config_options):
     database_engine = create_engine(config.database_config)
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
 
     ss = EventCreatorServer(
         config.server_name,
         db_config=config.database_config,
         tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,

synapse/app/federation_reader.py

@@ -16,9 +16,6 @@
 import logging
 import sys
 
-from twisted.internet import reactor
-from twisted.web.resource import NoResource
-
 import synapse
 from synapse import events
 from synapse.api.urls import FEDERATION_PREFIX
@@ -29,19 +26,13 @@ from synapse.config.logger import setup_logging
 from synapse.crypto import context_factory
 from synapse.federation.transport.server import TransportLayerServer
 from synapse.http.site import SynapseSite
-from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
-from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
 from synapse.replication.slave.storage.directory import DirectoryStore
 from synapse.replication.slave.storage.events import SlavedEventStore
 from synapse.replication.slave.storage.keys import SlavedKeyStore
-from synapse.replication.slave.storage.profile import SlavedProfileStore
-from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
-from synapse.replication.slave.storage.pushers import SlavedPusherStore
-from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
 from synapse.replication.slave.storage.room import RoomStore
-from synapse.replication.slave.storage.transactions import SlavedTransactionStore
+from synapse.replication.slave.storage.transactions import TransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
@@ -49,21 +40,18 @@ from synapse.util.httpresourcetree import create_resource_tree
 from synapse.util.logcontext import LoggingContext
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
+from twisted.internet import reactor
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.federation_reader")
 
 
 class FederationReaderSlavedStore(
-    SlavedProfileStore,
-    SlavedApplicationServiceStore,
-    SlavedPusherStore,
-    SlavedPushRuleStore,
-    SlavedReceiptsStore,
     SlavedEventStore,
     SlavedKeyStore,
     RoomStore,
     DirectoryStore,
-    SlavedTransactionStore,
+    TransactionStore,
     BaseSlavedStore,
 ):
     pass
@@ -83,7 +71,7 @@ class FederationReaderServer(HomeServer):
         for res in listener_config["resources"]:
             for name in res["names"]:
                 if name == "metrics":
-                    resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
+                    resources[METRICS_PREFIX] = MetricsResource(self)
                 elif name == "federation":
                     resources.update({
                         FEDERATION_PREFIX: TransportLayerServer(self),
@@ -99,7 +87,6 @@ class FederationReaderServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
-                self.version_string,
             )
         )
 
@@ -119,13 +106,6 @@ class FederationReaderServer(HomeServer):
                         globals={"hs": self},
                     )
                 )
-            elif listener["type"] == "metrics":
-                if not self.get_config().enable_metrics:
-                    logger.warn(("Metrics listener configured, but "
-                                 "enable_metrics is not True!"))
-                else:
-                    _base.listen_metrics(listener["bind_addresses"],
-                                         listener["port"])
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])
 
@@ -153,13 +133,11 @@ def start(config_options):
     database_engine = create_engine(config.database_config)
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
 
     ss = FederationReaderServer(
         config.server_name,
         db_config=config.database_config,
         tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,

synapse/app/federation_sender.py

@@ -16,9 +16,6 @@
 import logging
 import sys
 
-from twisted.internet import defer, reactor
-from twisted.web.resource import NoResource
-
 import synapse
 from synapse import events
 from synapse.app import _base
@@ -28,7 +25,6 @@ from synapse.config.logger import setup_logging
 from synapse.crypto import context_factory
 from synapse.federation import send_queue
 from synapse.http.site import SynapseSite
-from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage.deviceinbox import SlavedDeviceInboxStore
 from synapse.replication.slave.storage.devices import SlavedDeviceStore
@@ -36,21 +32,23 @@ from synapse.replication.slave.storage.events import SlavedEventStore
 from synapse.replication.slave.storage.presence import SlavedPresenceStore
 from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
-from synapse.replication.slave.storage.transactions import SlavedTransactionStore
+from synapse.replication.slave.storage.transactions import TransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
-from synapse.util.async_helpers import Linearizer
+from synapse.util.async import Linearizer
 from synapse.util.httpresourcetree import create_resource_tree
-from synapse.util.logcontext import LoggingContext, run_in_background
+from synapse.util.logcontext import LoggingContext, preserve_fn
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
+from twisted.internet import defer, reactor
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.federation_sender")
 
 
 class FederationSenderSlaveStore(
-    SlavedDeviceInboxStore, SlavedTransactionStore, SlavedReceiptsStore, SlavedEventStore,
+    SlavedDeviceInboxStore, TransactionStore, SlavedReceiptsStore, SlavedEventStore,
     SlavedRegistrationStore, SlavedDeviceStore, SlavedPresenceStore,
 ):
     def __init__(self, db_conn, hs):
@@ -91,7 +89,7 @@ class FederationSenderServer(HomeServer):
         for res in listener_config["resources"]:
             for name in res["names"]:
                 if name == "metrics":
-                    resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
+                    resources[METRICS_PREFIX] = MetricsResource(self)
 
         root_resource = create_resource_tree(resources, NoResource())
 
@@ -103,7 +101,6 @@ class FederationSenderServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
-                self.version_string,
             )
         )
 
@@ -123,13 +120,6 @@ class FederationSenderServer(HomeServer):
                         globals={"hs": self},
                     )
                 )
-            elif listener["type"] == "metrics":
-                if not self.get_config().enable_metrics:
-                    logger.warn(("Metrics listener configured, but "
-                                 "enable_metrics is not True!"))
-                else:
-                    _base.listen_metrics(listener["bind_addresses"],
-                                         listener["port"])
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])
 
@@ -186,13 +176,11 @@ def start(config_options):
     config.send_federation = True
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
 
     ps = FederationSenderServer(
         config.server_name,
         db_config=config.database_config,
         tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
@@ -241,7 +229,7 @@ class FederationSenderHandler(object):
         # presence, typing, etc.
         if stream_name == "federation":
             send_queue.process_rows_for_federation(self.federation_sender, rows)
-            run_in_background(self.update_token, token)
+            preserve_fn(self.update_token)(token)
 
         # We also need to poke the federation sender when new events happen
         elif stream_name == "events":
@@ -249,22 +237,19 @@ class FederationSenderHandler(object):
 
     @defer.inlineCallbacks
     def update_token(self, token):
-        try:
-            self.federation_position = token
+        self.federation_position = token
 
-            # We linearize here to ensure we don't have races updating the token
-            with (yield self._fed_position_linearizer.queue(None)):
-                if self._last_ack < self.federation_position:
-                    yield self.store.update_federation_out_pos(
-                        "federation", self.federation_position
-                    )
+        # We linearize here to ensure we don't have races updating the token
+        with (yield self._fed_position_linearizer.queue(None)):
+            if self._last_ack < self.federation_position:
+                yield self.store.update_federation_out_pos(
+                    "federation", self.federation_position
+                )
 
-                    # We ACK this token over replication so that the master can drop
-                    # its in memory queues
-                    self.replication_client.send_federation_ack(self.federation_position)
-                    self._last_ack = self.federation_position
-        except Exception:
-            logger.exception("Error updating federation stream position")
+                # We ACK this token over replication so that the master can drop
+                # its in memory queues
+                self.replication_client.send_federation_ack(self.federation_position)
+                self._last_ack = self.federation_position
 
 
 if __name__ == '__main__':

synapse/app/frontend_proxy.py

@@ -16,9 +16,6 @@
 import logging
 import sys
 
-from twisted.internet import defer, reactor
-from twisted.web.resource import NoResource
-
 import synapse
 from synapse import events
 from synapse.api.errors import SynapseError
@@ -28,9 +25,10 @@ from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
 from synapse.crypto import context_factory
 from synapse.http.server import JsonResource
-from synapse.http.servlet import RestServlet, parse_json_object_from_request
+from synapse.http.servlet import (
+    RestServlet, parse_json_object_from_request,
+)
 from synapse.http.site import SynapseSite
-from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
@@ -38,6 +36,7 @@ from synapse.replication.slave.storage.client_ips import SlavedClientIpStore
 from synapse.replication.slave.storage.devices import SlavedDeviceStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.replication.tcp.client import ReplicationClientHandler
+from synapse.rest.client.v1.base import ClientV1RestServlet, client_path_patterns
 from synapse.rest.client.v2_alpha._base import client_v2_patterns
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
@@ -45,10 +44,41 @@ from synapse.util.httpresourcetree import create_resource_tree
 from synapse.util.logcontext import LoggingContext
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
+from twisted.internet import defer, reactor
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.frontend_proxy")
 
 
+class PresenceStatusStubServlet(ClientV1RestServlet):
+    PATTERNS = client_path_patterns("/presence/(?P<user_id>[^/]*)/status")
+
+    def __init__(self, hs):
+        super(PresenceStatusStubServlet, self).__init__(hs)
+        self.http_client = hs.get_simple_http_client()
+        self.auth = hs.get_auth()
+        self.main_uri = hs.config.worker_main_http_uri
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, user_id):
+        # Pass through the auth headers, if any, in case the access token
+        # is there.
+        auth_headers = request.requestHeaders.getRawHeaders("Authorization", [])
+        headers = {
+            "Authorization": auth_headers,
+        }
+        result = yield self.http_client.get_json(
+            self.main_uri + request.uri,
+            headers=headers,
+        )
+        defer.returnValue((200, result))
+
+    @defer.inlineCallbacks
+    def on_PUT(self, request, user_id):
+        yield self.auth.get_user_by_req(request)
+        defer.returnValue((200, {}))
+
+
 class KeyUploadServlet(RestServlet):
     PATTERNS = client_v2_patterns("/keys/upload(/(?P<device_id>[^/]+))?$")
 
@@ -131,10 +161,11 @@ class FrontendProxyServer(HomeServer):
         for res in listener_config["resources"]:
             for name in res["names"]:
                 if name == "metrics":
-                    resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
+                    resources[METRICS_PREFIX] = MetricsResource(self)
                 elif name == "client":
                     resource = JsonResource(self, canonical_json=False)
                     KeyUploadServlet(self).register(resource)
+                    PresenceStatusStubServlet(self).register(resource)
                     resources.update({
                         "/_matrix/client/r0": resource,
                         "/_matrix/client/unstable": resource,
@@ -152,7 +183,6 @@ class FrontendProxyServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
-                self.version_string,
             )
         )
 
@@ -172,13 +202,6 @@ class FrontendProxyServer(HomeServer):
                         globals={"hs": self},
                     )
                 )
-            elif listener["type"] == "metrics":
-                if not self.get_config().enable_metrics:
-                    logger.warn(("Metrics listener configured, but "
-                                 "enable_metrics is not True!"))
-                else:
-                    _base.listen_metrics(listener["bind_addresses"],
-                                         listener["port"])
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])
 
@@ -208,13 +231,11 @@ def start(config_options):
     database_engine = create_engine(config.database_config)
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
 
     ss = FrontendProxyServer(
         config.server_name,
         db_config=config.database_config,
         tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,

synapse/app/homeserver.py

@@ -18,44 +18,27 @@ import logging
 import os
 import sys
 
-from six import iteritems
-
-from prometheus_client import Gauge
-
-from twisted.application import service
-from twisted.internet import defer, reactor
-from twisted.web.resource import EncodingResourceWrapper, NoResource
-from twisted.web.server import GzipEncoderFactory
-from twisted.web.static import File
-
 import synapse
 import synapse.config.logger
 from synapse import events
-from synapse.api.urls import (
-    CONTENT_REPO_PREFIX,
-    FEDERATION_PREFIX,
-    LEGACY_MEDIA_PREFIX,
-    MEDIA_PREFIX,
-    SERVER_KEY_PREFIX,
-    SERVER_KEY_V2_PREFIX,
-    STATIC_PREFIX,
-    WEB_CLIENT_PREFIX,
-)
+from synapse.api.urls import CONTENT_REPO_PREFIX, FEDERATION_PREFIX, \
+    LEGACY_MEDIA_PREFIX, MEDIA_PREFIX, SERVER_KEY_PREFIX, SERVER_KEY_V2_PREFIX, \
+    STATIC_PREFIX, WEB_CLIENT_PREFIX
 from synapse.app import _base
-from synapse.app._base import listen_ssl, listen_tcp, quit_with_error
+from synapse.app._base import quit_with_error, listen_ssl, listen_tcp
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.crypto import context_factory
 from synapse.federation.transport.server import TransportLayerServer
+from synapse.module_api import ModuleApi
 from synapse.http.additional_resource import AdditionalResource
 from synapse.http.server import RootRedirect
 from synapse.http.site import SynapseSite
-from synapse.metrics import RegistryProxy
-from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.metrics import register_memory_metrics
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
-from synapse.module_api import ModuleApi
-from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, check_requirements
-from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
+from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, \
+    check_requirements
+from synapse.replication.http import ReplicationRestResource, REPLICATION_PREFIX
 from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
 from synapse.rest import ClientRestResource
 from synapse.rest.key.v1.server_key_resource import LocalKey
@@ -72,6 +55,11 @@ from synapse.util.manhole import manhole
 from synapse.util.module_loader import load_module
 from synapse.util.rlimit import change_resource_limit
 from synapse.util.versionstring import get_version_string
+from twisted.application import service
+from twisted.internet import defer, reactor
+from twisted.web.resource import EncodingResourceWrapper, NoResource
+from twisted.web.server import GzipEncoderFactory
+from twisted.web.static import File
 
 logger = logging.getLogger("synapse.app.homeserver")
 
@@ -152,7 +140,6 @@ class SynapseHomeServer(HomeServer):
                     site_tag,
                     listener_config,
                     root_resource,
-                    self.version_string,
                 ),
                 self.tls_server_context_factory,
             )
@@ -166,7 +153,6 @@ class SynapseHomeServer(HomeServer):
                     site_tag,
                     listener_config,
                     root_resource,
-                    self.version_string,
                 )
             )
         logger.info("Synapse now listening on port %d", port)
@@ -196,15 +182,6 @@ class SynapseHomeServer(HomeServer):
                 "/_matrix/client/versions": client_resource,
             })
 
-        if name == "consent":
-            from synapse.rest.consent.consent_resource import ConsentResource
-            consent_resource = ConsentResource(self)
-            if compress:
-                consent_resource = gz_wrap(consent_resource)
-            resources.update({
-                "/_matrix/consent": consent_resource,
-            })
-
         if name == "federation":
             resources.update({
                 FEDERATION_PREFIX: TransportLayerServer(self),
@@ -242,7 +219,7 @@ class SynapseHomeServer(HomeServer):
             resources[WEB_CLIENT_PREFIX] = build_resource_for_web_client(self)
 
         if name == "metrics" and self.get_config().enable_metrics:
-            resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
+            resources[METRICS_PREFIX] = MetricsResource(self)
 
         if name == "replication":
             resources[REPLICATION_PREFIX] = ReplicationRestResource(self)
@@ -275,13 +252,6 @@ class SynapseHomeServer(HomeServer):
                     reactor.addSystemEventTrigger(
                         "before", "shutdown", server_listener.stopListening,
                     )
-            elif listener["type"] == "metrics":
-                if not self.get_config().enable_metrics:
-                    logger.warn(("Metrics listener configured, but "
-                                 "enable_metrics is not True!"))
-                else:
-                    _base.listen_metrics(listener["bind_addresses"],
-                                         listener["port"])
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])
 
@@ -302,11 +272,6 @@ class SynapseHomeServer(HomeServer):
             quit_with_error(e.message)
 
 
-# Gauges to expose monthly active user control metrics
-current_mau_gauge = Gauge("synapse_admin_mau:current", "Current MAU")
-max_mau_gauge = Gauge("synapse_admin_mau:max", "MAU Limit")
-
-
 def setup(config_options):
     """
     Args:
@@ -335,10 +300,14 @@ def setup(config_options):
     # check any extra requirements we have now we have a config
     check_requirements(config)
 
+    version_string = "Synapse/" + get_version_string(synapse)
+
+    logger.info("Server hostname: %s", config.server_name)
+    logger.info("Server version: %s", version_string)
+
     events.USE_FROZEN_DICTS = config.use_frozen_dicts
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
 
     database_engine = create_engine(config.database_config)
     config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection
@@ -347,9 +316,8 @@ def setup(config_options):
         config.server_name,
         db_config=config.database_config,
         tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
         config=config,
-        version_string="Synapse/" + get_version_string(synapse),
+        version_string=version_string,
         database_engine=database_engine,
     )
 
@@ -383,6 +351,8 @@ def setup(config_options):
         hs.get_datastore().start_doing_background_updates()
         hs.get_federation_client().start_get_pdu_cache()
 
+        register_memory_metrics(hs)
+
     reactor.callWhenRunning(start)
 
     return hs
@@ -437,9 +407,6 @@ def run(hs):
     # currently either 0 or 1
     stats_process = []
 
-    def start_phone_stats_home():
-        return run_as_background_process("phone_stats_home", phone_stats_home)
-
     @defer.inlineCallbacks
     def phone_stats_home():
         logger.info("Gathering stats for reporting")
@@ -456,10 +423,6 @@ def run(hs):
         total_nonbridged_users = yield hs.get_datastore().count_nonbridged_users()
         stats["total_nonbridged_users"] = total_nonbridged_users
 
-        daily_user_type_results = yield hs.get_datastore().count_daily_user_type()
-        for name, count in iteritems(daily_user_type_results):
-            stats["daily_user_type_" + name] = count
-
         room_count = yield hs.get_datastore().get_room_count()
         stats["total_room_count"] = room_count
 
@@ -468,7 +431,7 @@ def run(hs):
         stats["daily_messages"] = yield hs.get_datastore().count_daily_messages()
 
         r30_results = yield hs.get_datastore().count_r30_users()
-        for name, count in iteritems(r30_results):
+        for name, count in r30_results.iteritems():
             stats["r30_users_" + name] = count
 
         daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages()
@@ -510,41 +473,9 @@ def run(hs):
                 " changes across releases."
             )
 
-    def generate_user_daily_visit_stats():
-        return run_as_background_process(
-            "generate_user_daily_visits",
-            hs.get_datastore().generate_user_daily_visits,
-        )
-
-    # Rather than update on per session basis, batch up the requests.
-    # If you increase the loop period, the accuracy of user_daily_visits
-    # table will decrease
-    clock.looping_call(generate_user_daily_visit_stats, 5 * 60 * 1000)
-
-    # monthly active user limiting functionality
-    clock.looping_call(
-        hs.get_datastore().reap_monthly_active_users, 1000 * 60 * 60
-    )
-
-    @defer.inlineCallbacks
-    def generate_monthly_active_users():
-        count = 0
-        if hs.config.limit_usage_by_mau:
-            count = yield hs.get_datastore().get_monthly_active_count()
-        current_mau_gauge.set(float(count))
-        max_mau_gauge.set(float(hs.config.max_mau_value))
-
-    hs.get_datastore().initialise_reserved_users(
-        hs.config.mau_limits_reserved_threepids
-    )
-    generate_monthly_active_users()
-    if hs.config.limit_usage_by_mau:
-        clock.looping_call(generate_monthly_active_users, 5 * 60 * 1000)
-    # End of monthly active user settings
-
     if hs.config.report_stats:
         logger.info("Scheduling stats reporting for 3 hour intervals")
-        clock.looping_call(start_phone_stats_home, 3 * 60 * 60 * 1000)
+        clock.looping_call(phone_stats_home, 3 * 60 * 60 * 1000)
 
         # We need to defer this init for the cases that we daemonize
         # otherwise the process ID we get is that of the non-daemon process
@@ -552,7 +483,7 @@ def run(hs):
 
         # We wait 5 minutes to send the first set of stats as the server can
         # be quite busy the first few minutes
-        clock.call_later(5 * 60, start_phone_stats_home)
+        clock.call_later(5 * 60, phone_stats_home)
 
     if hs.config.daemonize and hs.config.print_pidfile:
         print (hs.config.pid_file)

synapse/app/media_repository.py

@@ -16,25 +16,23 @@
 import logging
 import sys
 
-from twisted.internet import reactor
-from twisted.web.resource import NoResource
-
 import synapse
 from synapse import events
-from synapse.api.urls import CONTENT_REPO_PREFIX, LEGACY_MEDIA_PREFIX, MEDIA_PREFIX
+from synapse.api.urls import (
+    CONTENT_REPO_PREFIX, LEGACY_MEDIA_PREFIX, MEDIA_PREFIX
+)
 from synapse.app import _base
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
 from synapse.crypto import context_factory
 from synapse.http.site import SynapseSite
-from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
 from synapse.replication.slave.storage.client_ips import SlavedClientIpStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
-from synapse.replication.slave.storage.transactions import SlavedTransactionStore
+from synapse.replication.slave.storage.transactions import TransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
 from synapse.rest.media.v0.content_repository import ContentRepoResource
 from synapse.server import HomeServer
@@ -44,6 +42,8 @@ from synapse.util.httpresourcetree import create_resource_tree
 from synapse.util.logcontext import LoggingContext
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
+from twisted.internet import reactor
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.media_repository")
 
@@ -52,7 +52,7 @@ class MediaRepositorySlavedStore(
     SlavedApplicationServiceStore,
     SlavedRegistrationStore,
     SlavedClientIpStore,
-    SlavedTransactionStore,
+    TransactionStore,
     BaseSlavedStore,
     MediaRepositoryStore,
 ):
@@ -73,7 +73,7 @@ class MediaRepositoryServer(HomeServer):
         for res in listener_config["resources"]:
             for name in res["names"]:
                 if name == "metrics":
-                    resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
+                    resources[METRICS_PREFIX] = MetricsResource(self)
                 elif name == "media":
                     media_repo = self.get_media_repository_resource()
                     resources.update({
@@ -94,7 +94,6 @@ class MediaRepositoryServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
-                self.version_string,
             )
         )
 
@@ -114,13 +113,6 @@ class MediaRepositoryServer(HomeServer):
                         globals={"hs": self},
                     )
                 )
-            elif listener["type"] == "metrics":
-                if not self.get_config().enable_metrics:
-                    logger.warn(("Metrics listener configured, but "
-                                 "enable_metrics is not True!"))
-                else:
-                    _base.listen_metrics(listener["bind_addresses"],
-                                         listener["port"])
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])
 
@@ -155,13 +147,11 @@ def start(config_options):
     database_engine = create_engine(config.database_config)
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
 
     ss = MediaRepositoryServer(
         config.server_name,
         db_config=config.database_config,
         tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,

synapse/app/pusher.py

@@ -16,9 +16,6 @@
 import logging
 import sys
 
-from twisted.internet import defer, reactor
-from twisted.web.resource import NoResource
-
 import synapse
 from synapse import events
 from synapse.app import _base
@@ -26,7 +23,6 @@ from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
 from synapse.http.site import SynapseSite
-from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.events import SlavedEventStore
@@ -37,9 +33,11 @@ from synapse.server import HomeServer
 from synapse.storage import DataStore
 from synapse.storage.engines import create_engine
 from synapse.util.httpresourcetree import create_resource_tree
-from synapse.util.logcontext import LoggingContext, run_in_background
+from synapse.util.logcontext import LoggingContext, preserve_fn
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
+from twisted.internet import defer, reactor
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.pusher")
 
@@ -94,7 +92,7 @@ class PusherServer(HomeServer):
         for res in listener_config["resources"]:
             for name in res["names"]:
                 if name == "metrics":
-                    resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
+                    resources[METRICS_PREFIX] = MetricsResource(self)
 
         root_resource = create_resource_tree(resources, NoResource())
 
@@ -106,7 +104,6 @@ class PusherServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
-                self.version_string,
             )
         )
 
@@ -126,13 +123,6 @@ class PusherServer(HomeServer):
                         globals={"hs": self},
                     )
                 )
-            elif listener["type"] == "metrics":
-                if not self.get_config().enable_metrics:
-                    logger.warn(("Metrics listener configured, but "
-                                 "enable_metrics is not True!"))
-                else:
-                    _base.listen_metrics(listener["bind_addresses"],
-                                         listener["port"])
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])
 
@@ -150,27 +140,24 @@ class PusherReplicationHandler(ReplicationClientHandler):
 
     def on_rdata(self, stream_name, token, rows):
         super(PusherReplicationHandler, self).on_rdata(stream_name, token, rows)
-        run_in_background(self.poke_pushers, stream_name, token, rows)
+        preserve_fn(self.poke_pushers)(stream_name, token, rows)
 
     @defer.inlineCallbacks
     def poke_pushers(self, stream_name, token, rows):
-        try:
-            if stream_name == "pushers":
-                for row in rows:
-                    if row.deleted:
-                        yield self.stop_pusher(row.user_id, row.app_id, row.pushkey)
-                    else:
-                        yield self.start_pusher(row.user_id, row.app_id, row.pushkey)
-            elif stream_name == "events":
-                yield self.pusher_pool.on_new_notifications(
-                    token, token,
-                )
-            elif stream_name == "receipts":
-                yield self.pusher_pool.on_new_receipts(
-                    token, token, set(row.room_id for row in rows)
-                )
-        except Exception:
-            logger.exception("Error poking pushers")
+        if stream_name == "pushers":
+            for row in rows:
+                if row.deleted:
+                    yield self.stop_pusher(row.user_id, row.app_id, row.pushkey)
+                else:
+                    yield self.start_pusher(row.user_id, row.app_id, row.pushkey)
+        elif stream_name == "events":
+            yield self.pusher_pool.on_new_notifications(
+                token, token,
+            )
+        elif stream_name == "receipts":
+            yield self.pusher_pool.on_new_receipts(
+                token, token, set(row.room_id for row in rows)
+            )
 
     def stop_pusher(self, user_id, app_id, pushkey):
         key = "%s:%s" % (app_id, pushkey)

synapse/app/synchrotron.py

@@ -17,11 +17,6 @@ import contextlib
 import logging
 import sys
 
-from six import iteritems
-
-from twisted.internet import defer, reactor
-from twisted.web.resource import NoResource
-
 import synapse
 from synapse.api.constants import EventTypes
 from synapse.app import _base
@@ -31,7 +26,6 @@ from synapse.config.logger import setup_logging
 from synapse.handlers.presence import PresenceHandler, get_interested_parties
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
-from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
@@ -41,12 +35,12 @@ from synapse.replication.slave.storage.deviceinbox import SlavedDeviceInboxStore
 from synapse.replication.slave.storage.devices import SlavedDeviceStore
 from synapse.replication.slave.storage.events import SlavedEventStore
 from synapse.replication.slave.storage.filtering import SlavedFilteringStore
-from synapse.replication.slave.storage.groups import SlavedGroupServerStore
 from synapse.replication.slave.storage.presence import SlavedPresenceStore
 from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
 from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.replication.slave.storage.room import RoomStore
+from synapse.replication.slave.storage.groups import SlavedGroupServerStore
 from synapse.replication.tcp.client import ReplicationClientHandler
 from synapse.rest.client.v1 import events
 from synapse.rest.client.v1.initial_sync import InitialSyncRestServlet
@@ -55,11 +49,14 @@ from synapse.rest.client.v2_alpha import sync
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
 from synapse.storage.presence import UserPresenceState
+from synapse.storage.roommember import RoomMemberStore
 from synapse.util.httpresourcetree import create_resource_tree
-from synapse.util.logcontext import LoggingContext, run_in_background
+from synapse.util.logcontext import LoggingContext, preserve_fn
 from synapse.util.manhole import manhole
 from synapse.util.stringutils import random_string
 from synapse.util.versionstring import get_version_string
+from twisted.internet import defer, reactor
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.synchrotron")
 
@@ -80,7 +77,9 @@ class SynchrotronSlavedStore(
     RoomStore,
     BaseSlavedStore,
 ):
-    pass
+    did_forget = (
+        RoomMemberStore.__dict__["did_forget"]
+    )
 
 
 UPDATE_SYNCING_USERS_MS = 10 * 1000
@@ -114,6 +113,7 @@ class SynchrotronPresence(object):
         logger.info("Presence process_id is %r", self.process_id)
 
     def send_user_sync(self, user_id, is_syncing, last_sync_ms):
+        return
         self.hs.get_tcp_replication().send_user_sync(user_id, is_syncing, last_sync_ms)
 
     def mark_as_coming_online(self, user_id):
@@ -211,8 +211,10 @@ class SynchrotronPresence(object):
         yield self.notify_from_replication(states, stream_id)
 
     def get_currently_syncing_users(self):
+        # presence is disabled on matrix.org, so we return the empty set
+        return set()
         return [
-            user_id for user_id, count in iteritems(self.user_to_num_current_syncs)
+            user_id for user_id, count in self.user_to_num_current_syncs.iteritems()
             if count > 0
         ]
 
@@ -256,7 +258,7 @@ class SynchrotronServer(HomeServer):
         for res in listener_config["resources"]:
             for name in res["names"]:
                 if name == "metrics":
-                    resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
+                    resources[METRICS_PREFIX] = MetricsResource(self)
                 elif name == "client":
                     resource = JsonResource(self, canonical_json=False)
                     sync.register_servlets(self, resource)
@@ -280,7 +282,6 @@ class SynchrotronServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
-                self.version_string,
             )
         )
 
@@ -300,13 +301,6 @@ class SynchrotronServer(HomeServer):
                         globals={"hs": self},
                     )
                 )
-            elif listener["type"] == "metrics":
-                if not self.get_config().enable_metrics:
-                    logger.warn(("Metrics listener configured, but "
-                                 "enable_metrics is not True!"))
-                else:
-                    _base.listen_metrics(listener["bind_addresses"],
-                                         listener["port"])
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])
 
@@ -334,7 +328,8 @@ class SyncReplicationHandler(ReplicationClientHandler):
 
     def on_rdata(self, stream_name, token, rows):
         super(SyncReplicationHandler, self).on_rdata(stream_name, token, rows)
-        run_in_background(self.process_and_notify, stream_name, token, rows)
+
+        preserve_fn(self.process_and_notify)(stream_name, token, rows)
 
     def get_streams_to_replicate(self):
         args = super(SyncReplicationHandler, self).get_streams_to_replicate()
@@ -346,58 +341,55 @@ class SyncReplicationHandler(ReplicationClientHandler):
 
     @defer.inlineCallbacks
     def process_and_notify(self, stream_name, token, rows):
-        try:
-            if stream_name == "events":
-                # We shouldn't get multiple rows per token for events stream, so
-                # we don't need to optimise this for multiple rows.
-                for row in rows:
-                    event = yield self.store.get_event(row.event_id)
-                    extra_users = ()
-                    if event.type == EventTypes.Member:
-                        extra_users = (event.state_key,)
-                    max_token = self.store.get_room_max_stream_ordering()
-                    self.notifier.on_new_room_event(
-                        event, token, max_token, extra_users
-                    )
-            elif stream_name == "push_rules":
-                self.notifier.on_new_event(
-                    "push_rules_key", token, users=[row.user_id for row in rows],
+        if stream_name == "events":
+            # We shouldn't get multiple rows per token for events stream, so
+            # we don't need to optimise this for multiple rows.
+            for row in rows:
+                event = yield self.store.get_event(row.event_id)
+                extra_users = ()
+                if event.type == EventTypes.Member:
+                    extra_users = (event.state_key,)
+                max_token = self.store.get_room_max_stream_ordering()
+                self.notifier.on_new_room_event(
+                    event, token, max_token, extra_users
                 )
-            elif stream_name in ("account_data", "tag_account_data",):
+        elif stream_name == "push_rules":
+            self.notifier.on_new_event(
+                "push_rules_key", token, users=[row.user_id for row in rows],
+            )
+        elif stream_name in ("account_data", "tag_account_data",):
+            self.notifier.on_new_event(
+                "account_data_key", token, users=[row.user_id for row in rows],
+            )
+        elif stream_name == "receipts":
+            self.notifier.on_new_event(
+                "receipt_key", token, rooms=[row.room_id for row in rows],
+            )
+        elif stream_name == "typing":
+            self.typing_handler.process_replication_rows(token, rows)
+            self.notifier.on_new_event(
+                "typing_key", token, rooms=[row.room_id for row in rows],
+            )
+        elif stream_name == "to_device":
+            entities = [row.entity for row in rows if row.entity.startswith("@")]
+            if entities:
                 self.notifier.on_new_event(
-                    "account_data_key", token, users=[row.user_id for row in rows],
+                    "to_device_key", token, users=entities,
                 )
-            elif stream_name == "receipts":
-                self.notifier.on_new_event(
-                    "receipt_key", token, rooms=[row.room_id for row in rows],
-                )
-            elif stream_name == "typing":
-                self.typing_handler.process_replication_rows(token, rows)
-                self.notifier.on_new_event(
-                    "typing_key", token, rooms=[row.room_id for row in rows],
-                )
-            elif stream_name == "to_device":
-                entities = [row.entity for row in rows if row.entity.startswith("@")]
-                if entities:
-                    self.notifier.on_new_event(
-                        "to_device_key", token, users=entities,
-                    )
-            elif stream_name == "device_lists":
-                all_room_ids = set()
-                for row in rows:
-                    room_ids = yield self.store.get_rooms_for_user(row.user_id)
-                    all_room_ids.update(room_ids)
-                self.notifier.on_new_event(
-                    "device_list_key", token, rooms=all_room_ids,
-                )
-            elif stream_name == "presence":
-                yield self.presence_handler.process_replication_rows(token, rows)
-            elif stream_name == "receipts":
-                self.notifier.on_new_event(
-                    "groups_key", token, users=[row.user_id for row in rows],
-                )
-        except Exception:
-            logger.exception("Error processing replication")
+        elif stream_name == "device_lists":
+            all_room_ids = set()
+            for row in rows:
+                room_ids = yield self.store.get_rooms_for_user(row.user_id)
+                all_room_ids.update(room_ids)
+            self.notifier.on_new_event(
+                "device_list_key", token, rooms=all_room_ids,
+            )
+        elif stream_name == "presence":
+            yield self.presence_handler.process_replication_rows(token, rows)
+        elif stream_name == "receipts":
+            self.notifier.on_new_event(
+                "groups_key", token, users=[row.user_id for row in rows],
+            )
 
 
 def start(config_options):

synapse/app/synctl.py

@@ -16,18 +16,15 @@
 
 import argparse
 import collections
-import errno
 import glob
 import os
 import os.path
 import signal
 import subprocess
 import sys
-import time
-
-from six import iteritems
-
 import yaml
+import errno
+import time
 
 SYNAPSE = [sys.executable, "-B", "-m", "synapse.app.homeserver"]
 
@@ -111,7 +108,7 @@ def stop(pidfile, app):
 
 
 Worker = collections.namedtuple("Worker", [
-    "app", "configfile", "pidfile", "cache_factor"
+    "app", "configfile", "pidfile", "cache_factor", "cache_factors",
 ])
 
 
@@ -175,7 +172,7 @@ def main():
         os.environ["SYNAPSE_CACHE_FACTOR"] = str(cache_factor)
 
     cache_factors = config.get("synctl_cache_factors", {})
-    for cache_name, factor in iteritems(cache_factors):
+    for cache_name, factor in cache_factors.iteritems():
         os.environ["SYNAPSE_CACHE_FACTOR_" + cache_name.upper()] = str(factor)
 
     worker_configfiles = []
@@ -218,6 +215,10 @@ def main():
                 or pidfile
             )
             worker_cache_factor = worker_config.get("synctl_cache_factor") or cache_factor
+            worker_cache_factors = (
+                worker_config.get("synctl_cache_factors")
+                or cache_factors
+            )
             daemonize = worker_config.get("daemonize") or config.get("daemonize")
             assert daemonize, "Main process must have daemonize set to true"
 
@@ -233,8 +234,10 @@ def main():
             assert worker_daemonize, "In config %r: expected '%s' to be True" % (
                 worker_configfile, "worker_daemonize")
             worker_cache_factor = worker_config.get("synctl_cache_factor")
+            worker_cache_factors = worker_config.get("synctl_cache_factors", {})
         workers.append(Worker(
             worker_app, worker_configfile, worker_pidfile, worker_cache_factor,
+            worker_cache_factors,
         ))
 
     action = options.action
@@ -269,15 +272,19 @@ def main():
             start(configfile)
 
         for worker in workers:
+            env = os.environ.copy()
+
             if worker.cache_factor:
                 os.environ["SYNAPSE_CACHE_FACTOR"] = str(worker.cache_factor)
 
+            for cache_name, factor in worker.cache_factors.iteritems():
+                os.environ["SYNAPSE_CACHE_FACTOR_" + cache_name.upper()] = str(factor)
+
             start_worker(worker.app, configfile, worker.configfile)
 
-            if cache_factor:
-                os.environ["SYNAPSE_CACHE_FACTOR"] = str(cache_factor)
-            else:
-                os.environ.pop("SYNAPSE_CACHE_FACTOR", None)
+            # Reset env back to the original
+            os.environ.clear()
+            os.environ.update(env)
 
 
 if __name__ == "__main__":

synapse/app/user_dir.py

@@ -17,9 +17,6 @@
 import logging
 import sys
 
-from twisted.internet import defer, reactor
-from twisted.web.resource import NoResource
-
 import synapse
 from synapse import events
 from synapse.app import _base
@@ -29,7 +26,6 @@ from synapse.config.logger import setup_logging
 from synapse.crypto import context_factory
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
-from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
@@ -43,9 +39,11 @@ from synapse.storage.engines import create_engine
 from synapse.storage.user_directory import UserDirectoryStore
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 from synapse.util.httpresourcetree import create_resource_tree
-from synapse.util.logcontext import LoggingContext, run_in_background
+from synapse.util.logcontext import LoggingContext, preserve_fn
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
+from twisted.internet import reactor
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.user_dir")
 
@@ -107,7 +105,7 @@ class UserDirectoryServer(HomeServer):
         for res in listener_config["resources"]:
             for name in res["names"]:
                 if name == "metrics":
-                    resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
+                    resources[METRICS_PREFIX] = MetricsResource(self)
                 elif name == "client":
                     resource = JsonResource(self, canonical_json=False)
                     user_directory.register_servlets(self, resource)
@@ -128,7 +126,6 @@ class UserDirectoryServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
-                self.version_string,
             )
         )
 
@@ -148,13 +145,6 @@ class UserDirectoryServer(HomeServer):
                         globals={"hs": self},
                     )
                 )
-            elif listener["type"] == "metrics":
-                if not self.get_config().enable_metrics:
-                    logger.warn(("Metrics listener configured, but "
-                                 "enable_metrics is not True!"))
-                else:
-                    _base.listen_metrics(listener["bind_addresses"],
-                                         listener["port"])
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])
 
@@ -174,14 +164,7 @@ class UserDirectoryReplicationHandler(ReplicationClientHandler):
             stream_name, token, rows
         )
         if stream_name == "current_state_deltas":
-            run_in_background(self._notify_directory)
-
-    @defer.inlineCallbacks
-    def _notify_directory(self):
-        try:
-            yield self.user_directory.notify_new_event()
-        except Exception:
-            logger.exception("Error notifiying user directory of state update")
+            preserve_fn(self.user_directory.notify_new_event)()
 
 
 def start(config_options):
@@ -214,13 +197,11 @@ def start(config_options):
     config.update_user_directory = True
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
 
     ps = UserDirectoryServer(
         config.server_name,
         db_config=config.database_config,
         tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,

synapse/appservice/__init__.py

@@ -12,17 +12,17 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from synapse.api.constants import EventTypes
+from synapse.util.caches.descriptors import cachedInlineCallbacks
+from synapse.types import GroupID, get_domain_from_id
+
+from twisted.internet import defer
+
 import logging
 import re
 
 from six import string_types
 
-from twisted.internet import defer
-
-from synapse.api.constants import EventTypes
-from synapse.types import GroupID, get_domain_from_id
-from synapse.util.caches.descriptors import cachedInlineCallbacks
-
 logger = logging.getLogger(__name__)
 
 
@@ -85,8 +85,7 @@ class ApplicationService(object):
     NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS]
 
     def __init__(self, token, hostname, url=None, namespaces=None, hs_token=None,
-                 sender=None, id=None, protocols=None, rate_limited=True,
-                 ip_range_whitelist=None):
+                 sender=None, id=None, protocols=None, rate_limited=True):
         self.token = token
         self.url = url
         self.hs_token = hs_token
@@ -94,7 +93,6 @@ class ApplicationService(object):
         self.server_name = hostname
         self.namespaces = self._check_namespaces(namespaces)
         self.id = id
-        self.ip_range_whitelist = ip_range_whitelist
 
         if "|" in self.id:
             raise Exception("application service ID cannot contain '|' character")
@@ -294,8 +292,4 @@ class ApplicationService(object):
         return self.rate_limited
 
     def __str__(self):
-        # copy dictionary and redact token fields so they don't get logged
-        dict_copy = self.__dict__.copy()
-        dict_copy["token"] = "<redacted>"
-        dict_copy["hs_token"] = "<redacted>"
-        return "ApplicationService: %s" % (dict_copy,)
+        return "ApplicationService: %s" % (self.__dict__,)

synapse/appservice/api.py

@@ -12,39 +12,21 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import logging
-import urllib
-
-from prometheus_client import Counter
-
 from twisted.internet import defer
 
 from synapse.api.constants import ThirdPartyEntityKind
 from synapse.api.errors import CodeMessageException
-from synapse.events.utils import serialize_event
 from synapse.http.client import SimpleHttpClient
-from synapse.types import ThirdPartyInstanceID
+from synapse.events.utils import serialize_event
+from synapse.util.logcontext import preserve_fn, make_deferred_yieldable
 from synapse.util.caches.response_cache import ResponseCache
+from synapse.types import ThirdPartyInstanceID
+
+import logging
+import urllib
 
 logger = logging.getLogger(__name__)
 
-sent_transactions_counter = Counter(
-    "synapse_appservice_api_sent_transactions",
-    "Number of /transactions/ requests sent",
-    ["service"]
-)
-
-failed_transactions_counter = Counter(
-    "synapse_appservice_api_failed_transactions",
-    "Number of /transactions/ requests that failed to send",
-    ["service"]
-)
-
-sent_events_counter = Counter(
-    "synapse_appservice_api_sent_events",
-    "Number of events sent to the AS",
-    ["service"]
-)
 
 HOUR_IN_MS = 60 * 60 * 1000
 
@@ -212,7 +194,12 @@ class ApplicationServiceApi(SimpleHttpClient):
                 defer.returnValue(None)
 
         key = (service.id, protocol)
-        return self.protocol_meta_cache.wrap(key, _get)
+        result = self.protocol_meta_cache.get(key)
+        if not result:
+            result = self.protocol_meta_cache.set(
+                key, preserve_fn(_get)()
+            )
+        return make_deferred_yieldable(result)
 
     @defer.inlineCallbacks
     def push_bulk(self, service, events, txn_id=None):
@@ -238,15 +225,12 @@ class ApplicationServiceApi(SimpleHttpClient):
                 args={
                     "access_token": service.hs_token
                 })
-            sent_transactions_counter.labels(service.id).inc()
-            sent_events_counter.labels(service.id).inc(len(events))
             defer.returnValue(True)
             return
         except CodeMessageException as e:
             logger.warning("push_bulk to %s received %s", uri, e.code)
         except Exception as ex:
             logger.warning("push_bulk to %s threw exception %s", uri, ex)
-        failed_transactions_counter.labels(service.id).inc()
         defer.returnValue(False)
 
     def _serialize(self, events):

synapse/appservice/scheduler.py

@@ -48,14 +48,14 @@ UP & quit           +---------- YES                       SUCCESS
 This is all tied together by the AppServiceScheduler which DIs the required
 components.
 """
-import logging
-
 from twisted.internet import defer
 
 from synapse.appservice import ApplicationServiceState
-from synapse.util.logcontext import run_in_background
+from synapse.util.logcontext import preserve_fn
 from synapse.util.metrics import Measure
 
+import logging
+
 logger = logging.getLogger(__name__)
 
 
@@ -106,7 +106,7 @@ class _ServiceQueuer(object):
     def enqueue(self, service, event):
         # if this service isn't being sent something
         self.queued_events.setdefault(service.id, []).append(event)
-        run_in_background(self._send_request, service)
+        preserve_fn(self._send_request)(service)
 
     @defer.inlineCallbacks
     def _send_request(self, service):
@@ -152,10 +152,10 @@ class _TransactionController(object):
                 if sent:
                     yield txn.complete(self.store)
                 else:
-                    run_in_background(self._start_recoverer, service)
-        except Exception:
-            logger.exception("Error creating appservice transaction")
-            run_in_background(self._start_recoverer, service)
+                    preserve_fn(self._start_recoverer)(service)
+        except Exception as e:
+            logger.exception(e)
+            preserve_fn(self._start_recoverer)(service)
 
     @defer.inlineCallbacks
     def on_recovered(self, recoverer):
@@ -176,20 +176,17 @@ class _TransactionController(object):
 
     @defer.inlineCallbacks
     def _start_recoverer(self, service):
-        try:
-            yield self.store.set_appservice_state(
-                service,
-                ApplicationServiceState.DOWN
-            )
-            logger.info(
-                "Application service falling behind. Starting recoverer. AS ID %s",
-                service.id
-            )
-            recoverer = self.recoverer_fn(service, self.on_recovered)
-            self.add_recoverers([recoverer])
-            recoverer.recover()
-        except Exception:
-            logger.exception("Error starting AS recoverer")
+        yield self.store.set_appservice_state(
+            service,
+            ApplicationServiceState.DOWN
+        )
+        logger.info(
+            "Application service falling behind. Starting recoverer. AS ID %s",
+            service.id
+        )
+        recoverer = self.recoverer_fn(service, self.on_recovered)
+        self.add_recoverers([recoverer])
+        recoverer.recover()
 
     @defer.inlineCallbacks
     def _is_service_up(self, service):

synapse/config/__init__.py

@@ -12,9 +12,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from ._base import ConfigError
-
-# export ConfigError if somebody does import *
-# this is largely a fudge to stop PEP8 moaning about the import
-__all__ = ["ConfigError"]

synapse/config/_base.py

@@ -16,12 +16,11 @@
 import argparse
 import errno
 import os
+import yaml
 from textwrap import dedent
 
 from six import integer_types
 
-import yaml
-
 
 class ConfigError(Exception):
     pass
@@ -282,15 +281,15 @@ class Config(object):
                     )
                 if not cls.path_exists(config_dir_path):
                     os.makedirs(config_dir_path)
-                with open(config_path, "w") as config_file:
-                    config_str, config = obj.generate_config(
+                with open(config_path, "wb") as config_file:
+                    config_bytes, config = obj.generate_config(
                         config_dir_path=config_dir_path,
                         server_name=server_name,
                         report_stats=(config_args.report_stats == "yes"),
                         is_generating_file=True
                     )
                     obj.invoke_all("generate_files", config)
-                    config_file.write(config_str)
+                    config_file.write(config_bytes)
                 print((
                     "A config file has been generated in %r for server name"
                     " %r with corresponding SSL keys and self-signed"

synapse/config/api.py

@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.api.constants import EventTypes
-
 from ._base import Config
 
+from synapse.api.constants import EventTypes
+
 
 class ApiConfig(Config):
 

synapse/config/appservice.py

@@ -12,18 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import logging
-
-from six import string_types
-from six.moves.urllib import parse as urlparse
-
-import yaml
-from netaddr import IPSet
+from ._base import Config, ConfigError
 
 from synapse.appservice import ApplicationService
 from synapse.types import UserID
 
-from ._base import Config, ConfigError
+import urllib
+import yaml
+import logging
+
+from six import string_types
 
 logger = logging.getLogger(__name__)
 
@@ -107,7 +105,7 @@ def _load_appservice(hostname, as_info, config_filename):
         )
 
     localpart = as_info["sender_localpart"]
-    if urlparse.quote(localpart) != localpart:
+    if urllib.quote(localpart) != localpart:
         raise ValueError(
             "sender_localpart needs characters which are not URL encoded."
         )
@@ -156,13 +154,6 @@ def _load_appservice(hostname, as_info, config_filename):
             " will not receive events or queries.",
             config_filename,
         )
-
-    ip_range_whitelist = None
-    if as_info.get('ip_range_whitelist'):
-        ip_range_whitelist = IPSet(
-            as_info.get('ip_range_whitelist')
-        )
-
     return ApplicationService(
         token=as_info["as_token"],
         hostname=hostname,
@@ -172,6 +163,5 @@ def _load_appservice(hostname, as_info, config_filename):
         sender=user_id,
         id=as_info["id"],
         protocols=protocols,
-        rate_limited=rate_limited,
-        ip_range_whitelist=ip_range_whitelist,
+        rate_limited=rate_limited
     )

synapse/config/consent_config.py

@@ -1,88 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2018 New Vector Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from ._base import Config
-
-DEFAULT_CONFIG = """\
-# User Consent configuration
-#
-# for detailed instructions, see
-# https://github.com/matrix-org/synapse/blob/master/docs/consent_tracking.md
-#
-# Parts of this section are required if enabling the 'consent' resource under
-# 'listeners', in particular 'template_dir' and 'version'.
-#
-# 'template_dir' gives the location of the templates for the HTML forms.
-# This directory should contain one subdirectory per language (eg, 'en', 'fr'),
-# and each language directory should contain the policy document (named as
-# '<version>.html') and a success page (success.html).
-#
-# 'version' specifies the 'current' version of the policy document. It defines
-# the version to be served by the consent resource if there is no 'v'
-# parameter.
-#
-# 'server_notice_content', if enabled, will send a user a "Server Notice"
-# asking them to consent to the privacy policy. The 'server_notices' section
-# must also be configured for this to work. Notices will *not* be sent to
-# guest users unless 'send_server_notice_to_guests' is set to true.
-#
-# 'block_events_error', if set, will block any attempts to send events
-# until the user consents to the privacy policy. The value of the setting is
-# used as the text of the error.
-#
-# user_consent:
-#   template_dir: res/templates/privacy
-#   version: 1.0
-#   server_notice_content:
-#     msgtype: m.text
-#     body: >-
-#       To continue using this homeserver you must review and agree to the
-#       terms and conditions at %(consent_uri)s
-#   send_server_notice_to_guests: True
-#   block_events_error: >-
-#     To continue using this homeserver you must review and agree to the
-#     terms and conditions at %(consent_uri)s
-#
-"""
-
-
-class ConsentConfig(Config):
-    def __init__(self):
-        super(ConsentConfig, self).__init__()
-
-        self.user_consent_version = None
-        self.user_consent_template_dir = None
-        self.user_consent_server_notice_content = None
-        self.user_consent_server_notice_to_guests = False
-        self.block_events_without_consent_error = None
-
-    def read_config(self, config):
-        consent_config = config.get("user_consent")
-        if consent_config is None:
-            return
-        self.user_consent_version = str(consent_config["version"])
-        self.user_consent_template_dir = consent_config["template_dir"]
-        self.user_consent_server_notice_content = consent_config.get(
-            "server_notice_content",
-        )
-        self.block_events_without_consent_error = consent_config.get(
-            "block_events_error",
-        )
-        self.user_consent_server_notice_to_guests = bool(consent_config.get(
-            "send_server_notice_to_guests", False,
-        ))
-
-    def default_config(self, **kwargs):
-        return DEFAULT_CONFIG

synapse/config/homeserver.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,32 +12,31 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from .tls import TlsConfig
+from .server import ServerConfig
+from .logger import LoggingConfig
+from .database import DatabaseConfig
+from .ratelimiting import RatelimitConfig
+from .repository import ContentRepositoryConfig
+from .captcha import CaptchaConfig
+from .voip import VoipConfig
+from .registration import RegistrationConfig
+from .metrics import MetricsConfig
 from .api import ApiConfig
 from .appservice import AppServiceConfig
-from .captcha import CaptchaConfig
-from .cas import CasConfig
-from .consent_config import ConsentConfig
-from .database import DatabaseConfig
-from .emailconfig import EmailConfig
-from .groups import GroupsConfig
-from .jwt import JWTConfig
 from .key import KeyConfig
-from .logger import LoggingConfig
-from .metrics import MetricsConfig
-from .password import PasswordConfig
-from .password_auth_providers import PasswordAuthProviderConfig
-from .push import PushConfig
-from .ratelimiting import RatelimitConfig
-from .registration import RegistrationConfig
-from .repository import ContentRepositoryConfig
 from .saml2 import SAML2Config
-from .server import ServerConfig
-from .server_notices_config import ServerNoticesConfig
-from .spam_checker import SpamCheckerConfig
-from .tls import TlsConfig
-from .user_directory import UserDirectoryConfig
-from .voip import VoipConfig
+from .cas import CasConfig
+from .password import PasswordConfig
+from .jwt import JWTConfig
+from .password_auth_providers import PasswordAuthProviderConfig
+from .emailconfig import EmailConfig
 from .workers import WorkerConfig
+from .push import PushConfig
+from .spam_checker import SpamCheckerConfig
+from .groups import GroupsConfig
+from .user_directory import UserDirectoryConfig
 
 
 class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig,
@@ -47,15 +45,12 @@ class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig,
                        AppServiceConfig, KeyConfig, SAML2Config, CasConfig,
                        JWTConfig, PasswordConfig, EmailConfig,
                        WorkerConfig, PasswordAuthProviderConfig, PushConfig,
-                       SpamCheckerConfig, GroupsConfig, UserDirectoryConfig,
-                       ConsentConfig,
-                       ServerNoticesConfig,
-                       ):
+                       SpamCheckerConfig, GroupsConfig, UserDirectoryConfig,):
     pass
 
 
 if __name__ == '__main__':
     import sys
     sys.stdout.write(
-        HomeServerConfig().generate_config(sys.argv[1], sys.argv[2], True)[0]
+        HomeServerConfig().generate_config(sys.argv[1], sys.argv[2])[0]
     )

synapse/config/jwt.py

@@ -15,6 +15,7 @@
 
 from ._base import Config, ConfigError
 
+
 MISSING_JWT = (
     """Missing jwt library. This is required for jwt login.
 

synapse/config/key.py

@@ -13,24 +13,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import hashlib
-import logging
-import os
+from ._base import Config, ConfigError
 
+from synapse.util.stringutils import random_string
 from signedjson.key import (
-    NACL_ED25519,
-    decode_signing_key_base64,
-    decode_verify_key_bytes,
-    generate_signing_key,
-    is_signing_algorithm_supported,
-    read_signing_keys,
-    write_signing_keys,
+    generate_signing_key, is_signing_algorithm_supported,
+    decode_signing_key_base64, decode_verify_key_bytes,
+    read_signing_keys, write_signing_keys, NACL_ED25519
 )
 from unpaddedbase64 import decode_base64
+from synapse.util.stringutils import random_string_with_symbols
 
-from synapse.util.stringutils import random_string, random_string_with_symbols
+import os
+import hashlib
+import logging
 
-from ._base import Config, ConfigError
 
 logger = logging.getLogger(__name__)
 
@@ -62,20 +59,14 @@ class KeyConfig(Config):
 
         self.expire_access_token = config.get("expire_access_token", False)
 
-        # a secret which is used to calculate HMACs for form values, to stop
-        # falsification of values
-        self.form_secret = config.get("form_secret", None)
-
     def default_config(self, config_dir_path, server_name, is_generating_file=False,
                        **kwargs):
         base_key_name = os.path.join(config_dir_path, server_name)
 
         if is_generating_file:
             macaroon_secret_key = random_string_with_symbols(50)
-            form_secret = '"%s"' % random_string_with_symbols(50)
         else:
             macaroon_secret_key = None
-            form_secret = 'null'
 
         return """\
         macaroon_secret_key: "%(macaroon_secret_key)s"
@@ -83,10 +74,6 @@ class KeyConfig(Config):
         # Used to enable access token expiration.
         expire_access_token: False
 
-        # a secret which is used to calculate HMACs for form values, to stop
-        # falsification of values
-        form_secret: %(form_secret)s
-
         ## Signing Keys ##
 
         # Path to the signing key to sign messages with

synapse/config/logger.py

@@ -12,22 +12,17 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import logging
-import logging.config
-import os
-import signal
-import sys
-from string import Template
-
-import yaml
-
-from twisted.logger import STDLibLogObserver, globalLogBeginner
-
-import synapse
-from synapse.util.logcontext import LoggingContextFilter
-from synapse.util.versionstring import get_version_string
 
 from ._base import Config
+from synapse.util.logcontext import LoggingContextFilter
+from twisted.logger import globalLogBeginner, STDLibLogObserver
+import logging
+import logging.config
+import yaml
+from string import Template
+import os
+import signal
+
 
 DEFAULT_LOG_CONFIG = Template("""
 version: 1
@@ -122,7 +117,7 @@ class LoggingConfig(Config):
         log_config = config.get("log_config")
         if log_config and not os.path.exists(log_config):
             log_file = self.abspath("homeserver.log")
-            with open(log_config, "w") as log_config_file:
+            with open(log_config, "wb") as log_config_file:
                 log_config_file.write(
                     DEFAULT_LOG_CONFIG.substitute(log_file=log_file)
                 )
@@ -193,8 +188,9 @@ def setup_logging(config, use_worker_options=False):
 
         def sighup(signum, stack):
             # it might be better to use a file watcher or something for this.
+            logging.info("Reloading log config from %s due to SIGHUP",
+                         log_config)
             load_log_config()
-            logging.info("Reloaded log config from %s due to SIGHUP", log_config)
 
         load_log_config()
 
@@ -206,15 +202,6 @@ def setup_logging(config, use_worker_options=False):
     if getattr(signal, "SIGHUP"):
         signal.signal(signal.SIGHUP, sighup)
 
-    # make sure that the first thing we log is a thing we can grep backwards
-    # for
-    logging.warn("***** STARTING SERVER *****")
-    logging.warn(
-        "Server %s version %s",
-        sys.argv[0], get_version_string(synapse),
-    )
-    logging.info("Server hostname: %s", config.server_name)
-
     # It's critical to point twisted's internal logging somewhere, otherwise it
     # stacks up and leaks kup to 64K object;
     # see: https://twistedmatrix.com/trac/ticket/8164