Fix typo

Co-Authored-By: anoadragon453 <1342360+anoadragon453@users.noreply.github.com>

.circleci/config.yml

@@ -4,21 +4,19 @@ jobs:
     machine: true
     steps:
       - checkout
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG}-py2 .
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
-      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py2
       - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
   dockerhubuploadlatest:
     machine: true
     steps:
       - checkout
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest-py2 .
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest -t matrixdotorg/synapse:latest-py3 --build-arg PYTHON_VERSION=3.6 .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest-py3 --build-arg PYTHON_VERSION=3.6 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       - run: docker push matrixdotorg/synapse:latest
-      - run: docker push matrixdotorg/synapse:latest-py2
       - run: docker push matrixdotorg/synapse:latest-py3
   sytestpy2:
     docker:

.codecov.yml

@@ -1,15 +0,0 @@
-comment:
-  layout: "diff"
-
-coverage:
-  status:
-    project:
-      default:
-        target: 0  # Target % coverage, can be auto. Turned off for now
-        threshold: null
-        base: auto
-    patch:
-      default:
-        target: 0
-        threshold: null
-        base: auto

.coveragerc

@@ -1,7 +1,11 @@
 [run]
 branch = True
 parallel = True
-include = synapse/*
+source = synapse
+
+[paths]
+source=
+   coverage
 
 [report]
 precision = 2

.gitignore

@@ -1,36 +1,63 @@
-# filename patterns
-*~
-.*.swp
-.#*
-*.deb
-*.egg
-*.egg-info
-*.lock
 *.pyc
-*.tac
+.*.swp
+*~
+*.lock
+
+.DS_Store
 _trial_temp/
 _trial_temp*/
+logs/
+dbs/
+*.egg
+dist/
+docs/build/
+*.egg-info
 
-# stuff that is likely to exist when you run a server locally
-/*.signing.key
-/*.tls.crt
-/*.tls.key
-/uploads
-/media_store/
+cmdclient_config.json
+homeserver*.db
+homeserver*.log
+homeserver*.log.*
+homeserver*.pid
+/homeserver*.yaml
 
-# IDEs
-/.idea/
-/.ropeproject/
-/.vscode/
+*.signing.key
+*.tls.crt
+*.tls.dh
+*.tls.key
 
-# build products
-/.coverage*
-!/.coveragerc
-/.tox
-/build/
-/coverage.*
-/dist/
-/docs/build/
-/htmlcov
-/pip-wheel-metadata/
+.coverage
+.coverage.*
+!.coverage.rc
+htmlcov
 
+demo/*/*.db
+demo/*/*.log
+demo/*/*.log.*
+demo/*/*.pid
+demo/media_store.*
+demo/etc
+
+uploads
+cache
+
+.idea/
+media_store/
+
+*.tac
+
+build/
+venv/
+venv*/
+*venv/
+
+localhost-800*/
+static/client/register/register_config.js
+.tox
+
+env/
+*.config
+
+.vscode/
+.ropeproject/
+*.deb
+/debs

.travis.yml

@@ -1,4 +1,4 @@
-dist: xenial
+sudo: false
 language: python
 
 cache:
@@ -22,7 +22,6 @@ branches:
     - master
     - develop
     - /^release-v/
-    - rav/pg95
 
 # When running the tox environments that call Twisted Trial, we can pass the -j
 # flag to run the tests concurrently. We set this to 2 for CPU bound tests
@@ -30,68 +29,45 @@ branches:
 matrix:
   fast_finish: true
   include:
-  - name: "pep8"
-    python: 3.6
-    env: TOX_ENV="pep8,check_isort,packaging"
+  - python: 2.7
+    env: TOX_ENV=packaging
 
-  - name: "py2.7 / sqlite"
-    python: 2.7
+  - python: 3.6
+    env: TOX_ENV="pep8,check_isort"
+
+  - python: 2.7
     env: TOX_ENV=py27,codecov TRIAL_FLAGS="-j 2"
 
-  - name: "py2.7 / sqlite / olddeps"
-    python: 2.7
+  - python: 2.7
     env: TOX_ENV=py27-old TRIAL_FLAGS="-j 2"
 
-  - name: "py2.7 / postgres9.5"
-    python: 2.7
-    addons:
-      postgresql: "9.5"
+  - python: 2.7
     env: TOX_ENV=py27-postgres,codecov TRIAL_FLAGS="-j 4"
     services:
       - postgresql
 
-  - name: "py3.5 / sqlite"
-    python: 3.5
+  - python: 3.5
     env: TOX_ENV=py35,codecov TRIAL_FLAGS="-j 2"
 
-  - name: "py3.7 / sqlite"
-    python: 3.7
-    env: TOX_ENV=py37,codecov TRIAL_FLAGS="-j 2"
+  - python: 3.6
+    env: TOX_ENV=py36,codecov TRIAL_FLAGS="-j 2"
 
-  - name: "py3.7 / postgres9.4"
-    python: 3.7
-    addons:
-      postgresql: "9.4"
-    env: TOX_ENV=py37-postgres TRIAL_FLAGS="-j 4"
-    services:
-      - postgresql
-
-  - name: "py3.7 / postgres9.5"
-    python: 3.7
-    addons:
-      postgresql: "9.5"
-    env: TOX_ENV=py37-postgres,codecov TRIAL_FLAGS="-j 4"
+  - python: 3.6
+    env: TOX_ENV=py36-postgres,codecov TRIAL_FLAGS="-j 4"
     services:
       - postgresql
 
   - # we only need to check for the newsfragment if it's a PR build
     if: type = pull_request
-    name: "check-newsfragment"
     python: 3.6
-    script: scripts-dev/check-newsfragment
+    env: TOX_ENV=check-newsfragment
+    script:
+      - git remote set-branches --add origin develop
+      - git fetch origin develop
+      - tox -e $TOX_ENV
 
 install:
-  # this just logs the postgres version we will be testing against (if any)
-  - psql -At -U postgres -c 'select version();' || true
-
   - pip install tox
 
-  # if we don't have python3.6 in this environment, travis unhelpfully gives us
-  # a `python3.6` on our path which does nothing but spit out a warning. Tox
-  # tries to run it (even if we're not running a py36 env), so the build logs
-  # then have warnings which look like errors. To reduce the noise, remove the
-  # non-functional python3.6.
-  - ( ! command -v python3.6 || python3.6 --version ) &>/dev/null || rm -f $(command -v python3.6)
-
 script:
   - tox -e $TOX_ENV

CHANGES.md

@@ -1,194 +1,3 @@
-Synapse 0.99.2 (2019-03-01)
-===========================
-
-Features
---------
-
-- Added an HAProxy example in the reverse proxy documentation. Contributed by Benoît S. (“Benpro”). ([\#4541](https://github.com/matrix-org/synapse/issues/4541))
-- Add basic optional sentry integration. ([\#4632](https://github.com/matrix-org/synapse/issues/4632), [\#4694](https://github.com/matrix-org/synapse/issues/4694))
-- Transfer bans on room upgrade. ([\#4642](https://github.com/matrix-org/synapse/issues/4642))
-- Add configurable room list publishing rules. ([\#4647](https://github.com/matrix-org/synapse/issues/4647))
-- Support .well-known delegation when issuing certificates through ACME. ([\#4652](https://github.com/matrix-org/synapse/issues/4652))
-- Allow registration and login to be handled by a worker instance. ([\#4666](https://github.com/matrix-org/synapse/issues/4666), [\#4670](https://github.com/matrix-org/synapse/issues/4670), [\#4682](https://github.com/matrix-org/synapse/issues/4682))
-- Reduce the overhead of creating outbound federation connections over TLS by caching the TLS client options. ([\#4674](https://github.com/matrix-org/synapse/issues/4674))
-- Add prometheus metrics for number of outgoing EDUs, by type. ([\#4695](https://github.com/matrix-org/synapse/issues/4695))
-- Return correct error code when inviting a remote user to a room whose homeserver does not support the room version. ([\#4721](https://github.com/matrix-org/synapse/issues/4721))
-- Prevent showing rooms to other servers that were set to not federate. ([\#4746](https://github.com/matrix-org/synapse/issues/4746))
-
-
-Bugfixes
---------
-
-- Fix possible exception when paginating. ([\#4263](https://github.com/matrix-org/synapse/issues/4263))
-- The dependency checker now correctly reports a version mismatch for optional
-  dependencies, instead of reporting the dependency missing. ([\#4450](https://github.com/matrix-org/synapse/issues/4450))
-- Set CORS headers on .well-known requests. ([\#4651](https://github.com/matrix-org/synapse/issues/4651))
-- Fix kicking guest users on guest access revocation in worker mode. ([\#4667](https://github.com/matrix-org/synapse/issues/4667))
-- Fix an issue in the database migration script where the
-  `e2e_room_keys.is_verified` column wasn't considered as
-  a boolean. ([\#4680](https://github.com/matrix-org/synapse/issues/4680))
-- Fix TaskStopped exceptions in logs when outbound requests time out. ([\#4690](https://github.com/matrix-org/synapse/issues/4690))
-- Fix ACME config for python 2. ([\#4717](https://github.com/matrix-org/synapse/issues/4717))
-- Fix paginating over federation persisting incorrect state. ([\#4718](https://github.com/matrix-org/synapse/issues/4718))
-
-
-Internal Changes
-----------------
-
-- Run `black` to reformat user directory code. ([\#4635](https://github.com/matrix-org/synapse/issues/4635))
-- Reduce number of exceptions we log. ([\#4643](https://github.com/matrix-org/synapse/issues/4643), [\#4668](https://github.com/matrix-org/synapse/issues/4668))
-- Introduce upsert batching functionality in the database layer. ([\#4644](https://github.com/matrix-org/synapse/issues/4644))
-- Fix various spelling mistakes. ([\#4657](https://github.com/matrix-org/synapse/issues/4657))
-- Cleanup request exception logging. ([\#4669](https://github.com/matrix-org/synapse/issues/4669), [\#4737](https://github.com/matrix-org/synapse/issues/4737), [\#4738](https://github.com/matrix-org/synapse/issues/4738))
-- Improve replication performance by reducing cache invalidation traffic. ([\#4671](https://github.com/matrix-org/synapse/issues/4671), [\#4715](https://github.com/matrix-org/synapse/issues/4715), [\#4748](https://github.com/matrix-org/synapse/issues/4748))
-- Test against Postgres 9.5 as well as 9.4. ([\#4676](https://github.com/matrix-org/synapse/issues/4676))
-- Run unit tests against python 3.7. ([\#4677](https://github.com/matrix-org/synapse/issues/4677))
-- Attempt to clarify installation instructions/config. ([\#4681](https://github.com/matrix-org/synapse/issues/4681))
-- Clean up gitignores. ([\#4688](https://github.com/matrix-org/synapse/issues/4688))
-- Minor tweaks to acme docs. ([\#4689](https://github.com/matrix-org/synapse/issues/4689))
-- Improve the logging in the pusher process. ([\#4691](https://github.com/matrix-org/synapse/issues/4691))
-- Better checks on newsfragments. ([\#4698](https://github.com/matrix-org/synapse/issues/4698), [\#4750](https://github.com/matrix-org/synapse/issues/4750))
-- Avoid some redundant work when processing read receipts. ([\#4706](https://github.com/matrix-org/synapse/issues/4706))
-- Run `push_receipts_to_remotes` as background job. ([\#4707](https://github.com/matrix-org/synapse/issues/4707))
-- Add prometheus metrics for number of badge update pushes. ([\#4709](https://github.com/matrix-org/synapse/issues/4709))
-- Reduce pusher logging on startup ([\#4716](https://github.com/matrix-org/synapse/issues/4716))
-- Don't log exceptions when failing to fetch remote server keys. ([\#4722](https://github.com/matrix-org/synapse/issues/4722))
-- Correctly proxy exception in frontend_proxy worker. ([\#4723](https://github.com/matrix-org/synapse/issues/4723))
-- Add database version to phonehome stats. ([\#4753](https://github.com/matrix-org/synapse/issues/4753))
-
-
-Synapse 0.99.1.1 (2019-02-14)
-=============================
-
-Bugfixes
---------
-
-- Fix "TypeError: '>' not supported" when starting without an existing certificate.
-  Fix a bug where an existing certificate would be reprovisoned every day. ([\#4648](https://github.com/matrix-org/synapse/issues/4648))
-
-
-Synapse 0.99.1 (2019-02-14)
-===========================
-
-Features
---------
-
-- Include m.room.encryption on invites by default ([\#3902](https://github.com/matrix-org/synapse/issues/3902))
-- Federation OpenID listener resource can now be activated even if federation is disabled ([\#4420](https://github.com/matrix-org/synapse/issues/4420))
-- Synapse's ACME support will now correctly reprovision a certificate that approaches its expiry while Synapse is running. ([\#4522](https://github.com/matrix-org/synapse/issues/4522))
-- Add ability to update backup versions ([\#4580](https://github.com/matrix-org/synapse/issues/4580))
-- Allow the "unavailable" presence status for /sync.
-  This change makes Synapse compliant with r0.4.0 of the Client-Server specification. ([\#4592](https://github.com/matrix-org/synapse/issues/4592))
-- There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners ([\#4613](https://github.com/matrix-org/synapse/issues/4613), [\#4615](https://github.com/matrix-org/synapse/issues/4615), [\#4617](https://github.com/matrix-org/synapse/issues/4617), [\#4636](https://github.com/matrix-org/synapse/issues/4636))
-- The default configuration no longer requires TLS certificates. ([\#4614](https://github.com/matrix-org/synapse/issues/4614))
-
-
-Bugfixes
---------
-
-- Copy over room federation ability on room upgrade. ([\#4530](https://github.com/matrix-org/synapse/issues/4530))
-- Fix noisy "twisted.internet.task.TaskStopped" errors in logs ([\#4546](https://github.com/matrix-org/synapse/issues/4546))
-- Synapse is now tolerant of the `tls_fingerprints` option being None or not specified. ([\#4589](https://github.com/matrix-org/synapse/issues/4589))
-- Fix 'no unique or exclusion constraint' error ([\#4591](https://github.com/matrix-org/synapse/issues/4591))
-- Transfer Server ACLs on room upgrade. ([\#4608](https://github.com/matrix-org/synapse/issues/4608))
-- Fix failure to start when not TLS certificate was given even if TLS was disabled. ([\#4618](https://github.com/matrix-org/synapse/issues/4618))
-- Fix self-signed cert notice from generate-config. ([\#4625](https://github.com/matrix-org/synapse/issues/4625))
-- Fix performance of `user_ips` table deduplication background update ([\#4626](https://github.com/matrix-org/synapse/issues/4626), [\#4627](https://github.com/matrix-org/synapse/issues/4627))
-
-
-Internal Changes
-----------------
-
-- Change the user directory state query to use a filtered call to the db instead of a generic one. ([\#4462](https://github.com/matrix-org/synapse/issues/4462))
-- Reject federation transactions if they include more than 50 PDUs or 100 EDUs. ([\#4513](https://github.com/matrix-org/synapse/issues/4513))
-- Reduce duplication of ``synapse.app`` code. ([\#4567](https://github.com/matrix-org/synapse/issues/4567))
-- Fix docker upload job to push -py2 images. ([\#4576](https://github.com/matrix-org/synapse/issues/4576))
-- Add port configuration information to ACME instructions. ([\#4578](https://github.com/matrix-org/synapse/issues/4578))
-- Update MSC1711 FAQ to calrify .well-known usage ([\#4584](https://github.com/matrix-org/synapse/issues/4584))
-- Clean up default listener configuration ([\#4586](https://github.com/matrix-org/synapse/issues/4586))
-- Clarifications for reverse proxy docs ([\#4607](https://github.com/matrix-org/synapse/issues/4607))
-- Move ClientTLSOptionsFactory init out of `refresh_certificates` ([\#4611](https://github.com/matrix-org/synapse/issues/4611))
-- Fail cleanly if listener config lacks a 'port' ([\#4616](https://github.com/matrix-org/synapse/issues/4616))
-- Remove redundant entries from docker config ([\#4619](https://github.com/matrix-org/synapse/issues/4619))
-- README updates ([\#4621](https://github.com/matrix-org/synapse/issues/4621))
-
-
-Synapse 0.99.0 (2019-02-05)
-===========================
-
-Synapse v0.99.x is a precursor to the upcoming Synapse v1.0 release. It contains foundational changes to room architecture and the federation security model necessary to support the upcoming r0 release of the Server to Server API.
-
-Features
---------
-
-- Synapse's cipher string has been updated to require ECDH key exchange. Configuring and generating dh_params is no longer required, and they will be ignored. ([\#4229](https://github.com/matrix-org/synapse/issues/4229))
-- Synapse can now automatically provision TLS certificates via ACME (the protocol used by CAs like Let's Encrypt). ([\#4384](https://github.com/matrix-org/synapse/issues/4384), [\#4492](https://github.com/matrix-org/synapse/issues/4492), [\#4525](https://github.com/matrix-org/synapse/issues/4525), [\#4572](https://github.com/matrix-org/synapse/issues/4572), [\#4564](https://github.com/matrix-org/synapse/issues/4564), [\#4566](https://github.com/matrix-org/synapse/issues/4566), [\#4547](https://github.com/matrix-org/synapse/issues/4547), [\#4557](https://github.com/matrix-org/synapse/issues/4557))
-- Implement MSC1708 (.well-known routing for server-server federation) ([\#4408](https://github.com/matrix-org/synapse/issues/4408), [\#4409](https://github.com/matrix-org/synapse/issues/4409), [\#4426](https://github.com/matrix-org/synapse/issues/4426), [\#4427](https://github.com/matrix-org/synapse/issues/4427), [\#4428](https://github.com/matrix-org/synapse/issues/4428), [\#4464](https://github.com/matrix-org/synapse/issues/4464), [\#4468](https://github.com/matrix-org/synapse/issues/4468), [\#4487](https://github.com/matrix-org/synapse/issues/4487), [\#4488](https://github.com/matrix-org/synapse/issues/4488), [\#4489](https://github.com/matrix-org/synapse/issues/4489), [\#4497](https://github.com/matrix-org/synapse/issues/4497), [\#4511](https://github.com/matrix-org/synapse/issues/4511), [\#4516](https://github.com/matrix-org/synapse/issues/4516), [\#4520](https://github.com/matrix-org/synapse/issues/4520), [\#4521](https://github.com/matrix-org/synapse/issues/4521), [\#4539](https://github.com/matrix-org/synapse/issues/4539), [\#4542](https://github.com/matrix-org/synapse/issues/4542), [\#4544](https://github.com/matrix-org/synapse/issues/4544))
-- Search now includes results from predecessor rooms after a room upgrade. ([\#4415](https://github.com/matrix-org/synapse/issues/4415))
-- Config option to disable requesting MSISDN on registration. ([\#4423](https://github.com/matrix-org/synapse/issues/4423))
-- Add a metric for tracking event stream position of the user directory. ([\#4445](https://github.com/matrix-org/synapse/issues/4445))
-- Support exposing server capabilities in CS API (MSC1753, MSC1804) ([\#4472](https://github.com/matrix-org/synapse/issues/4472), [81b7e7eed](https://github.com/matrix-org/synapse/commit/81b7e7eed323f55d6550e7a270a9dc2c4c7b0fe0)))
-- Add support for room version 3 ([\#4483](https://github.com/matrix-org/synapse/issues/4483), [\#4499](https://github.com/matrix-org/synapse/issues/4499), [\#4515](https://github.com/matrix-org/synapse/issues/4515), [\#4523](https://github.com/matrix-org/synapse/issues/4523), [\#4535](https://github.com/matrix-org/synapse/issues/4535))
-- Synapse will now reload TLS certificates from disk upon SIGHUP. ([\#4495](https://github.com/matrix-org/synapse/issues/4495), [\#4524](https://github.com/matrix-org/synapse/issues/4524))
-- The matrixdotorg/synapse Docker images now use Python 3 by default. ([\#4558](https://github.com/matrix-org/synapse/issues/4558))
-
-Bugfixes
---------
-
-- Prevent users with access tokens predating the introduction of device IDs from creating spurious entries in the user_ips table. ([\#4369](https://github.com/matrix-org/synapse/issues/4369))
-- Fix typo in ALL_USER_TYPES definition to ensure type is a tuple ([\#4392](https://github.com/matrix-org/synapse/issues/4392))
-- Fix high CPU usage due to remote devicelist updates ([\#4397](https://github.com/matrix-org/synapse/issues/4397))
-- Fix potential bug where creating or joining a room could fail ([\#4404](https://github.com/matrix-org/synapse/issues/4404))
-- Fix bug when rejecting remote invites ([\#4405](https://github.com/matrix-org/synapse/issues/4405), [\#4527](https://github.com/matrix-org/synapse/issues/4527))
-- Fix incorrect logcontexts after a Deferred was cancelled ([\#4407](https://github.com/matrix-org/synapse/issues/4407))
-- Ensure encrypted room state is persisted across room upgrades. ([\#4411](https://github.com/matrix-org/synapse/issues/4411))
-- Copy over whether a room is a direct message and any associated room tags on room upgrade. ([\#4412](https://github.com/matrix-org/synapse/issues/4412))
-- Fix None guard in calling config.server.is_threepid_reserved ([\#4435](https://github.com/matrix-org/synapse/issues/4435))
-- Don't send IP addresses as SNI ([\#4452](https://github.com/matrix-org/synapse/issues/4452))
-- Fix UnboundLocalError in post_urlencoded_get_json ([\#4460](https://github.com/matrix-org/synapse/issues/4460))
-- Add a timeout to filtered room directory queries. ([\#4461](https://github.com/matrix-org/synapse/issues/4461))
-- Workaround for login error when using both LDAP and internal authentication. ([\#4486](https://github.com/matrix-org/synapse/issues/4486))
-- Fix a bug where setting a relative consent directory path would cause a crash. ([\#4512](https://github.com/matrix-org/synapse/issues/4512))
-
-
-Deprecations and Removals
--------------------------
-
-- Synapse no longer generates self-signed TLS certificates when generating a configuration file. ([\#4509](https://github.com/matrix-org/synapse/issues/4509))
-
-
-Improved Documentation
-----------------------
-
-- Update debian installation instructions ([\#4526](https://github.com/matrix-org/synapse/issues/4526))
-
-
-Internal Changes
-----------------
-
-- Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+. ([\#4306](https://github.com/matrix-org/synapse/issues/4306), [\#4459](https://github.com/matrix-org/synapse/issues/4459), [\#4466](https://github.com/matrix-org/synapse/issues/4466), [\#4471](https://github.com/matrix-org/synapse/issues/4471), [\#4477](https://github.com/matrix-org/synapse/issues/4477), [\#4505](https://github.com/matrix-org/synapse/issues/4505))
-- Update README to use the new virtualenv everywhere ([\#4342](https://github.com/matrix-org/synapse/issues/4342))
-- Add better logging for unexpected errors while sending transactions ([\#4368](https://github.com/matrix-org/synapse/issues/4368))
-- Apply a unique index to the user_ips table, preventing duplicates. ([\#4370](https://github.com/matrix-org/synapse/issues/4370), [\#4432](https://github.com/matrix-org/synapse/issues/4432), [\#4434](https://github.com/matrix-org/synapse/issues/4434))
-- Silence travis-ci build warnings by removing non-functional python3.6 ([\#4377](https://github.com/matrix-org/synapse/issues/4377))
-- Fix a comment in the generated config file ([\#4387](https://github.com/matrix-org/synapse/issues/4387))
-- Add ground work for implementing future federation API versions ([\#4390](https://github.com/matrix-org/synapse/issues/4390))
-- Update dependencies on msgpack and pymacaroons to use the up-to-date packages. ([\#4399](https://github.com/matrix-org/synapse/issues/4399))
-- Tweak codecov settings to make them less loud. ([\#4400](https://github.com/matrix-org/synapse/issues/4400))
-- Implement server support for MSC1794 - Federation v2 Invite API ([\#4402](https://github.com/matrix-org/synapse/issues/4402))
-- debian package: symlink to explicit python version ([\#4433](https://github.com/matrix-org/synapse/issues/4433))
-- Add infrastructure to support different event formats ([\#4437](https://github.com/matrix-org/synapse/issues/4437), [\#4447](https://github.com/matrix-org/synapse/issues/4447), [\#4448](https://github.com/matrix-org/synapse/issues/4448), [\#4470](https://github.com/matrix-org/synapse/issues/4470), [\#4481](https://github.com/matrix-org/synapse/issues/4481), [\#4482](https://github.com/matrix-org/synapse/issues/4482), [\#4493](https://github.com/matrix-org/synapse/issues/4493), [\#4494](https://github.com/matrix-org/synapse/issues/4494), [\#4496](https://github.com/matrix-org/synapse/issues/4496), [\#4510](https://github.com/matrix-org/synapse/issues/4510), [\#4514](https://github.com/matrix-org/synapse/issues/4514))
-- Generate the debian config during build ([\#4444](https://github.com/matrix-org/synapse/issues/4444))
-- Clarify documentation for the `public_baseurl` config param ([\#4458](https://github.com/matrix-org/synapse/issues/4458), [\#4498](https://github.com/matrix-org/synapse/issues/4498))
-- Fix quoting for allowed_local_3pids example config ([\#4476](https://github.com/matrix-org/synapse/issues/4476))
-- Remove deprecated --process-dependency-links option from UPGRADE.rst ([\#4485](https://github.com/matrix-org/synapse/issues/4485))
-- Make it possible to set the log level for tests via an environment variable ([\#4506](https://github.com/matrix-org/synapse/issues/4506))
-- Reduce the log level of linearizer lock acquirement to DEBUG. ([\#4507](https://github.com/matrix-org/synapse/issues/4507))
-- Fix code to comply with linting in PyFlakes 3.7.1. ([\#4519](https://github.com/matrix-org/synapse/issues/4519))
-- Add some debug for membership syncing issues ([\#4538](https://github.com/matrix-org/synapse/issues/4538))
-- Docker: only copy what we need to the build image ([\#4562](https://github.com/matrix-org/synapse/issues/4562))
-
-
 Synapse 0.34.1.1 (2019-01-11)
 =============================
 

CONTRIBUTING.rst

@@ -30,7 +30,7 @@ use github's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Travis CI
+We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Travis CI 
 <https://travis-ci.org/matrix-org/synapse>`_ for continuous integration. All
 pull requests to synapse get automatically tested by Travis and CircleCI.
 If your change breaks the build, this will be shown in GitHub, so please
@@ -74,39 +74,16 @@ entry. These are managed by Towncrier
 To create a changelog entry, make a new file in the ``changelog.d``
 file named in the format of ``PRnumber.type``. The type can be
 one of ``feature``, ``bugfix``, ``removal`` (also used for
-deprecations), or ``misc`` (for internal-only changes).
-
-The content of the file is your changelog entry, which can contain Markdown
-formatting. The entry should end with a full stop ('.') for consistency.
-
-Adding credits to the changelog is encouraged, we value your
-contributions and would like to have you shouted out in the release notes!
+deprecations), or ``misc`` (for internal-only changes). The content of
+the file is your changelog entry, which can contain Markdown
+formatting. Adding credits to the changelog is encouraged, we value
+your contributions and would like to have you shouted out in the
+release notes!
 
 For example, a fix in PR #1234 would have its changelog entry in
 ``changelog.d/1234.bugfix``, and contain content like "The security levels of
 Florbs are now validated when recieved over federation. Contributed by Jane
-Matrix.".
-
-Debian changelog
-----------------
-
-Changes which affect the debian packaging files (in ``debian``) are an
-exception.
-
-In this case, you will need to add an entry to the debian changelog for the
-next release. For this, run the following command::
-
-  dch
-
-This will make up a new version number (if there isn't already an unreleased
-version in flight), and open an editor where you can add a new changelog entry.
-(Our release process will ensure that the version number and maintainer name is
-corrected for the release.)
-
-If your change affects both the debian packaging *and* files outside the debian
-directory, you will need both a regular newsfragment *and* an entry in the
-debian changelog. (Though typically such changes should be submitted as two
-separate pull requests.)
+Matrix".
 
 Attribution
 ~~~~~~~~~~~

INSTALL.md

@@ -1,426 +0,0 @@
-* [Installing Synapse](#installing-synapse)
-  * [Installing from source](#installing-from-source)
-    * [Platform-Specific Instructions](#platform-specific-instructions)
-    * [Troubleshooting Installation](#troubleshooting-installation)
-  * [Prebuilt packages](#prebuilt-packages)
-* [Setting up Synapse](#setting-up-synapse)
-  * [TLS certificates](#tls-certificates)
-  * [Registering a user](#registering-a-user)
-  * [Setting up a TURN server](#setting-up-a-turn-server)
-  * [URL previews](#url-previews)
-
-# Installing Synapse
-
-## Installing from source
-
-(Prebuilt packages are available for some platforms - see [Prebuilt packages](#prebuilt-packages).)
-
-System requirements:
-
-- POSIX-compliant system (tested on Linux & OS X)
-- Python 3.5, 3.6, 3.7, or 2.7
-- At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
-
-Synapse is written in Python but some of the libraries it uses are written in
-C. So before we can install Synapse itself we need a working C compiler and the
-header files for Python C extensions. See [Platform-Specific
-Instructions](#platform-specific-instructions) for information on installing
-these on various platforms.
-
-To install the Synapse homeserver run:
-
-```
-mkdir -p ~/synapse
-virtualenv -p python3 ~/synapse/env
-source ~/synapse/env/bin/activate
-pip install --upgrade pip
-pip install --upgrade setuptools
-pip install matrix-synapse[all]
-```
-
-This will download Synapse from [PyPI](https://pypi.org/project/matrix-synapse)
-and install it, along with the python libraries it uses, into a virtual environment
-under `~/synapse/env`.  Feel free to pick a different directory if you
-prefer.
-
-This Synapse installation can then be later upgraded by using pip again with the
-update flag:
-
-```
-source ~/synapse/env/bin/activate
-pip install -U matrix-synapse[all]
-```
-
-Before you can start Synapse, you will need to generate a configuration
-file. To do this, run (in your virtualenv, as before)::
-
-```
-cd ~/synapse
-python -m synapse.app.homeserver \
-    --server-name my.domain.name \
-    --config-path homeserver.yaml \
-    --generate-config \
-    --report-stats=[yes|no]
-```
-
-... substituting an appropriate value for `--server-name`. The server name
-determines the "domain" part of user-ids for users on your server: these will
-all be of the format `@user:my.domain.name`. It also determines how other
-matrix servers will reach yours for Federation. For a test configuration,
-set this to the hostname of your server. For a more production-ready setup, you
-will probably want to specify your domain (`example.com`) rather than a
-matrix-specific hostname here (in the same way that your email address is
-probably `user@example.com` rather than `user@email.example.com`) - but
-doing so may require more advanced setup. - see [Setting up Federation](README.rst#setting-up-federation). Beware that the server name cannot be changed later.
-
-This command will generate you a config file that you can then customise, but it will
-also generate a set of keys for you. These keys will allow your Home Server to
-identify itself to other Home Servers, so don't lose or delete them. It would be
-wise to back them up somewhere safe. (If, for whatever reason, you do need to
-change your Home Server's keys, you may find that other Home Servers have the
-old key cached. If you update the signing key, you should change the name of the
-key in the `<server name>.signing.key` file (the second word) to something
-different. See the
-[spec](https://matrix.org/docs/spec/server_server/latest.html#retrieving-server-keys)
-for more information on key management.)
-
-You will need to give Synapse a TLS certficate before it will start - see [TLS
-certificates](#tls-certificates).
-
-To actually run your new homeserver, pick a working directory for Synapse to
-run (e.g. `~/synapse`), and::
-
-    cd ~/synapse
-    source env/bin/activate
-    synctl start
-
-### Platform-Specific Instructions
-
-#### Debian/Ubuntu/Raspbian
-
-Installing prerequisites on Ubuntu or Debian:
-
-```
-sudo apt-get install build-essential python3-dev libffi-dev \
-                     python-pip python-setuptools sqlite3 \
-                     libssl-dev python-virtualenv libjpeg-dev libxslt1-dev
-```
-
-#### ArchLinux
-
-Installing prerequisites on ArchLinux:
-
-```
-sudo pacman -S base-devel python python-pip \
-               python-setuptools python-virtualenv sqlite3
-```
-
-#### CentOS/Fedora
-
-Installing prerequisites on CentOS 7 or Fedora 25:
-
-```
-sudo yum install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
-                 lcms2-devel libwebp-devel tcl-devel tk-devel redhat-rpm-config \
-                 python-virtualenv libffi-devel openssl-devel
-sudo yum groupinstall "Development Tools"
-```
-
-#### Mac OS X
-
-Installing prerequisites on Mac OS X:
-
-```
-xcode-select --install
-sudo easy_install pip
-sudo pip install virtualenv
-brew install pkg-config libffi
-```
-
-#### OpenSUSE
-
-Installing prerequisites on openSUSE:
-
-```
-sudo zypper in -t pattern devel_basis
-sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \
-               python-devel libffi-devel libopenssl-devel libjpeg62-devel
-```
-
-#### OpenBSD
-
-Installing prerequisites on OpenBSD:
-
-```
-doas pkg_add python libffi py-pip py-setuptools sqlite3 py-virtualenv \
-              libxslt jpeg
-```
-
-There is currently no port for OpenBSD. Additionally, OpenBSD's security
-settings require a slightly more difficult installation process.
-
-XXX: I suspect this is out of date.
-
-1. Create a new directory in `/usr/local` called `_synapse`. Also, create a
-   new user called `_synapse` and set that directory as the new user's home.
-   This is required because, by default, OpenBSD only allows binaries which need
-   write and execute permissions on the same memory space to be run from
-   `/usr/local`.
-2. `su` to the new `_synapse` user and change to their home directory.
-3. Create a new virtualenv: `virtualenv -p python2.7 ~/.synapse`
-4. Source the virtualenv configuration located at
-   `/usr/local/_synapse/.synapse/bin/activate`. This is done in `ksh` by
-   using the `.` command, rather than `bash`'s `source`.
-5. Optionally, use `pip` to install `lxml`, which Synapse needs to parse
-   webpages for their titles.
-6. Use `pip` to install this repository: `pip install matrix-synapse`
-7. Optionally, change `_synapse`'s shell to `/bin/false` to reduce the
-   chance of a compromised Synapse server being used to take over your box.
-
-After this, you may proceed with the rest of the install directions.
-
-#### Windows
-
-If you wish to run or develop Synapse on Windows, the Windows Subsystem For
-Linux provides a Linux environment on Windows 10 which is capable of using the
-Debian, Fedora, or source installation methods. More information about WSL can
-be found at https://docs.microsoft.com/en-us/windows/wsl/install-win10 for
-Windows 10 and https://docs.microsoft.com/en-us/windows/wsl/install-on-server
-for Windows Server.
-
-### Troubleshooting Installation
-
-XXX a bunch of this is no longer relevant.
-
-Synapse requires pip 8 or later, so if your OS provides too old a version you
-may need to manually upgrade it::
-
-    sudo pip install --upgrade pip
-
-Installing may fail with `Could not find any downloads that satisfy the requirement pymacaroons-pynacl (from matrix-synapse==0.12.0)`.
-You can fix this by manually upgrading pip and virtualenv::
-
-    sudo pip install --upgrade virtualenv
-
-You can next rerun `virtualenv -p python3 synapse` to update the virtual env.
-
-Installing may fail during installing virtualenv with `InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.`
-You can fix this  by manually installing ndg-httpsclient::
-
-    pip install --upgrade ndg-httpsclient
-
-Installing may fail with `mock requires setuptools>=17.1. Aborting installation`.
-You can fix this by upgrading setuptools::
-
-    pip install --upgrade setuptools
-
-If pip crashes mid-installation for reason (e.g. lost terminal), pip may
-refuse to run until you remove the temporary installation directory it
-created. To reset the installation::
-
-    rm -rf /tmp/pip_install_matrix
-
-pip seems to leak *lots* of memory during installation.  For instance, a Linux
-host with 512MB of RAM may run out of memory whilst installing Twisted.  If this
-happens, you will have to individually install the dependencies which are
-failing, e.g.::
-
-    pip install twisted
-
-## Prebuilt packages
-
-As an alternative to installing from source, prebuilt packages are available
-for a number of platforms.
-
-### Docker images and Ansible playbooks
-
-There is an offical synapse image available at
-https://hub.docker.com/r/matrixdotorg/synapse which can be used with
-the docker-compose file available at [contrib/docker](contrib/docker). Further information on
-this including configuration options is available in the README on
-hub.docker.com.
-
-Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
-Dockerfile to automate a synapse server in a single Docker image, at
-https://hub.docker.com/r/avhost/docker-matrix/tags/
-
-Slavi Pantaleev has created an Ansible playbook,
-which installs the offical Docker image of Matrix Synapse
-along with many other Matrix-related services (Postgres database, riot-web, coturn, mxisd, SSL support, etc.).
-For more details, see
-https://github.com/spantaleev/matrix-docker-ansible-deploy
-
-
-### Debian/Ubuntu
-
-#### Matrix.org packages
-
-Matrix.org provides Debian/Ubuntu packages of the latest stable version of
-Synapse via https://matrix.org/packages/debian/. To use them:
-
-```
-sudo apt install -y lsb-release curl apt-transport-https
-echo "deb https://matrix.org/packages/debian `lsb_release -cs` main" |
-    sudo tee /etc/apt/sources.list.d/matrix-org.list
-curl "https://matrix.org/packages/debian/repo-key.asc" |
-    sudo apt-key add -
-sudo apt update
-sudo apt install matrix-synapse-py3
-```
-
-#### Downstream Debian/Ubuntu packages
-
-For `buster` and `sid`, Synapse is available in the Debian repositories and
-it should be possible to install it with simply:
-
-```
-    sudo apt install matrix-synapse
-```
-
-There is also a version of `matrix-synapse` in `stretch-backports`. Please see
-the [Debian documentation on
-backports](https://backports.debian.org/Instructions/) for information on how
-to use them.
-
-We do not recommend using the packages in downstream Ubuntu at this time, as
-they are old and suffer from known security vulnerabilities.
-
-### Fedora
-
-Synapse is in the Fedora repositories as `matrix-synapse`:
-
-```
-sudo dnf install matrix-synapse
-```
-
-Oleg Girko provides Fedora RPMs at
-https://obs.infoserver.lv/project/monitor/matrix-synapse
-
-### OpenSUSE
-
-Synapse is in the OpenSUSE repositories as `matrix-synapse`:
-
-```
-sudo zypper install matrix-synapse
-```
-
-### SUSE Linux Enterprise Server
-
-Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 repository at
-https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/
-
-### ArchLinux
-
-The quickest way to get up and running with ArchLinux is probably with the community package
-https://www.archlinux.org/packages/community/any/matrix-synapse/, which should pull in most of
-the necessary dependencies.
-
-pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):
-
-```
-sudo pip install --upgrade pip
-```
-
-If you encounter an error with lib bcrypt causing an Wrong ELF Class:
-ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly
-compile it under the right architecture. (This should not be needed if
-installing under virtualenv):
-
-```
-sudo pip uninstall py-bcrypt
-sudo pip install py-bcrypt
-```
-
-### FreeBSD
-
-Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from:
-
- - Ports: `cd /usr/ports/net-im/py-matrix-synapse && make install clean`
- - Packages: `pkg install py27-matrix-synapse`
-
-
-### NixOS
-
-Robin Lambertz has packaged Synapse for NixOS at:
-https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-synapse.nix
-
-# Setting up Synapse
-
-Once you have installed synapse as above, you will need to configure it.
-
-## TLS certificates
-
-The default configuration exposes a single HTTP port: http://localhost:8008. It
-is suitable for local testing, but for any practical use, you will either need
-to enable a reverse proxy, or configure Synapse to expose an HTTPS port.
-
-For information on using a reverse proxy, see
-[docs/reverse_proxy.rst](docs/reverse_proxy.rst).
-
-To configure Synapse to expose an HTTPS port, you will need to edit
-`homeserver.yaml`, as follows:
-
-* First, under the `listeners` section, uncomment the configuration for the
-  TLS-enabled listener. (Remove the hash sign (`#`) at the start of
-  each line). The relevant lines are like this:
-
-  ```
-    - port: 8448
-      type: http
-      tls: true
-      resources:
-        - names: [client, federation]
-  ```
-* You will also need to uncomment the `tls_certificate_path` and
-  `tls_private_key_path` lines under the `TLS` section. You can either
-  point these settings at an existing certificate and key, or you can
-  enable Synapse's built-in ACME (Let's Encrypt) support.  Instructions
-  for having Synapse automatically provision and renew federation 
-  certificates through ACME can be found at [ACME.md](docs/ACME.md).
-
-## Registering a user
-
-You will need at least one user on your server in order to use a Matrix
-client. Users can be registered either via a Matrix client, or via a
-commandline script.
-
-To get started, it is easiest to use the command line to register new
-users. This can be done as follows:
-
-```
-$ source ~/synapse/env/bin/activate
-$ synctl start # if not already running
-$ register_new_matrix_user -c homeserver.yaml http://localhost:8008
-New user localpart: erikj
-Password:
-Confirm password:
-Make admin [no]:
-Success!
-```
-
-This process uses a setting `registration_shared_secret` in
-`homeserver.yaml`, which is shared between Synapse itself and the
-`register_new_matrix_user` script. It doesn't matter what it is (a random
-value is generated by `--generate-config`), but it should be kept secret, as
-anyone with knowledge of it can register users on your server even if
-`enable_registration` is `false`.
-
-## Setting up a TURN server
-
-For reliable VoIP calls to be routed via this homeserver, you MUST configure
-a TURN server.  See [docs/turn-howto.rst](docs/turn-howto.rst) for details.
-
-## URL previews
-
-Synapse includes support for previewing URLs, which is disabled by default.  To
-turn it on you must enable the `url_preview_enabled: True` config parameter
-and explicitly specify the IP ranges that Synapse is not allowed to spider for
-previewing in the `url_preview_ip_range_blacklist` configuration parameter.
-This is critical from a security perspective to stop arbitrary Matrix users
-spidering 'internal' URLs on your network.  At the very least we recommend that
-your loopback and RFC1918 IP addresses are blacklisted.
-
-This also requires the optional lxml and netaddr python dependencies to be
-installed.  This in turn requires the libxml2 library to be available - on
-Debian/Ubuntu this means `apt-get install libxml2-dev`, or equivalent for
-your OS.

MANIFEST.in

@@ -15,7 +15,6 @@ recursive-include docs *
 recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
-recursive-include tests *.pem
 recursive-include tests *.py
 
 recursive-include synapse/res *
@@ -38,7 +37,6 @@ prune docker
 prune .circleci
 prune .coveragerc
 prune debian
-prune .codecov.yml
 
 exclude jenkins*
 recursive-exclude jenkins *.sh

README.rst

@@ -26,6 +26,7 @@ via IRC bridge at irc://irc.freenode.net/matrix.
 Synapse is currently in rapid development, but as of version 0.5 we believe it
 is sufficiently stable to be run as an internet-facing service for real usage!
 
+
 About Matrix
 ============
 
@@ -80,27 +81,210 @@ Thanks for using Matrix!
 Synapse Installation
 ====================
 
-For details on how to install synapse, see `<INSTALL.md>`_.
+Synapse is the reference Python/Twisted Matrix homeserver implementation.
 
+System requirements:
+
+- POSIX-compliant system (tested on Linux & OS X)
+- Python 3.5, 3.6, 3.7, or 2.7
+- At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
+
+Installing from source
+----------------------
+
+(Prebuilt packages are available for some platforms - see `Platform-Specific
+Instructions`_.)
+
+Synapse is written in Python but some of the libraries it uses are written in
+C. So before we can install Synapse itself we need a working C compiler and the
+header files for Python C extensions.
+
+Installing prerequisites on Ubuntu or Debian::
+
+    sudo apt-get install build-essential python3-dev libffi-dev \
+                         python-pip python-setuptools sqlite3 \
+                         libssl-dev python-virtualenv libjpeg-dev libxslt1-dev
+
+Installing prerequisites on ArchLinux::
+
+    sudo pacman -S base-devel python python-pip \
+                   python-setuptools python-virtualenv sqlite3
+
+Installing prerequisites on CentOS 7 or Fedora 25::
+
+    sudo yum install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
+                     lcms2-devel libwebp-devel tcl-devel tk-devel redhat-rpm-config \
+                     python-virtualenv libffi-devel openssl-devel
+    sudo yum groupinstall "Development Tools"
+
+Installing prerequisites on Mac OS X::
+
+    xcode-select --install
+    sudo easy_install pip
+    sudo pip install virtualenv
+    brew install pkg-config libffi
+
+Installing prerequisites on Raspbian::
+
+    sudo apt-get install build-essential python3-dev libffi-dev \
+                         python-pip python-setuptools sqlite3 \
+                         libssl-dev python-virtualenv libjpeg-dev
+
+Installing prerequisites on openSUSE::
+
+    sudo zypper in -t pattern devel_basis
+    sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \
+                   python-devel libffi-devel libopenssl-devel libjpeg62-devel
+
+Installing prerequisites on OpenBSD::
+
+    doas pkg_add python libffi py-pip py-setuptools sqlite3 py-virtualenv \
+                 libxslt jpeg
+
+To install the Synapse homeserver run::
+
+    mkdir -p ~/synapse
+    virtualenv -p python3 ~/synapse/env
+    source ~/synapse/env/bin/activate
+    pip install --upgrade pip
+    pip install --upgrade setuptools
+    pip install matrix-synapse[all]
+
+This installs Synapse, along with the libraries it uses, into a virtual
+environment under ``~/synapse/env``.  Feel free to pick a different directory
+if you prefer.
+
+This Synapse installation can then be later upgraded by using pip again with the
+update flag::
+
+    source ~/synapse/env/bin/activate
+    pip install -U matrix-synapse[all]
+
+In case of problems, please see the _`Troubleshooting` section below.
+
+There is an offical synapse image available at
+https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with
+the docker-compose file available at `contrib/docker <contrib/docker>`_. Further information on
+this including configuration options is available in the README on
+hub.docker.com.
+
+Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
+Dockerfile to automate a synapse server in a single Docker image, at
+https://hub.docker.com/r/avhost/docker-matrix/tags/
+
+Slavi Pantaleev has created an Ansible playbook,
+which installs the offical Docker image of Matrix Synapse
+along with many other Matrix-related services (Postgres database, riot-web, coturn, mxisd, SSL support, etc.).
+For more details, see
+https://github.com/spantaleev/matrix-docker-ansible-deploy
+
+Configuring Synapse
+-------------------
+
+Before you can start Synapse, you will need to generate a configuration
+file. To do this, run (in your virtualenv, as before)::
+
+    cd ~/.synapse
+    python -m synapse.app.homeserver \
+        --server-name my.domain.name \
+        --config-path homeserver.yaml \
+        --generate-config \
+        --report-stats=[yes|no]
+
+... substituting an appropriate value for ``--server-name``. The server name
+determines the "domain" part of user-ids for users on your server: these will
+all be of the format ``@user:my.domain.name``. It also determines how other
+matrix servers will reach yours for `Federation`_. For a test configuration,
+set this to the hostname of your server. For a more production-ready setup, you
+will probably want to specify your domain (``example.com``) rather than a
+matrix-specific hostname here (in the same way that your email address is
+probably ``user@example.com`` rather than ``user@email.example.com``) - but
+doing so may require more advanced setup - see `Setting up
+Federation`_. Beware that the server name cannot be changed later.
+
+This command will generate you a config file that you can then customise, but it will
+also generate a set of keys for you. These keys will allow your Home Server to
+identify itself to other Home Servers, so don't lose or delete them. It would be
+wise to back them up somewhere safe. (If, for whatever reason, you do need to
+change your Home Server's keys, you may find that other Home Servers have the
+old key cached. If you update the signing key, you should change the name of the
+key in the ``<server name>.signing.key`` file (the second word) to something
+different. See `the spec`__ for more information on key management.)
+
+.. __: `key_management`_
+
+The default configuration exposes two HTTP ports: 8008 and 8448. Port 8008 is
+configured without TLS; it should be behind a reverse proxy for TLS/SSL
+termination on port 443 which in turn should be used for clients. Port 8448
+is configured to use TLS with a self-signed certificate. If you would like
+to do initial test with a client without having to setup a reverse proxy,
+you can temporarly use another certificate. (Note that a self-signed
+certificate is fine for `Federation`_). You can do so by changing
+``tls_certificate_path``, ``tls_private_key_path`` and ``tls_dh_params_path``
+in ``homeserver.yaml``; alternatively, you can use a reverse-proxy, but be sure
+to read `Using a reverse proxy with Synapse`_ when doing so.
+
+Apart from port 8448 using TLS, both ports are the same in the default
+configuration.
+
+Registering a user
+------------------
+
+You will need at least one user on your server in order to use a Matrix
+client. Users can be registered either `via a Matrix client`__, or via a
+commandline script.
+
+.. __: `client-user-reg`_
+
+To get started, it is easiest to use the command line to register new users::
+
+    $ source ~/synapse/env/bin/activate
+    $ synctl start # if not already running
+    $ register_new_matrix_user -c homeserver.yaml https://localhost:8448
+    New user localpart: erikj
+    Password:
+    Confirm password:
+    Make admin [no]:
+    Success!
+
+This process uses a setting ``registration_shared_secret`` in
+``homeserver.yaml``, which is shared between Synapse itself and the
+``register_new_matrix_user`` script. It doesn't matter what it is (a random
+value is generated by ``--generate-config``), but it should be kept secret, as
+anyone with knowledge of it can register users on your server even if
+``enable_registration`` is ``false``.
+
+Setting up a TURN server
+------------------------
+
+For reliable VoIP calls to be routed via this homeserver, you MUST configure
+a TURN server.  See `<docs/turn-howto.rst>`_ for details.
+
+Running Synapse
+===============
+
+To actually run your new homeserver, pick a working directory for Synapse to
+run (e.g. ``~/synapse``), and::
+
+    cd ~/synapse
+    source env/bin/activate
+    synctl start
 
 Connecting to Synapse from a client
 ===================================
 
 The easiest way to try out your new Synapse installation is by connecting to it
-from a web client.
+from a web client. The easiest option is probably the one at
+https://riot.im/app. You will need to specify a "Custom server" when you log on
+or register: set this to ``https://domain.tld`` if you setup a reverse proxy
+following the recommended setup, or ``https://localhost:8448`` - remember to specify the
+port (``:8448``) if not ``:443`` unless you changed the configuration. (Leave the identity
+server as the default - see `Identity servers`_.)
 
-Unless you are running a test instance of Synapse on your local machine, in
-general, you will need to enable TLS support before you can successfully
-connect from a client: see `<INSTALL.md#tls-certificates>`_.
-
-An easy way to get started is to login or register via Riot at 
-https://riot.im/app/#/login or https://riot.im/app/#/register respectively. 
-You will need to change the server you are logging into from ``matrix.org``
-and instead specify a Homeserver URL of ``https://<server_name>:8448`` 
-(or just ``https://<server_name>`` if you are using a reverse proxy). 
-(Leave the identity server as the default - see `Identity servers`_.) 
-If you prefer to use another client, refer to our 
-`client breakdown <https://matrix.org/docs/projects/clients-matrix>`_.
+If using port 8448 you will run into errors until you accept the self-signed
+certificate. You can easily do this by going to ``https://localhost:8448``
+directly with your browser and accept the presented certificate. You can then
+go back in your web client and proceed further.
 
 If all goes well you should at least be able to log in, create a room, and
 start sending messages.
@@ -128,12 +312,6 @@ create the account. Your name will take the form of::
 As when logging in, you will need to specify a "Custom server".  Specify your
 desired ``localpart`` in the 'User name' box.
 
-ACME setup
-==========
-
-For details on having Synapse manage your federation TLS certificates
-automatically, please see `<docs/ACME.md>`_.
-
 
 Security Note
 =============
@@ -151,11 +329,151 @@ server on the same domain.
 See https://github.com/vector-im/riot-web/issues/1977 and
 https://developer.github.com/changes/2014-04-25-user-content-security for more details.
 
+
+Platform-Specific Instructions
+==============================
+
+Debian
+------
+
+Matrix provides official Debian packages via apt from https://matrix.org/packages/debian/.
+Note that these packages do not include a client - choose one from
+https://matrix.org/docs/projects/try-matrix-now.html (or build your own with one of our SDKs :)
+
+Fedora
+------
+
+Synapse is in the Fedora repositories as ``matrix-synapse``::
+
+    sudo dnf install matrix-synapse
+
+Oleg Girko provides Fedora RPMs at
+https://obs.infoserver.lv/project/monitor/matrix-synapse
+
+OpenSUSE
+--------
+
+Synapse is in the OpenSUSE repositories as ``matrix-synapse``::
+
+    sudo zypper install matrix-synapse
+
+SUSE Linux Enterprise Server
+----------------------------
+
+Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 repository at
+https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/
+
+ArchLinux
+---------
+
+The quickest way to get up and running with ArchLinux is probably with the community package
+https://www.archlinux.org/packages/community/any/matrix-synapse/, which should pull in most of
+the necessary dependencies.
+
+pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 )::
+
+    sudo pip install --upgrade pip
+
+If you encounter an error with lib bcrypt causing an Wrong ELF Class:
+ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly
+compile it under the right architecture. (This should not be needed if
+installing under virtualenv)::
+
+    sudo pip uninstall py-bcrypt
+    sudo pip install py-bcrypt
+
+FreeBSD
+-------
+
+Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from:
+
+ - Ports: ``cd /usr/ports/net-im/py-matrix-synapse && make install clean``
+ - Packages: ``pkg install py27-matrix-synapse``
+
+
+OpenBSD
+-------
+
+There is currently no port for OpenBSD. Additionally, OpenBSD's security
+settings require a slightly more difficult installation process.
+
+1) Create a new directory in ``/usr/local`` called ``_synapse``. Also, create a
+   new user called ``_synapse`` and set that directory as the new user's home.
+   This is required because, by default, OpenBSD only allows binaries which need
+   write and execute permissions on the same memory space to be run from
+   ``/usr/local``.
+2) ``su`` to the new ``_synapse`` user and change to their home directory.
+3) Create a new virtualenv: ``virtualenv -p python2.7 ~/.synapse``
+4) Source the virtualenv configuration located at
+   ``/usr/local/_synapse/.synapse/bin/activate``. This is done in ``ksh`` by
+   using the ``.`` command, rather than ``bash``'s ``source``.
+5) Optionally, use ``pip`` to install ``lxml``, which Synapse needs to parse
+   webpages for their titles.
+6) Use ``pip`` to install this repository: ``pip install matrix-synapse``
+7) Optionally, change ``_synapse``'s shell to ``/bin/false`` to reduce the
+   chance of a compromised Synapse server being used to take over your box.
+
+After this, you may proceed with the rest of the install directions.
+
+NixOS
+-----
+
+Robin Lambertz has packaged Synapse for NixOS at:
+https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-synapse.nix
+
+Windows Install
+---------------
+
+If you wish to run or develop Synapse on Windows, the Windows Subsystem For
+Linux provides a Linux environment on Windows 10 which is capable of using the
+Debian, Fedora, or source installation methods. More information about WSL can
+be found at https://docs.microsoft.com/en-us/windows/wsl/install-win10 for
+Windows 10 and https://docs.microsoft.com/en-us/windows/wsl/install-on-server
+for Windows Server.
+
 Troubleshooting
 ===============
 
+Troubleshooting Installation
+----------------------------
+
+Synapse requires pip 8 or later, so if your OS provides too old a version you
+may need to manually upgrade it::
+
+    sudo pip install --upgrade pip
+
+Installing may fail with ``Could not find any downloads that satisfy the requirement pymacaroons-pynacl (from matrix-synapse==0.12.0)``.
+You can fix this by manually upgrading pip and virtualenv::
+
+    sudo pip install --upgrade virtualenv
+
+You can next rerun ``virtualenv -p python3 synapse`` to update the virtual env.
+
+Installing may fail during installing virtualenv with ``InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.``
+You can fix this  by manually installing ndg-httpsclient::
+
+    pip install --upgrade ndg-httpsclient
+
+Installing may fail with ``mock requires setuptools>=17.1. Aborting installation``.
+You can fix this by upgrading setuptools::
+
+    pip install --upgrade setuptools
+
+If pip crashes mid-installation for reason (e.g. lost terminal), pip may
+refuse to run until you remove the temporary installation directory it
+created. To reset the installation::
+
+    rm -rf /tmp/pip_install_matrix
+
+pip seems to leak *lots* of memory during installation.  For instance, a Linux
+host with 512MB of RAM may run out of memory whilst installing Twisted.  If this
+happens, you will have to individually install the dependencies which are
+failing, e.g.::
+
+    pip install twisted
+
 Running out of File Handles
----------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If synapse runs out of filehandles, it typically fails badly - live-locking
 at 100% CPU, and/or failing to accept new TCP connections (blocking the
@@ -175,32 +493,9 @@ Separately, Synapse may leak file handles if inbound HTTP requests get stuck
 during processing - e.g. blocked behind a lock or talking to a remote server etc.
 This is best diagnosed by matching up the 'Received request' and 'Processed request'
 log lines and looking for any 'Processed request' lines which take more than
-a few seconds to execute.  Please let us know at #synapse:matrix.org if
+a few seconds to execute.  Please let us know at #matrix-dev:matrix.org if
 you see this failure mode so we can help debug it, however.
 
-Help!! Synapse eats all my RAM!
--------------------------------
-
-Synapse's architecture is quite RAM hungry currently - we deliberately
-cache a lot of recent room data and metadata in RAM in order to speed up
-common requests.  We'll improve this in future, but for now the easiest
-way to either reduce the RAM usage (at the risk of slowing things down)
-is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment
-variable.  The default is 0.5, which can be decreased to reduce RAM usage
-in memory constrained enviroments, or increased if performance starts to
-degrade.
-
-Using `libjemalloc <http://jemalloc.net/>`_ can also yield a significant
-improvement in overall amount, and especially in terms of giving back RAM
-to the OS. To use it, the library must simply be put in the LD_PRELOAD
-environment variable when launching Synapse. On Debian, this can be done
-by installing the ``libjemalloc1`` package and adding this line to
-``/etc/default/matrix-synapse``::
-
-    LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
-
-This can make a significant difference on Python 2.7 - it's unclear how
-much of an improvement it provides on Python 3.x.
 
 Upgrading an existing Synapse
 =============================
@@ -220,19 +515,21 @@ Federation is the process by which users on different servers can participate
 in the same room. For this to work, those other servers must be able to contact
 yours to send messages.
 
-The ``server_name`` in your ``homeserver.yaml`` file determines the way that
-other servers will reach yours. By default, they will treat it as a hostname
-and try to connect to port 8448. This is easy to set up and will work with the
-default configuration, provided you set the ``server_name`` to match your
-machine's public DNS hostname, and give Synapse a TLS certificate which is
-valid for your ``server_name``.
+As explained in `Configuring synapse`_, the ``server_name`` in your
+``homeserver.yaml`` file determines the way that other servers will reach
+yours. By default, they will treat it as a hostname and try to connect to
+port 8448. This is easy to set up and will work with the default configuration,
+provided you set the ``server_name`` to match your machine's public DNS
+hostname.
 
 For a more flexible configuration, you can set up a DNS SRV record. This allows
 you to run your server on a machine that might not have the same name as your
 domain name. For example, you might want to run your server at
 ``synapse.example.com``, but have your Matrix user-ids look like
 ``@user:example.com``. (A SRV record also allows you to change the port from
-the default 8448).
+the default 8448. However, if you are thinking of using a reverse-proxy on the
+federation port, which is not recommended, be sure to read
+`Reverse-proxying the federation port`_ first.)
 
 To use a SRV record, first create your SRV record and publish it in DNS. This
 should have the format ``_matrix._tcp.<yourdomain.com> <ttl> IN SRV 10 0 <port>
@@ -267,8 +564,9 @@ largest boxes pause for thought.)
 Troubleshooting
 ---------------
 
-You can use the `federation tester <https://matrix.org/federationtester>`_ to
-check if your homeserver is all set.
+You can use the federation tester to check if your homeserver is all set:
+``https://matrix.org/federationtester/api/report?server_name=<your_server_name>``
+If any of the attributes under "checks" is false, federation won't work.
 
 The typical failure mode with federation is that when you try to join a room,
 it is rejected with "401: Unauthorized". Generally this means that other
@@ -277,6 +575,8 @@ complicated dance which requires connections in both directions).
 
 So, things to check are:
 
+* If you are trying to use a reverse-proxy, read `Reverse-proxying the
+  federation port`_.
 * If you are not using a SRV record, check that your ``server_name`` (the part
   of your user-id after the ``:``) matches your hostname, and that port 8448 on
   that hostname is reachable from outside your network.
@@ -284,11 +584,6 @@ So, things to check are:
   (it should be ``_matrix._tcp.<server_name>``), and that the port and hostname
   it specifies are reachable from outside your network.
 
-Another common problem is that people on other servers can't join rooms that
-you invite them to. This can be caused by an incorrectly-configured reverse
-proxy: see `<docs/reverse_proxy.rst>`_ for instructions on how to correctly
-configure a reverse proxy.
-
 Running a Demo Federation of Synapses
 -------------------------------------
 
@@ -316,6 +611,7 @@ The advantages of Postgres include:
 For information on how to install and use PostgreSQL, please see
 `docs/postgres.rst <docs/postgres.rst>`_.
 
+
 .. _reverse-proxy:
 
 Using a reverse proxy with Synapse
@@ -329,7 +625,118 @@ It is recommended to put a reverse proxy such as
 doing so is that it means that you can expose the default https port (443) to
 Matrix clients without needing to run Synapse with root privileges.
 
-For information on configuring one, see `<docs/reverse_proxy.rst>`_.
+The most important thing to know here is that Matrix clients and other Matrix
+servers do not necessarily need to connect to your server via the same
+port. Indeed, clients will use port 443 by default, whereas servers default to
+port 8448. Where these are different, we refer to the 'client port' and the
+'federation port'.
+
+The next most important thing to know is that using a reverse-proxy on the
+federation port has a number of pitfalls. It is possible, but be sure to read
+`Reverse-proxying the federation port`_.
+
+The recommended setup is therefore to configure your reverse-proxy on port 443
+to port 8008 of synapse for client connections, but to also directly expose port
+8448 for server-server connections. All the Matrix endpoints begin ``/_matrix``,
+so an example nginx configuration might look like::
+
+  server {
+      listen 443 ssl;
+      listen [::]:443 ssl;
+      server_name matrix.example.com;
+
+      location /_matrix {
+          proxy_pass http://localhost:8008;
+          proxy_set_header X-Forwarded-For $remote_addr;
+      }
+  }
+
+an example Caddy configuration might look like::
+
+    matrix.example.com {
+      proxy /_matrix http://localhost:8008 {
+        transparent
+      }
+    }
+
+and an example Apache configuration might look like::
+
+    <VirtualHost *:443>
+        SSLEngine on
+        ServerName matrix.example.com;
+
+        <Location /_matrix>
+            ProxyPass http://127.0.0.1:8008/_matrix nocanon
+            ProxyPassReverse http://127.0.0.1:8008/_matrix
+        </Location>
+    </VirtualHost>
+
+You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true``
+for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are
+recorded correctly.
+
+Having done so, you can then use ``https://matrix.example.com`` (instead of
+``https://matrix.example.com:8448``) as the "Custom server" when `Connecting to
+Synapse from a client`_.
+
+Reverse-proxying the federation port
+------------------------------------
+
+There are two issues to consider before using a reverse-proxy on the federation
+port:
+
+* Due to the way SSL certificates are managed in the Matrix federation protocol
+  (see `spec`__), Synapse needs to be configured with the path to the SSL
+  certificate, *even if you do not terminate SSL at Synapse*.
+
+  .. __: `key_management`_
+
+* Until v0.33.3, Synapse did not support SNI on the federation port
+  (`bug #1491 <https://github.com/matrix-org/synapse/issues/1491>`_). This bug
+  is now fixed, but means that federating with older servers can be unreliable
+  when using name-based virtual hosting.
+
+Furthermore, a number of the normal reasons for using a reverse-proxy do not
+apply:
+
+* Other servers will connect on port 8448 by default, so there is no need to
+  listen on port 443 (for federation, at least), which avoids the need for root
+  privileges and virtual hosting.
+
+* A self-signed SSL certificate is fine for federation, so there is no need to
+  automate renewals. (The certificate generated by ``--generate-config`` is
+  valid for 10 years.)
+
+If you want to set up a reverse-proxy on the federation port despite these
+caveats, you will need to do the following:
+
+* In ``homeserver.yaml``, set ``tls_certificate_path`` to the path to the SSL
+  certificate file used by your reverse-proxy, and set ``no_tls`` to ``True``.
+  (``tls_private_key_path`` will be ignored if ``no_tls`` is ``True``.)
+
+* In your reverse-proxy configuration:
+
+  * If there are other virtual hosts on the same port, make sure that the
+    *default* one uses the certificate configured above.
+
+  * Forward ``/_matrix`` to Synapse.
+
+* If your reverse-proxy is not listening on port 8448, publish a SRV record to
+  tell other servers how to find you. See `Setting up Federation`_.
+
+When updating the SSL certificate, just update the file pointed to by
+``tls_certificate_path`` and then restart Synapse. (You may like to use a symbolic link
+to help make this process atomic.)
+
+The most common mistake when setting up federation is not to tell Synapse about
+your SSL certificate. To check it, you can visit
+``https://matrix.org/federationtester/api/report?server_name=<your_server_name>``.
+Unfortunately, there is no UI for this yet, but, you should see
+``"MatchingTLSFingerprint": true``. If not, check that
+``Certificates[0].SHA256Fingerprint`` (the fingerprint of the certificate
+presented by your reverse-proxy) matches ``Keys.tls_fingerprints[0].sha256``
+(the fingerprint of the certificate Synapse is using).
+
 
 Identity Servers
 ================
@@ -361,6 +768,24 @@ an email address with your account, or send an invite to another user via their
 email address.
 
 
+URL Previews
+============
+
+Synapse 0.15.0 introduces a new API for previewing URLs at
+``/_matrix/media/r0/preview_url``.  This is disabled by default.  To turn it on
+you must enable the ``url_preview_enabled: True`` config parameter and
+explicitly specify the IP ranges that Synapse is not allowed to spider for
+previewing in the ``url_preview_ip_range_blacklist`` configuration parameter.
+This is critical from a security perspective to stop arbitrary Matrix users
+spidering 'internal' URLs on your network.  At the very least we recommend that
+your loopback and RFC1918 IP addresses are blacklisted.
+
+This also requires the optional lxml and netaddr python dependencies to be
+installed.  This in turn requires the libxml2 library to be available - on
+Debian/Ubuntu this means ``apt-get install libxml2-dev``, or equivalent for
+your OS.
+
+
 Password reset
 ==============
 
@@ -371,7 +796,8 @@ A manual password reset can be done via direct database access as follows.
 
 First calculate the hash of the new password::
 
-    $ ~/synapse/env/bin/hash_password
+    $ source ~/.synapse/bin/activate
+    $ ./scripts/hash_password
     Password:
     Confirm password:
     $2a$12$xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
@@ -387,7 +813,7 @@ Synapse Development
 
 Before setting up a development environment for synapse, make sure you have the
 system dependencies (such as the python header files) installed - see
-`Installing from source <INSTALL.md#installing-from-source>`_.
+`Installing from source`_.
 
 To check out a synapse for development, clone the git repo into a working
 directory of your choice::
@@ -398,7 +824,7 @@ directory of your choice::
 Synapse has a number of external dependencies, that are easiest
 to install using pip and a virtualenv::
 
-    virtualenv -p python3 env
+    virtualenv -p python2.7 env
     source env/bin/activate
     python -m pip install -e .[all]
 
@@ -440,3 +866,27 @@ sphinxcontrib-napoleon::
 Building internal API documentation::
 
     python setup.py build_sphinx
+
+
+Help!! Synapse eats all my RAM!
+===============================
+
+Synapse's architecture is quite RAM hungry currently - we deliberately
+cache a lot of recent room data and metadata in RAM in order to speed up
+common requests.  We'll improve this in future, but for now the easiest
+way to either reduce the RAM usage (at the risk of slowing things down)
+is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment
+variable.  The default is 0.5, which can be decreased to reduce RAM usage
+in memory constrained enviroments, or increased if performance starts to
+degrade.
+
+Using `libjemalloc <http://jemalloc.net/>`_ can also yield a significant
+improvement in overall amount, and especially in terms of giving back RAM
+to the OS. To use it, the library must simply be put in the LD_PRELOAD
+environment variable when launching Synapse. On Debian, this can be done
+by installing the ``libjemalloc1`` package and adding this line to
+``/etc/default/matrix-synapse``::
+
+    LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
+
+.. _`key_management`: https://matrix.org/docs/spec/server_server/unstable.html#retrieving-server-keys

UPGRADE.rst

@@ -5,20 +5,20 @@ Before upgrading check if any special steps are required to upgrade from the
 what you currently have installed to current version of synapse. The extra
 instructions that may be required are listed later in this document.
 
-1. If synapse was installed in a virtualenv then activate that virtualenv before
-   upgrading. If synapse is installed in a virtualenv in ``~/synapse/env`` then
+1. If synapse was installed in a virtualenv then active that virtualenv before
+   upgrading. If synapse is installed in a virtualenv in ``~/.synapse/`` then
    run:
 
    .. code:: bash
 
-       source ~/synapse/env/bin/activate
+       source ~/.synapse/bin/activate
 
 2. If synapse was installed using pip then upgrade to the latest version by
    running:
 
    .. code:: bash
 
-       pip install --upgrade matrix-synapse[all]
+       pip install --upgrade --process-dependency-links matrix-synapse
 
        # restart synapse
        synctl restart
@@ -31,15 +31,14 @@ instructions that may be required are listed later in this document.
 
        # Pull the latest version of the master branch.
        git pull
-
-       # Update synapse and its python dependencies.
-       pip install --upgrade .[all]
+       # Update the versions of synapse's python dependencies.
+       python synapse/python_dependencies.py | xargs pip install --upgrade
 
        # restart synapse
        ./synctl restart
 
 
-To check whether your update was successful, you can check the Server header
+To check whether your update was sucessful, you can check the Server header
 returned by the Client-Server API:
 
 .. code:: bash
@@ -49,16 +48,6 @@ returned by the Client-Server API:
     # configured on port 443.
     curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"
 
-Upgrading to v0.99.0
-====================
-
-Please be aware that, before Synapse v1.0 is released around March 2019, you
-will need to replace any self-signed certificates with those verified by a
-root CA. Information on how to do so can be found at `the ACME docs
-<docs/ACME.md>`_.
-
-For more information on configuring TLS certificates see the `FAQ <docs/MSC1711_certificates_FAQ.md>`_.
-
 Upgrading to v0.34.0
 ====================
 

changelog.d/4406.feature

@@ -0,0 +1 @@
+Add a simple HTTP proxy option for Synapse to send federation traffic through before reaching other servers.

contrib/prometheus/README

@@ -6,10 +6,8 @@ To use it, first install prometheus by following the instructions at
   http://prometheus.io/
 
 ### for Prometheus v1
-
 Add a new job to the main prometheus.conf file:
 
-```yaml
   job: {
     name: "synapse"
 
@@ -17,12 +15,10 @@ Add a new job to the main prometheus.conf file:
       target: "http://SERVER.LOCATION.HERE:PORT/_synapse/metrics"
     }
   }
-```
 
 ### for Prometheus v2
 Add a new job to the main prometheus.yml file:
 
-```yaml
   - job_name: "synapse"
     metrics_path: "/_synapse/metrics"
     # when endpoint uses https:
@@ -30,14 +26,11 @@ Add a new job to the main prometheus.yml file:
 
     static_configs:
     - targets: ['SERVER.LOCATION:PORT']
-```
 
 To use `synapse.rules` add
 
-```yaml
     rule_files:
       - "/PATH/TO/synapse-v2.rules"
-```
 
 Metrics are disabled by default when running synapse; they must be enabled
 with the 'enable-metrics' option, either in the synapse config file or as a

debian/build_virtualenv

@@ -6,16 +6,7 @@
 set -e
 
 export DH_VIRTUALENV_INSTALL_ROOT=/opt/venvs
-
-# make sure that the virtualenv links to the specific version of python, by
-# dereferencing the python3 symlink.
-#
-# Otherwise, if somebody tries to install (say) the stretch package on buster,
-# they will get a confusing error about "No module named 'synapse'", because
-# python won't look in the right directory. At least this way, the error will
-# be a *bit* more obvious.
-#
-SNAKE=`readlink -e /usr/bin/python3`
+SNAKE=/usr/bin/python3
 
 # try to set the CFLAGS so any compiled C extensions are compiled with the most
 # generic as possible x64 instructions, so that compiling it on a new Intel chip
@@ -45,10 +36,6 @@ dh_virtualenv \
     --extra-pip-arg="--compile" \
     --extras="all"
 
-PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
-VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"
-TARGET_PYTHON="${VIRTUALENV_DIR}/bin/python"
-
 # we copy the tests to a temporary directory so that we can put them on the
 # PYTHONPATH without putting the uninstalled synapse on the pythonpath.
 tmpdir=`mktemp -d`
@@ -57,35 +44,5 @@ trap "rm -r $tmpdir" EXIT
 cp -r tests "$tmpdir"
 
 PYTHONPATH="$tmpdir" \
-    "${TARGET_PYTHON}" -B -m twisted.trial --reporter=text -j2 tests
-
-# build the config file
-"${TARGET_PYTHON}" -B "${VIRTUALENV_DIR}/bin/generate_config" \
-        --config-dir="/etc/matrix-synapse" \
-        --data-dir="/var/lib/matrix-synapse" |
-    perl -pe '
-# tweak the paths to the tls certs and signing keys
-/^tls_.*_path:/ and s/SERVERNAME/homeserver/;
-/^signing_key_path:/ and s/SERVERNAME/homeserver/;
-
-# tweak the pid file location
-/^pid_file:/ and s#:.*#: "/var/run/matrix-synapse.pid"#;
-
-# tweak the path to the log config
-/^log_config:/ and s/SERVERNAME\.log\.config/log.yaml/;
-
-# tweak the path to the media store
-/^media_store_path:/ and s#/media_store#/media#;
-
-# remove the server_name setting, which is set in a separate file
-/^server_name:/ and $_ = "#\n# This is set in /etc/matrix-synapse/conf.d/server_name.yaml for Debian installations.\n# $_";
-
-# remove the report_stats setting, which is set in a separate file
-/^# report_stats:/ and $_ = "";
-
-' > "${PACKAGE_BUILD_DIR}/etc/matrix-synapse/homeserver.yaml"
-
-
-# add a dependency on the right version of python to substvars.
-PYPKG=`basename $SNAKE`
-echo "synapse:pydepends=$PYPKG" >> debian/matrix-synapse-py3.substvars
+    debian/matrix-synapse-py3/opt/venvs/matrix-synapse/bin/python \
+        -B -m twisted.trial --reporter=text -j2 tests

debian/changelog

@@ -1,38 +1,3 @@
-matrix-synapse-py3 (0.99.2) stable; urgency=medium
-
-  * Fix overwriting of config settings on upgrade.
-  * New synapse release 0.99.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 01 Mar 2019 10:55:08 +0000
-
-matrix-synapse-py3 (0.99.1.1) stable; urgency=medium
-
-  * New synapse release 0.99.1.1
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 14 Feb 2019 17:19:44 +0000
-
-matrix-synapse-py3 (0.99.1) stable; urgency=medium
-
-  [ Damjan Georgievski ]
-  * Added ExecReload= in service unit file to send a HUP signal
-
-  [ Synapse Packaging team ]
-  * New synapse release 0.99.1
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 14 Feb 2019 14:12:26 +0000
-
-matrix-synapse-py3 (0.99.0) stable; urgency=medium
-
-  * New synapse release 0.99.0
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 5 Feb 2019 18:25:00 +0000
-
-matrix-synapse-py3 (0.34.1.1++1) stable; urgency=medium
-
-  * Update conflicts specifications to allow smoother transition from matrix-synapse.
-
- -- Synapse Packaging team <packages@matrix.org>  Sat, 12 Jan 2019 12:58:35 +0000
-
 matrix-synapse-py3 (0.34.1.1) stable; urgency=high
 
   * New synapse release 0.34.1.1

debian/config

@@ -4,9 +4,6 @@ set -e
 
 . /usr/share/debconf/confmodule
 
-# try to update the debconf db according to whatever is in the config files
-/opt/venvs/matrix-synapse/lib/manage_debconf.pl read || true
-
 db_input high matrix-synapse/server-name || true
 db_input high matrix-synapse/report-stats || true
 db_go

debian/control

@@ -19,16 +19,16 @@ Homepage: https://github.com/matrix-org/synapse
 Package: matrix-synapse-py3
 Architecture: amd64
 Provides: matrix-synapse
-Conflicts:
- matrix-synapse (<< 0.34.0.1-0matrix2),
- matrix-synapse (>= 0.34.0.1-1),
+Breaks:
+ matrix-synapse (<< 0.34.0-0matrix2),
+ matrix-synapse (>= 0.34.0-1),
 Pre-Depends: dpkg (>= 1.16.1)
 Depends:
  adduser,
  debconf,
  python3-distutils|libpython3-stdlib (<< 3.6),
+ python3,
  ${misc:Depends},
- ${synapse:pydepends},
 # some of our scripts use perl, but none of them are important,
 # so we put perl:Depends in Suggests rather than Depends.
 Suggests:

debian/homeserver.yaml

@@ -0,0 +1,617 @@
+# vim:ft=yaml
+# PEM encoded X509 certificate for TLS.
+# You can replace the self-signed certificate that synapse
+# autogenerates on launch with your own SSL certificate + key pair
+# if you like.  Any required intermediary certificates can be
+# appended after the primary certificate in hierarchical order.
+tls_certificate_path: "/etc/matrix-synapse/homeserver.tls.crt"
+
+# PEM encoded private key for TLS
+tls_private_key_path: "/etc/matrix-synapse/homeserver.tls.key"
+
+# PEM dh parameters for ephemeral keys
+tls_dh_params_path: "/etc/matrix-synapse/homeserver.tls.dh"
+
+# Don't bind to the https port
+no_tls: False
+
+# List of allowed TLS fingerprints for this server to publish along
+# with the signing keys for this server. Other matrix servers that
+# make HTTPS requests to this server will check that the TLS
+# certificates returned by this server match one of the fingerprints.
+#
+# Synapse automatically adds the fingerprint of its own certificate
+# to the list. So if federation traffic is handled directly by synapse
+# then no modification to the list is required.
+#
+# If synapse is run behind a load balancer that handles the TLS then it
+# will be necessary to add the fingerprints of the certificates used by
+# the loadbalancers to this list if they are different to the one
+# synapse is using.
+#
+# Homeservers are permitted to cache the list of TLS fingerprints
+# returned in the key responses up to the "valid_until_ts" returned in
+# key. It may be necessary to publish the fingerprints of a new
+# certificate and wait until the "valid_until_ts" of the previous key
+# responses have passed before deploying it.
+#
+# You can calculate a fingerprint from a given TLS listener via:
+# openssl s_client -connect $host:$port < /dev/null 2> /dev/null |
+#   openssl x509 -outform DER | openssl sha256 -binary | base64 | tr -d '='
+# or by checking matrix.org/federationtester/api/report?server_name=$host
+#
+tls_fingerprints: []
+# tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
+
+
+## Server ##
+
+# When running as a daemon, the file to store the pid in
+pid_file: "/var/run/matrix-synapse.pid"
+
+# CPU affinity mask. Setting this restricts the CPUs on which the
+# process will be scheduled. It is represented as a bitmask, with the
+# lowest order bit corresponding to the first logical CPU and the
+# highest order bit corresponding to the last logical CPU. Not all CPUs
+# may exist on a given system but a mask may specify more CPUs than are
+# present.
+#
+# For example:
+#    0x00000001  is processor #0,
+#    0x00000003  is processors #0 and #1,
+#    0xFFFFFFFF  is all processors (#0 through #31).
+#
+# Pinning a Python process to a single CPU is desirable, because Python
+# is inherently single-threaded due to the GIL, and can suffer a
+# 30-40% slowdown due to cache blow-out and thread context switching
+# if the scheduler happens to schedule the underlying threads across
+# different cores. See
+# https://www.mirantis.com/blog/improve-performance-python-programs-restricting-single-cpu/.
+#
+# cpu_affinity: 0xFFFFFFFF
+
+# The path to the web client which will be served at /_matrix/client/
+# if 'webclient' is configured under the 'listeners' configuration.
+#
+# web_client_location: "/path/to/web/root"
+
+# The public-facing base URL for the client API (not including _matrix/...)
+# public_baseurl: https://example.com:8448/
+
+# Set the soft limit on the number of file descriptors synapse can use
+# Zero is used to indicate synapse should set the soft limit to the
+# hard limit.
+soft_file_limit: 0
+
+# The GC threshold parameters to pass to `gc.set_threshold`, if defined
+# gc_thresholds: [700, 10, 10]
+
+# Set the limit on the returned events in the timeline in the get
+# and sync operations. The default value is -1, means no upper limit.
+# filter_timeline_limit: 5000
+
+# Whether room invites to users on this server should be blocked
+# (except those sent by local server admins). The default is False.
+# block_non_admin_invites: True
+
+# Restrict federation to the following whitelist of domains.
+# N.B. we recommend also firewalling your federation listener to limit
+# inbound federation traffic as early as possible, rather than relying
+# purely on this application-layer restriction.  If not specified, the
+# default is to whitelist everything.
+#
+# federation_domain_whitelist:
+#  - lon.example.com
+#  - nyc.example.com
+#  - syd.example.com
+
+# List of ports that Synapse should listen on, their purpose and their
+# configuration.
+listeners:
+  # Main HTTPS listener
+  # For when matrix traffic is sent directly to synapse.
+  -
+    # The port to listen for HTTPS requests on.
+    port: 8448
+
+    # Local addresses to listen on.
+    # On Linux and Mac OS, `::` will listen on all IPv4 and IPv6
+    # addresses by default. For most other OSes, this will only listen
+    # on IPv6.
+    bind_addresses:
+      - '::'
+      - '0.0.0.0'
+
+    # This is a 'http' listener, allows us to specify 'resources'.
+    type: http
+
+    tls: true
+
+    # Use the X-Forwarded-For (XFF) header as the client IP and not the
+    # actual client IP.
+    x_forwarded: false
+
+    # List of HTTP resources to serve on this listener.
+    resources:
+      -
+        # List of resources to host on this listener.
+        names:
+          - client     # The client-server APIs, both v1 and v2
+          - webclient  # The bundled webclient.
+
+        # Should synapse compress HTTP responses to clients that support it?
+        # This should be disabled if running synapse behind a load balancer
+        # that can do automatic compression.
+        compress: true
+
+      - names: [federation]  # Federation APIs
+        compress: false
+
+    # optional list of additional endpoints which can be loaded via
+    # dynamic modules
+    # additional_resources:
+    #   "/_matrix/my/custom/endpoint":
+    #     module: my_module.CustomRequestHandler
+    #     config: {}
+
+  # Unsecure HTTP listener,
+  # For when matrix traffic passes through loadbalancer that unwraps TLS.
+  - port: 8008
+    tls: false
+    bind_addresses: ['::', '0.0.0.0']
+    type: http
+
+    x_forwarded: false
+
+    resources:
+      - names: [client, webclient]
+        compress: true
+      - names: [federation]
+        compress: false
+
+  # Turn on the twisted ssh manhole service on localhost on the given
+  # port.
+  # - port: 9000
+  #   bind_addresses: ['::1', '127.0.0.1']
+  #   type: manhole
+
+
+# Database configuration
+database:
+  # The database engine name
+  name: "sqlite3"
+  # Arguments to pass to the engine
+  args:
+    # Path to the database
+    database: "/var/lib/matrix-synapse/homeserver.db"
+
+# Number of events to cache in memory.
+event_cache_size: "10K"
+
+
+# A yaml python logging config file
+log_config: "/etc/matrix-synapse/log.yaml"
+
+
+
+## Ratelimiting ##
+
+# Number of messages a client can send per second
+rc_messages_per_second: 0.2
+
+# Number of message a client can send before being throttled
+rc_message_burst_count: 10.0
+
+# The federation window size in milliseconds
+federation_rc_window_size: 1000
+
+# The number of federation requests from a single server in a window
+# before the server will delay processing the request.
+federation_rc_sleep_limit: 10
+
+# The duration in milliseconds to delay processing events from
+# remote servers by if they go over the sleep limit.
+federation_rc_sleep_delay: 500
+
+# The maximum number of concurrent federation requests allowed
+# from a single server
+federation_rc_reject_limit: 50
+
+# The number of federation requests to concurrently process from a
+# single server
+federation_rc_concurrent: 3
+
+
+
+# Directory where uploaded images and attachments are stored.
+media_store_path: "/var/lib/matrix-synapse/media"
+
+# Media storage providers allow media to be stored in different
+# locations.
+# media_storage_providers:
+# - module: file_system
+#   # Whether to write new local files.
+#   store_local: false
+#   # Whether to write new remote media
+#   store_remote: false
+#   # Whether to block upload requests waiting for write to this
+#   # provider to complete
+#   store_synchronous: false
+#   config:
+#     directory: /mnt/some/other/directory
+
+# Directory where in-progress uploads are stored.
+uploads_path: "/var/lib/matrix-synapse/uploads"
+
+# The largest allowed upload size in bytes
+max_upload_size: "10M"
+
+# Maximum number of pixels that will be thumbnailed
+max_image_pixels: "32M"
+
+# Whether to generate new thumbnails on the fly to precisely match
+# the resolution requested by the client. If true then whenever
+# a new resolution is requested by the client the server will
+# generate a new thumbnail. If false the server will pick a thumbnail
+# from a precalculated list.
+dynamic_thumbnails: false
+
+# List of thumbnail to precalculate when an image is uploaded.
+thumbnail_sizes:
+- width: 32
+  height: 32
+  method: crop
+- width: 96
+  height: 96
+  method: crop
+- width: 320
+  height: 240
+  method: scale
+- width: 640
+  height: 480
+  method: scale
+- width: 800
+  height: 600
+  method: scale
+
+# Is the preview URL API enabled?  If enabled, you *must* specify
+# an explicit url_preview_ip_range_blacklist of IPs that the spider is
+# denied from accessing.
+url_preview_enabled: False
+
+# List of IP address CIDR ranges that the URL preview spider is denied
+# from accessing.  There are no defaults: you must explicitly
+# specify a list for URL previewing to work.  You should specify any
+# internal services in your network that you do not want synapse to try
+# to connect to, otherwise anyone in any Matrix room could cause your
+# synapse to issue arbitrary GET requests to your internal services,
+# causing serious security issues.
+#
+# url_preview_ip_range_blacklist:
+# - '127.0.0.0/8'
+# - '10.0.0.0/8'
+# - '172.16.0.0/12'
+# - '192.168.0.0/16'
+# - '100.64.0.0/10'
+# - '169.254.0.0/16'
+#
+# List of IP address CIDR ranges that the URL preview spider is allowed
+# to access even if they are specified in url_preview_ip_range_blacklist.
+# This is useful for specifying exceptions to wide-ranging blacklisted
+# target IP ranges - e.g. for enabling URL previews for a specific private
+# website only visible in your network.
+#
+# url_preview_ip_range_whitelist:
+# - '192.168.1.1'
+
+# Optional list of URL matches that the URL preview spider is
+# denied from accessing.  You should use url_preview_ip_range_blacklist
+# in preference to this, otherwise someone could define a public DNS
+# entry that points to a private IP address and circumvent the blacklist.
+# This is more useful if you know there is an entire shape of URL that
+# you know that will never want synapse to try to spider.
+#
+# Each list entry is a dictionary of url component attributes as returned
+# by urlparse.urlsplit as applied to the absolute form of the URL.  See
+# https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit
+# The values of the dictionary are treated as an filename match pattern
+# applied to that component of URLs, unless they start with a ^ in which
+# case they are treated as a regular expression match.  If all the
+# specified component matches for a given list item succeed, the URL is
+# blacklisted.
+#
+# url_preview_url_blacklist:
+# # blacklist any URL with a username in its URI
+# - username: '*'
+#
+# # blacklist all *.google.com URLs
+# - netloc: 'google.com'
+# - netloc: '*.google.com'
+#
+# # blacklist all plain HTTP URLs
+# - scheme: 'http'
+#
+# # blacklist http(s)://www.acme.com/foo
+# - netloc: 'www.acme.com'
+#   path: '/foo'
+#
+# # blacklist any URL with a literal IPv4 address
+# - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'
+
+# The largest allowed URL preview spidering size in bytes
+max_spider_size: "10M"
+
+
+
+
+## Captcha ##
+# See docs/CAPTCHA_SETUP for full details of configuring this.
+
+# This Home Server's ReCAPTCHA public key.
+recaptcha_public_key: "YOUR_PUBLIC_KEY"
+
+# This Home Server's ReCAPTCHA private key.
+recaptcha_private_key: "YOUR_PRIVATE_KEY"
+
+# Enables ReCaptcha checks when registering, preventing signup
+# unless a captcha is answered. Requires a valid ReCaptcha
+# public/private key.
+enable_registration_captcha: False
+
+# A secret key used to bypass the captcha test entirely.
+#captcha_bypass_secret: "YOUR_SECRET_HERE"
+
+# The API endpoint to use for verifying m.login.recaptcha responses.
+recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify"
+
+
+## Turn ##
+
+# The public URIs of the TURN server to give to clients
+turn_uris: []
+
+# The shared secret used to compute passwords for the TURN server
+turn_shared_secret: "YOUR_SHARED_SECRET"
+
+# The Username and password if the TURN server needs them and
+# does not use a token
+#turn_username: "TURNSERVER_USERNAME"
+#turn_password: "TURNSERVER_PASSWORD"
+
+# How long generated TURN credentials last
+turn_user_lifetime: "1h"
+
+# Whether guests should be allowed to use the TURN server.
+# This defaults to True, otherwise VoIP will be unreliable for guests.
+# However, it does introduce a slight security risk as it allows users to
+# connect to arbitrary endpoints without having first signed up for a
+# valid account (e.g. by passing a CAPTCHA).
+turn_allow_guests: False
+
+
+## Registration ##
+
+# Enable registration for new users.
+enable_registration: False
+
+# The user must provide all of the below types of 3PID when registering.
+#
+# registrations_require_3pid:
+#     - email
+#     - msisdn
+
+# Mandate that users are only allowed to associate certain formats of
+# 3PIDs with accounts on this server.
+#
+# allowed_local_3pids:
+#     - medium: email
+#       pattern: ".*@matrix\.org"
+#     - medium: email
+#       pattern: ".*@vector\.im"
+#     - medium: msisdn
+#       pattern: "\+44"
+
+# If set, allows registration by anyone who also has the shared
+# secret, even if registration is otherwise disabled.
+# registration_shared_secret: <PRIVATE STRING>
+
+# Set the number of bcrypt rounds used to generate password hash.
+# Larger numbers increase the work factor needed to generate the hash.
+# The default number is 12 (which equates to 2^12 rounds).
+# N.B. that increasing this will exponentially increase the time required
+# to register or login - e.g. 24 => 2^24 rounds which will take >20 mins.
+bcrypt_rounds: 12
+
+# Allows users to register as guests without a password/email/etc, and
+# participate in rooms hosted on this server which have been made
+# accessible to anonymous users.
+allow_guest_access: False
+
+# The list of identity servers trusted to verify third party
+# identifiers by this server.
+trusted_third_party_id_servers:
+    - matrix.org
+    - vector.im
+    - riot.im
+
+# Users who register on this homeserver will automatically be joined
+# to these rooms
+#auto_join_rooms:
+#    - "#example:example.com"
+
+
+## Metrics ###
+
+# Enable collection and rendering of performance metrics
+enable_metrics: False
+
+## API Configuration ##
+
+# A list of event types that will be included in the room_invite_state
+room_invite_state_types:
+    - "m.room.join_rules"
+    - "m.room.canonical_alias"
+    - "m.room.avatar"
+    - "m.room.name"
+
+
+# A list of application service config file to use
+app_service_config_files: []
+
+
+# macaroon_secret_key: <PRIVATE STRING>
+
+# Used to enable access token expiration.
+expire_access_token: False
+
+## Signing Keys ##
+
+# Path to the signing key to sign messages with
+signing_key_path: "/etc/matrix-synapse/homeserver.signing.key"
+
+# The keys that the server used to sign messages with but won't use
+# to sign new messages. E.g. it has lost its private key
+old_signing_keys: {}
+#  "ed25519:auto":
+#    # Base64 encoded public key
+#    key: "The public part of your old signing key."
+#    # Millisecond POSIX timestamp when the key expired.
+#    expired_ts: 123456789123
+
+# How long key response published by this server is valid for.
+# Used to set the valid_until_ts in /key/v2 APIs.
+# Determines how quickly servers will query to check which keys
+# are still valid.
+key_refresh_interval: "1d" # 1 Day.
+
+# The trusted servers to download signing keys from.
+perspectives:
+  servers:
+    "matrix.org":
+      verify_keys:
+        "ed25519:auto":
+          key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"
+
+
+
+# Enable SAML2 for registration and login. Uses pysaml2
+# config_path:      Path to the sp_conf.py configuration file
+# idp_redirect_url: Identity provider URL which will redirect
+#                   the user back to /login/saml2 with proper info.
+# See pysaml2 docs for format of config.
+#saml2_config:
+#   enabled: true
+#   config_path: "/home/erikj/git/synapse/sp_conf.py"
+#   idp_redirect_url: "http://test/idp"
+
+
+
+# Enable CAS for registration and login.
+#cas_config:
+#   enabled: true
+#   server_url: "https://cas-server.com"
+#   service_url: "https://homeserver.domain.com:8448"
+#   #required_attributes:
+#   #    name: value
+
+
+# The JWT needs to contain a globally unique "sub" (subject) claim.
+#
+# jwt_config:
+#    enabled: true
+#    secret: "a secret"
+#    algorithm: "HS256"
+
+
+
+# Enable password for login.
+password_config:
+   enabled: true
+   # Uncomment and change to a secret random string for extra security.
+   # DO NOT CHANGE THIS AFTER INITIAL SETUP!
+   #pepper: ""
+
+
+
+# Enable sending emails for notification events
+# Defining a custom URL for Riot is only needed if email notifications
+# should contain links to a self-hosted installation of Riot; when set
+# the "app_name" setting is ignored.
+#
+# If your SMTP server requires authentication, the optional smtp_user &
+# smtp_pass variables should be used
+#
+#email:
+#   enable_notifs: false
+#   smtp_host: "localhost"
+#   smtp_port: 25
+#   smtp_user: "exampleusername"
+#   smtp_pass: "examplepassword"
+#   require_transport_security: False
+#   notif_from: "Your Friendly %(app)s Home Server <noreply@example.com>"
+#   app_name: Matrix
+#   template_dir: res/templates
+#   notif_template_html: notif_mail.html
+#   notif_template_text: notif_mail.txt
+#   notif_for_new_users: True
+#   riot_base_url: "http://localhost/riot"
+
+
+# password_providers:
+#     - module: "ldap_auth_provider.LdapAuthProvider"
+#       config:
+#         enabled: true
+#         uri: "ldap://ldap.example.com:389"
+#         start_tls: true
+#         base: "ou=users,dc=example,dc=com"
+#         attributes:
+#            uid: "cn"
+#            mail: "email"
+#            name: "givenName"
+#         #bind_dn:
+#         #bind_password:
+#         #filter: "(objectClass=posixAccount)"
+
+
+
+# Clients requesting push notifications can either have the body of
+# the message sent in the notification poke along with other details
+# like the sender, or just the event ID and room ID (`event_id_only`).
+# If clients choose the former, this option controls whether the
+# notification request includes the content of the event (other details
+# like the sender are still included). For `event_id_only` push, it
+# has no effect.
+
+# For modern android devices the notification content will still appear
+# because it is loaded by the app. iPhone, however will send a
+# notification saying only that a message arrived and who it came from.
+#
+#push:
+#   include_content: true
+
+
+# spam_checker:
+#     module: "my_custom_project.SuperSpamChecker"
+#     config:
+#         example_option: 'things'
+
+
+# Whether to allow non server admins to create groups on this server
+enable_group_creation: false
+
+# If enabled, non server admins can only create groups with local parts
+# starting with this prefix
+# group_creation_prefix: "unofficial/"
+
+
+
+# User Directory configuration
+#
+# 'search_all_users' defines whether to search all users visible to your HS
+# when searching the user directory, rather than limiting to users visible
+# in public rooms.  Defaults to false.  If you set it True, you'll have to run
+# UPDATE user_directory_stream_pos SET stream_id = NULL;
+# on your database to tell it to rebuild the user_directory search indexes.
+#
+#user_directory:
+#   search_all_users: false

debian/install

@@ -1,2 +1,2 @@
+debian/homeserver.yaml etc/matrix-synapse
 debian/log.yaml etc/matrix-synapse
-debian/manage_debconf.pl /opt/venvs/matrix-synapse/lib/

debian/manage_debconf.pl

@@ -1,130 +0,0 @@
-#!/usr/bin/perl
-#
-# Interface between our config files and the debconf database.
-#
-# Usage:
-#
-#   manage_debconf.pl <action>
-#
-# where <action> can be:
-#
-#   read:    read the configuration from the yaml into debconf
-#   update:  update the yaml config according to the debconf database
-use strict;
-use warnings;
-
-use Debconf::Client::ConfModule (qw/get set/);
-
-# map from the name of a setting in our .yaml file to the relevant debconf
-# setting.
-my %MAPPINGS=(
-    server_name => 'matrix-synapse/server-name',
-    report_stats => 'matrix-synapse/report-stats',
-);
-
-# enable debug if dpkg --debug
-my $DEBUG = $ENV{DPKG_MAINTSCRIPT_DEBUG};
-
-sub read_config {
-    my @files = @_;
-
-    foreach my $file (@files)  {
-        print STDERR "reading $file\n" if $DEBUG;
-
-        open my $FH, "<", $file or next;
-
-        # rudimentary parsing which (a) avoids having to depend on a yaml library,
-        # and (b) is tolerant of yaml errors
-        while($_ = <$FH>) {
-            while (my ($setting, $debconf) = each %MAPPINGS) {
-                $setting = quotemeta $setting;
-                if(/^${setting}\s*:(.*)$/) {
-                    my $val = $1;
-
-                    # remove leading/trailing whitespace
-                    $val =~ s/^\s*//;
-                    $val =~ s/\s*$//;
-
-                    # remove surrounding quotes
-                    if ($val =~ /^"(.*)"$/ || $val =~ /^'(.*)'$/) {
-                        $val = $1;
-                    }
-
-                    print STDERR ">> $debconf = $val\n" if $DEBUG;
-                    set($debconf, $val);
-                }
-            }
-        }
-        close $FH;
-    }
-}
-
-sub update_config {
-    my @files = @_;
-
-    my %substs = ();
-    while (my ($setting, $debconf) = each %MAPPINGS) {
-        my @res = get($debconf);
-        $substs{$setting} = $res[1] if $res[0] == 0;
-    }
-
-    foreach my $file (@files) {
-        print STDERR "checking $file\n" if $DEBUG;
-
-        open my $FH, "<", $file or next;
-
-        my $updated = 0;
-
-        # read the whole file into memory
-        my @lines = <$FH>;
-
-        while (my ($setting, $val) = each %substs) {
-            $setting = quotemeta $setting;
-
-            map {
-                if (/^${setting}\s*:\s*(.*)\s*$/) {
-                    my $current = $1;
-                    if ($val ne $current) {
-                        $_ = "${setting}: $val\n";
-                        $updated = 1;
-                    }
-                }
-            } @lines;
-        }
-        close $FH;
-
-        next unless $updated;
-
-        print STDERR "updating $file\n" if $DEBUG;
-        open $FH, ">", $file or die "unable to update $file";
-        print $FH @lines;
-        close $FH;
-    }
-}
-
-
-my $cmd = $ARGV[0];
-
-my $read = 0;
-my $update = 0;
-
-if (not $cmd) {
-    die "must specify a command to perform\n";
-} elsif ($cmd eq 'read') {
-    $read = 1;
-} elsif ($cmd eq 'update') {
-    $update = 1;
-} else {
-    die "unknown command '$cmd'\n";
-}
-
-my @files = (
-    "/etc/matrix-synapse/homeserver.yaml",
-    glob("/etc/matrix-synapse/conf.d/*.yaml"),
-);
-
-if ($read) {
-    read_config(@files);
-} elsif ($update) {
-    update_config(@files);
-}

debian/matrix-synapse-py3.postinst

@@ -8,36 +8,19 @@ USER="matrix-synapse"
 
 case "$1" in
   configure|reconfigure)
-
-    # generate template config files if they don't exist
+    # Set server name in config file
     mkdir -p "/etc/matrix-synapse/conf.d/"
-    if [ ! -e "$CONFIGFILE_SERVERNAME" ]; then
-        cat > "$CONFIGFILE_SERVERNAME" <<EOF
-# This file is autogenerated, and will be recreated on upgrade if it is deleted.
-# Any changes you make will be preserved.
+    db_get matrix-synapse/server-name
 
-# The domain name of the server, with optional explicit port.
-# This is used by remote servers to connect to this server,
-# e.g. matrix.org, localhost:8080, etc.
-# This is also the last part of your UserID.
-#
-server_name: ''
-EOF
+    if [ "$RET" ]; then
+        echo "server_name: $RET" > $CONFIGFILE_SERVERNAME
     fi
 
-    if [ ! -e "$CONFIGFILE_REPORTSTATS" ]; then
-        cat > "$CONFIGFILE_REPORTSTATS" <<EOF
-# This file is autogenerated, and will be recreated on upgrade if it is deleted.
-# Any changes you make will be preserved.
-
-# Whether to report anonymized homeserver usage statistics.
-report_stats: false
-EOF
+    db_get matrix-synapse/report-stats
+    if [ "$RET" ]; then
+        echo "report_stats: $RET" > $CONFIGFILE_REPORTSTATS
     fi
 
-    # update the config files according to whatever is in the debconf database
-    /opt/venvs/matrix-synapse/lib/manage_debconf.pl update
-
     if ! getent passwd $USER >/dev/null; then
       adduser --quiet --system --no-create-home --home /var/lib/matrix-synapse $USER
     fi

debian/matrix-synapse.service

@@ -8,7 +8,6 @@ WorkingDirectory=/var/lib/matrix-synapse
 EnvironmentFile=/etc/default/matrix-synapse
 ExecStartPre=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys
 ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/
-ExecReload=/bin/kill -HUP $MAINPID
 Restart=always
 RestartSec=3
 

demo/.gitignore

@@ -1,7 +0,0 @@
-*.db
-*.log
-*.log.*
-*.pid
-
-/media_store.*
-/etc

demo/demo.tls.dh

@@ -0,0 +1,9 @@
+2048-bit DH parameters taken from rfc3526
+-----BEGIN DH PARAMETERS-----
+MIIBCAKCAQEA///////////JD9qiIWjCNMTGYouA3BzRKQJOCIpnzHQCC76mOxOb
+IlFKCHmONATd75UZs806QxswKwpt8l8UN0/hNW1tUcJF5IW1dmJefsb0TELppjft
+awv/XLb0Brft7jhr+1qJn6WunyQRfEsf5kkoZlHs5Fs9wgB8uKFjvwWY2kg2HFXT
+mmkWP6j9JM9fg2VdI9yjrZYcYvNWIIVSu57VKQdwlpZtZww1Tkq8mATxdGwIyhgh
+fDKQXkYuNs474553LBgOhgObJ4Oi7Aeij7XFXfBvTFLJ3ivL9pVYFxg5lUl86pVq
+5RXSJhiY+gUQFXKOWoqsqmj//////////wIBAg==
+-----END DH PARAMETERS-----

docker/Dockerfile

@@ -1,16 +1,3 @@
-# Dockerfile to build the matrixdotorg/synapse docker images.
-#
-# To build the image, run `docker build` command from the root of the
-# synapse repository:
-#
-#    docker build -f docker/Dockerfile .
-#
-# There is an optional PYTHON_VERSION build argument which sets the
-# version of python to build against: for example:
-#
-#    docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.6 .
-#
-
 ARG PYTHON_VERSION=2
 
 ###
@@ -44,10 +31,7 @@ RUN pip install --prefix="/install" --no-warn-script-location \
 
 # now install synapse and all of the python deps to /install.
 
-COPY synapse /synapse/synapse/
-COPY scripts /synapse/scripts/
-COPY MANIFEST.in README.rst setup.py synctl /synapse/
-
+COPY . /synapse
 RUN pip install --prefix="/install" --no-warn-script-location \
         /synapse[all]
 
@@ -72,6 +56,6 @@ COPY ./docker/conf /conf
 
 VOLUME ["/data"]
 
-EXPOSE 8008/tcp 8009/tcp 8448/tcp
+EXPOSE 8008/tcp 8448/tcp
 
 ENTRYPOINT ["/start.py"]

docker/Dockerfile-dhvirtualenv

@@ -58,11 +58,7 @@ RUN apt-get update -qq -o Acquire::Languages=none \
         sqlite3
 
 COPY --from=builder /dh-virtualenv_1.1-1_all.deb /
-
-# install dhvirtualenv. Update the apt cache again first, in case we got a
-# cached cache from docker the first time.
-RUN apt-get update -qq -o Acquire::Languages=none \
-    && apt-get install -yq /dh-virtualenv_1.1-1_all.deb
+RUN apt-get install -yq /dh-virtualenv_1.1-1_all.deb
 
 WORKDIR /synapse/source
 ENTRYPOINT ["bash","/synapse/source/docker/build_debian.sh"]

docker/README.md

@@ -1,21 +1,22 @@
 # Synapse Docker
 
-This Docker image will run Synapse as a single process. By default it uses a
-sqlite database; for production use you should connect it to a separate
-postgres database.
-
-The image also does *not* provide a TURN server.
+This Docker image will run Synapse as a single process. It does not provide a database
+server or a TURN server, you should run these separately.
 
 ## Run
 
+We do not currently offer a `latest` image, as this has somewhat undefined semantics.
+We instead release only tagged versions so upgrading between releases is entirely
+within your control.
+
 ### Using docker-compose (easier)
 
-This image is designed to run either with an automatically generated
-configuration file or with a custom configuration that requires manual editing.
+This image is designed to run either with an automatically generated configuration
+file or with a custom configuration that requires manual editing.
 
 An easy way to make use of this image is via docker-compose. See the
-[contrib/docker](../contrib/docker) section of the synapse project for
-examples.
+[contrib/docker](../contrib/docker)
+section of the synapse project for examples.
 
 ### Without Compose (harder)
 
@@ -31,7 +32,7 @@ docker run \
     -v ${DATA_PATH}:/data \
     -e SYNAPSE_SERVER_NAME=my.matrix.host \
     -e SYNAPSE_REPORT_STATS=yes \
-    matrixdotorg/synapse:latest
+    docker.io/matrixdotorg/synapse:latest
 ```
 
 ## Volumes
@@ -52,28 +53,6 @@ In order to setup an application service, simply create an ``appservices``
 directory in the data volume and write the application service Yaml
 configuration file there. Multiple application services are supported.
 
-## TLS certificates
-
-Synapse requires a valid TLS certificate. You can do one of the following:
-
- * Provide your own certificate and key (as
-   `${DATA_PATH}/${SYNAPSE_SERVER_NAME}.crt` and
-   `${DATA_PATH}/${SYNAPSE_SERVER_NAME}.key`, or elsewhere by providing an
-   entire config as `${SYNAPSE_CONFIG_PATH}`).
-
- * Use a reverse proxy to terminate incoming TLS, and forward the plain http
-   traffic to port 8008 in the container. In this case you should set `-e
-   SYNAPSE_NO_TLS=1`.
-
- * Use the ACME (Let's Encrypt) support built into Synapse. This requires
-   `${SYNAPSE_SERVER_NAME}` port 80 to be forwarded to port 8009 in the
-   container, for example with `-p 80:8009`. To enable it in the docker
-   container, set `-e SYNAPSE_ACME=1`.
-
-If you don't do any of these, Synapse will fail to start with an error similar to:
-
-    synapse.config._base.ConfigError: Error accessing file '/data/<server_name>.tls.crt' (config for tls_certificate): No such file or directory
-
 ## Environment
 
 Unless you specify a custom path for the configuration file, a very generic
@@ -92,7 +71,7 @@ then customize it manually. No other environment variable is required.
 Otherwise, a dynamic configuration file will be used. The following environment
 variables are available for configuration:
 
-* ``SYNAPSE_SERVER_NAME`` (mandatory), the server public hostname.
+* ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname.
 * ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous
   statistics reporting back to the Matrix project which helps us to get funding.
 * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if
@@ -101,6 +80,7 @@ variables are available for configuration:
   the Synapse instance.
 * ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server.
 * ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`].
+* ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`].
 * ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public
   key in order to enable recaptcha upon registration.
 * ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private
@@ -108,9 +88,7 @@ variables are available for configuration:
 * ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN
   uris to enable TURN for this homeserver.
 * ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required.
-* ``SYNAPSE_MAX_UPLOAD_SIZE``, set this variable to change the max upload size
-  [default `10M`].
-* ``SYNAPSE_ACME``: set this to enable the ACME certificate renewal support.
+* ``SYNAPSE_MAX_UPLOAD_SIZE``, set this variable to change the max upload size [default `10M`].
 
 Shared secrets, that will be initialized to random values if not set:
 
@@ -121,25 +99,27 @@ Shared secrets, that will be initialized to random values if not set:
 
 Database specific values (will use SQLite if not set):
 
-* `POSTGRES_DB` - The database name for the synapse postgres
-  database. [default: `synapse`]
-* `POSTGRES_HOST` - The host of the postgres database if you wish to use
-  postgresql instead of sqlite3. [default: `db` which is useful when using a
-  container on the same docker network in a compose file where the postgres
-  service is called `db`]
-* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If
-  this is set then postgres will be used instead of sqlite3.** [default: none]
-  **NOTE**: You are highly encouraged to use postgresql! Please use the compose
-  file to make it easier to deploy.
-* `POSTGRES_USER` - The user for the synapse postgres database. [default:
-  `matrix`]
+* `POSTGRES_DB` - The database name for the synapse postgres database. [default: `synapse`]
+* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`]
+* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy.
+* `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`]
 
 Mail server specific values (will not send emails if not set):
 
 * ``SYNAPSE_SMTP_HOST``, hostname to the mail server.
-* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default
-  ``25``].
-* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if
-  any.
-* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail
-  server if any.
+* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default ``25``].
+* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if any.
+* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail server if any.
+
+## Build
+
+Build the docker image with the `docker build` command from the root of the synapse repository.
+
+```
+docker build -t docker.io/matrixdotorg/synapse . -f docker/Dockerfile
+```
+
+The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:<version>` where `<version>` is the same as the release tag in the synapse git repository.
+
+You may have a local Python wheel cache available, in which case copy the relevant
+packages in the ``cache/`` directory at the root of the project.

docker/build_debian_packages.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Build the Debian packages using Docker images.
+#
+# This script builds the Docker images and then executes them sequentially, each
+# one building a Debian package for the targeted operating system. It is
+# designed to be a "single command" to produce all the images.
+#
+# By default, builds for all known distributions, but a list of distributions
+# can be passed on the commandline for debugging.
+
+set -ex
+
+cd `dirname $0`
+
+if [ $# -lt 1 ]; then
+    DISTS=(
+        debian:stretch
+        debian:buster
+        debian:sid
+        ubuntu:xenial
+        ubuntu:bionic
+        ubuntu:cosmic
+    )
+else
+    DISTS=("$@")
+fi
+
+# Make the dir where the debs will live.
+#
+# Note that we deliberately put this outside the source tree, otherwise we tend
+# to get source packages which are full of debs. (We could hack around that
+# with more magic in the build_debian.sh script, but that doesn't solve the
+# problem for natively-run dpkg-buildpakage).
+
+mkdir -p ../../debs
+
+# Build each OS image;
+for i in "${DISTS[@]}"; do
+    TAG=$(echo ${i} | cut -d ":" -f 2)
+    docker build --tag dh-venv-builder:${TAG} --build-arg distro=${i} -f Dockerfile-dhvirtualenv .
+    docker run -it --rm --volume=$(pwd)/../\:/synapse/source:ro --volume=$(pwd)/../../debs:/debs \
+           -e TARGET_USERID=$(id -u) \
+           -e TARGET_GROUPID=$(id -g) \
+           dh-venv-builder:${TAG}
+done

docker/conf/homeserver.yaml

@@ -2,18 +2,11 @@
 
 ## TLS ##
 
-{% if not SYNAPSE_NO_TLS %}
-
 tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt"
 tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key"
-
-{% if SYNAPSE_ACME %}
-acme:
-    enabled: true
-    port: 8009
-{% endif %}
-
-{% endif %}
+tls_dh_params_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.dh"
+no_tls: {{ "True" if SYNAPSE_NO_TLS else "False" }}
+tls_fingerprints: []
 
 ## Server ##
 

docker/start.py

@@ -47,8 +47,9 @@ if mode == "generate":
 
 # In normal mode, generate missing keys if any, then run synapse
 else:
+    # Parse the configuration file
     if "SYNAPSE_CONFIG_PATH" in environ:
-        config_path = environ["SYNAPSE_CONFIG_PATH"]
+        args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]]
     else:
         check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"))
         generate_secrets(environ, {
@@ -57,21 +58,10 @@ else:
         })
         environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml")
         if not os.path.exists("/compiled"): os.mkdir("/compiled")
-
-        config_path = "/compiled/homeserver.yaml"
-
-        convert("/conf/homeserver.yaml", config_path, environ)
+        convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ)
         convert("/conf/log.config", "/compiled/log.config", environ)
         subprocess.check_output(["chown", "-R", ownership, "/data"])
-
-
-    args += [
-        "--config-path", config_path,
-
-        # tell synapse to put any generated keys in /data rather than /compiled
-        "--keys-directory", "/data",
-    ]
-
+        args += ["--config-path", "/compiled/homeserver.yaml"]
     # Generate missing keys and start synapse
     subprocess.check_output(args + ["--generate-keys"])
     os.execv("/sbin/su-exec", ["su-exec", ownership] + args)

docs/ACME.md

@@ -1,129 +0,0 @@
-# ACME
-
-Synapse v1.0 will require valid TLS certificates for communication between
-servers (port `8448` by default) in addition to those that are client-facing
-(port `443`). If you do not already have a valid certificate for your domain,
-the easiest way to get one is with Synapse's new ACME support, which will use
-the ACME protocol to provision a certificate automatically. Synapse v0.99.0+
-will provision server-to-server certificates automatically for you for free
-through [Let's Encrypt](https://letsencrypt.org/) if you tell it to.
-
-In the case that your `server_name` config variable is the same as
-the hostname that the client connects to, then the same certificate can be
-used between client and federation ports without issue.
-
-If your configuration file does not already have an `acme` section, you can
-generate an example config by running the `generate_config` executable. For
-example:
-
-```
-~/synapse/env3/bin/generate_config
-```
-
-You will need to provide Let's Encrypt (or another ACME provider) access to
-your Synapse ACME challenge responder on port 80, at the domain of your
-homeserver. This requires you to either change the port of the ACME listener
-provided by Synapse to a high port and reverse proxy to it, or use a tool
-like `authbind` to allow Synapse to listen on port 80 without root access.
-(Do not run Synapse with root permissions!) Detailed instructions are
-available under "ACME setup" below.
-
-If you already have certificates, you will need to back up or delete them
-(files `example.com.tls.crt` and `example.com.tls.key` in Synapse's root
-directory), Synapse's ACME implementation will not overwrite them.
-
-You may wish to use alternate methods such as Certbot to obtain a certificate
-from Let's Encrypt, depending on your server configuration. Of course, if you
-already have a valid certificate for your homeserver's domain, that can be
-placed in Synapse's config directory without the need for any ACME setup.
-
-## ACME setup
-
-The main steps for enabling ACME support in short summary are:
-
-1. Allow Synapse to listen for incoming ACME challenges.
-1. Enable ACME support in `homeserver.yaml`.
-1. Move your old certificates (files `example.com.tls.crt` and `example.com.tls.key` out of the way if they currently exist at the paths specified in `homeserver.yaml`.
-1. Restart Synapse.
-
-Detailed instructions for each step are provided below.
-
-### Listening on port 80
-
-In order for Synapse to complete the ACME challenge to provision a
-certificate, it needs access to port 80. Typically listening on port 80 is
-only granted to applications running as root. There are thus two solutions to
-this problem.
-
-#### Using a reverse proxy
-
-A reverse proxy such as Apache or nginx allows a single process (the web
-server) to listen on port 80 and proxy traffic to the appropriate program
-running on your server. It is the recommended method for setting up ACME as
-it allows you to use your existing webserver while also allowing Synapse to
-provision certificates as needed.
-
-For nginx users, add the following line to your existing `server` block:
-
-```
-location /.well-known/acme-challenge {
-    proxy_pass http://localhost:8009/;
-}
-```
-
-For Apache, add the following to your existing webserver config:
-
-```
-ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge
-```
-
-Make sure to restart/reload your webserver after making changes.
-
-Now make the relevant changes in `homeserver.yaml` to enable ACME support:
-
-```
-acme:
-    enabled: true
-    port: 8009
-```
-
-#### Authbind
-
-`authbind` allows a program which does not run as root to bind to
-low-numbered ports in a controlled way. The setup is simpler, but requires a
-webserver not to already be running on port 80. **This includes every time
-Synapse renews a certificate**, which may be cumbersome if you usually run a
-web server on port 80. Nevertheless, if you're sure port 80 is not being used
-for any other purpose then all that is necessary is the following:
-
-Install `authbind`. For example, on Debian/Ubuntu:
-
-```
-sudo apt-get install authbind
-```
-
-Allow `authbind` to bind port 80:
-
-```
-sudo touch /etc/authbind/byport/80
-sudo chmod 777 /etc/authbind/byport/80
-```
-
-When Synapse is started, use the following syntax:
-
-```
-authbind --deep <synapse start command>
-```
-
-Make the relevant changes in `homeserver.yaml` to enable ACME support:
-
-```
-acme:
-    enabled: true
-```
-
-### (Re)starting synapse
-
-Ensure that the certificate paths specified in `homeserver.yaml` (`tls_certificate_path` and `tls_private_key_path`) do not currently point to any files. Synapse will not provision certificates if files exist, as it does not want to overwrite existing certificates.
-
-Finally, start/restart Synapse.

docs/MSC1711_certificates_FAQ.md

@@ -1,338 +0,0 @@
-# MSC1711 Certificates FAQ
-
-The goal of Synapse 0.99.0 is to act as a stepping stone to Synapse 1.0.0. It
-supports the r0.1 release of the server to server specification, but is
-compatible with both the legacy Matrix federation behaviour (pre-r0.1) as well
-as post-r0.1 behaviour, in order to allow for a smooth upgrade across the
-federation.
-
-The most important thing to know is that Synapse 1.0.0 will require a valid TLS
-certificate on federation endpoints. Self signed certificates will not be
-sufficient.
-
-Synapse 0.99.0 makes it easy to configure TLS certificates and will
-interoperate with both >= 1.0.0 servers as well as existing servers yet to
-upgrade.
-
-**It is critical that all admins upgrade to 0.99.0 and configure a valid TLS
-certificate.** Admins will have 1 month to do so, after which 1.0.0 will be
-released and those servers without a valid certificate will not longer be able
-to federate with >= 1.0.0 servers.
-
-Full details on how to carry out this configuration change is given
-[below](#configuring-certificates-for-compatibility-with-synapse-100). A
-timeline and some frequently asked questions are also given below.
-
-For more details and context on the release of the r0.1 Server/Server API and
-imminent Matrix 1.0 release, you can also see our
-[main talk from FOSDEM 2019](https://matrix.org/blog/2019/02/04/matrix-at-fosdem-2019/).
-
-## Contents
-* Timeline
-* Configuring certificates for compatibility with Synapse 1.0
-* FAQ
-  * Synapse 0.99.0 has just been released, what do I need to do right now?
-  * How do I upgrade?
-  * What will happen if I do not set up a valid federation certificate
-    immediately?
-  * What will happen if I do nothing at all?
-  * When do I need a SRV record or .well-known URI?
-  * Can I still use an SRV record?
-  * I have created a .well-known URI. Do I still need an SRV record?
-  * It used to work just fine, why are you breaking everything?
-  * Can I manage my own certificates rather than having Synapse renew
-    certificates itself?
-  * Do you still recommend against using a reverse proxy on the federation port?
-  * Do I still need to give my TLS certificates to Synapse if I am using a
-    reverse proxy?
-  * Do I need the same certificate for the client and federation port?
-  * How do I tell Synapse to reload my keys/certificates after I replace them?
-
-## Timeline
-
-**5th Feb 2019  - Synapse 0.99.0 is released.**
-
-All server admins are encouraged to upgrade.
-
-0.99.0:
-
--   provides support for ACME to make setting up Let's Encrypt certs easy, as
-    well as .well-known support.
-
--   does not enforce that a valid CA cert is present on the federation API, but
-    rather makes it easy to set one up.
-
--   provides support for .well-known
-
-Admins should upgrade and configure a valid CA cert. Homeservers that require a
-.well-known entry (see below), should retain their SRV record and use it
-alongside their .well-known record.
-
-**>= 5th March 2019  - Synapse 1.0.0 is released**
-
-1.0.0 will land no sooner than 1 month after 0.99.0, leaving server admins one
-month after 5th February to upgrade to 0.99.0 and deploy their certificates. In
-accordance with the the [S2S spec](https://matrix.org/docs/spec/server_server/r0.1.0.html)
-1.0.0 will enforce certificate validity. This means that any homeserver without a
-valid certificate after this point will no longer be able to federate with
-1.0.0 servers.
-
-
-## Configuring certificates for compatibility with Synapse 1.0.0
-
-### If you do not currently have an SRV record
-
-In this case, your `server_name` points to the host where your Synapse is
-running. There is no need to create a `.well-known` URI or an SRV record, but
-you will need to give Synapse a valid, signed, certificate.
-
-The easiest way to do that is with Synapse's built-in ACME (Let's Encrypt)
-support. Full details are in [ACME.md](./ACME.md) but, in a nutshell:
-
- 1. Allow Synapse to listen on port 80 with `authbind`, or forward it from a
-    reverse proxy.
- 2. Enable acme support in `homeserver.yaml`.
- 3. Move your old certificates out of the way.
- 4. Restart Synapse.
-
-### If you do have an SRV record currently
-
-If you are using an SRV record, your matrix domain (`server_name`) may not
-point to the same host that your Synapse is running on (the 'target
-domain'). (If it does, you can follow the recommendation above; otherwise, read
-on.)
-
-Let's assume that your `server_name` is `example.com`, and your Synapse is
-hosted at a target domain of `customer.example.net`. Currently you should have
-an SRV record which looks like:
-
-```
-_matrix._tcp.example.com. IN SRV 10 5 8000 customer.example.net.
-```
-
-In this situation, you have three choices for how to proceed:
-
-#### Option 1: give Synapse a certificate for your matrix domain
-
-Synapse 1.0 will expect your server to present a TLS certificate for your
-`server_name` (`example.com` in the above example). You can achieve this by
-doing one of the following:
-
- * Acquire a certificate for the `server_name` yourself (for example, using
-   `certbot`), and give it and the key to Synapse via `tls_certificate_path`
-   and `tls_private_key_path`, or:
-
- * Use Synapse's [ACME support](./ACME.md), and forward port 80 on the
-   `server_name` domain to your Synapse instance.
-
-#### Option 2: run Synapse behind a reverse proxy
-
-If you have an existing reverse proxy set up with correct TLS certificates for
-your domain, you can simply route all traffic through the reverse proxy by
-updating the SRV record appropriately (or removing it, if the proxy listens on
-8448).
-
-See [reverse_proxy.rst](reverse_proxy.rst) for information on setting up a
-reverse proxy.
-
-#### Option 3: add a .well-known file to delegate your matrix traffic
-
-This will allow you to keep Synapse on a separate domain, without having to
-give it a certificate for the matrix domain.
-
-You can do this with a `.well-known` file as follows:
-
- 1. Keep the SRV record in place - it is needed for backwards compatibility
-    with Synapse 0.34 and earlier.
-
- 2. Give synapse a certificate corresponding to the target domain
-    (`customer.example.net` in the above example). Currently Synapse's ACME
-    support [does not support
-    this](https://github.com/matrix-org/synapse/issues/4552), so you will have
-    to acquire a certificate yourself and give it to Synapse via
-    `tls_certificate_path` and `tls_private_key_path`.
-
- 3. Restart Synapse to ensure the new certificate is loaded.
-
- 4. Arrange for a `.well-known` file at
-    `https://<server_name>/.well-known/matrix/server` with contents:
-
-    ```json
-    {"m.server": "<target server name>"}
-    ```
-
-    where the target server name is resolved as usual (i.e. SRV lookup, falling
-    back to talking to port 8448).
-
-    In the above example, where synapse is listening on port 8000,
-    `https://example.com/.well-known/matrix/server` should have `m.server` set to one of:
-
-    1. `customer.example.net` ─ with a SRV record on
-       `_matrix._tcp.customer.example.com` pointing to port 8000, or:
-
-    2. `customer.example.net` ─ updating synapse to listen on the default port
-       8448, or:
-
-    3. `customer.example.net:8000` ─ ensuring that if there is a reverse proxy
-       on `customer.example.net:8000` it correctly handles HTTP requests with
-       Host header set to `customer.example.net:8000`.
-
-
-## FAQ
-
-### Synapse 0.99.0 has just been released, what do I need to do right now?
-
-Upgrade as soon as you can in preparation for Synapse 1.0.0, and update your
-TLS certificates as [above](#configuring-certificates-for-compatibility-with-synapse-100).
-
-### What will happen if I do not set up a valid federation certificate immediately?
-
-Nothing initially, but once 1.0.0 is in the wild it will not be possible to
-federate with 1.0.0 servers.
-
-### What will happen if I do nothing at all?
-
-If the admin takes no action at all, and remains on a Synapse < 0.99.0 then the
-homeserver will be unable to federate with those who have implemented
-.well-known. Then, as above, once the month upgrade window has expired the
-homeserver will not be able to federate with any Synapse >= 1.0.0
-
-### When do I need a SRV record or .well-known URI?
-
-If your homeserver listens on the default federation port (8448), and your
-`server_name` points to the host that your homeserver runs on, you do not need an
-SRV record or `.well-known/matrix/server` URI.
-
-For instance, if you registered `example.com` and pointed its DNS A record at a
-fresh Upcloud VPS or similar, you could install Synapse 0.99 on that host,
-giving it a server_name of `example.com`, and it would automatically generate a
-valid TLS certificate for you via Let's Encrypt and no SRV record or
-`.well-known` URI would be needed.
-
-This is the common case, although you can add an SRV record or
-`.well-known/matrix/server` URI for completeness if you wish.
-
-**However**, if your server does not listen on port 8448, or if your `server_name`
-does not point to the host that your homeserver runs on, you will need to let
-other servers know how to find it.
-
-In this case, you should see ["If you do have an SRV record
-currently"](#if-you-do-have-an-srv-record-currently) above.
-
-### Can I still use an SRV record?
-
-Firstly, if you didn't need an SRV record before (because your server is
-listening on port 8448 of your server_name), you certainly don't need one now:
-the defaults are still the same.
-
-If you previously had an SRV record, you can keep using it provided you are
-able to give Synapse a TLS certificate corresponding to your server name. For
-example, suppose you had the following SRV record, which directs matrix traffic
-for example.com to matrix.example.com:443:
-
-```
-_matrix._tcp.example.com. IN SRV 10 5 443 matrix.example.com
-```
-
-In this case, Synapse must be given a certificate for example.com - or be
-configured to acquire one from Let's Encrypt.
-
-If you are unable to give Synapse a certificate for your server_name, you will
-also need to use a .well-known URI instead. However, see also "I have created a
-.well-known URI. Do I still need an SRV record?".
-
-### I have created a .well-known URI. Do I still need an SRV record?
-
-As of Synapse 0.99, Synapse will first check for the existence of a `.well-known`
-URI and follow any delegation it suggests. It will only then check for the
-existence of an SRV record.
-
-That means that the SRV record will often be redundant. However, you should
-remember that there may still be older versions of Synapse in the federation
-which do not understand `.well-known` URIs, so if you removed your SRV record you
-would no longer be able to federate with them.
-
-It is therefore best to leave the SRV record in place for now. Synapse 0.34 and
-earlier will follow the SRV record (and not care about the invalid
-certificate). Synapse 0.99 and later will follow the .well-known URI, with the
-correct certificate chain.
-
-### It used to work just fine, why are you breaking everything?
-
-We have always wanted Matrix servers to be as easy to set up as possible, and
-so back when we started federation in 2014 we didn't want admins to have to go
-through the cumbersome process of buying a valid TLS certificate to run a
-server. This was before Let's Encrypt came along and made getting a free and
-valid TLS certificate straightforward. So instead, we adopted a system based on
-[Perspectives](https://en.wikipedia.org/wiki/Convergence_(SSL)): an approach
-where you check a set of "notary servers" (in practice, homeservers) to vouch
-for the validity of a certificate rather than having it signed by a CA. As long
-as enough different notaries agree on the certificate's validity, then it is
-trusted.
-
-However, in practice this has never worked properly. Most people only use the
-default notary server (matrix.org), leading to inadvertent centralisation which
-we want to eliminate. Meanwhile, we never implemented the full consensus
-algorithm to query the servers participating in a room to determine consensus
-on whether a given certificate is valid. This is fiddly to get right
-(especially in face of sybil attacks), and we found ourselves questioning
-whether it was worth the effort to finish the work and commit to maintaining a
-secure certificate validation system as opposed to focusing on core Matrix
-development.
-
-Meanwhile, Let's Encrypt came along in 2016, and put the final nail in the
-coffin of the Perspectives project (which was already pretty dead). So, the
-Spec Core Team decided that a better approach would be to mandate valid TLS
-certificates for federation alongside the rest of the Web. More details can be
-found in
-[MSC1711](https://github.com/matrix-org/matrix-doc/blob/master/proposals/1711-x509-for-federation.md#background-the-failure-of-the-perspectives-approach).
-
-This results in a breaking change, which is disruptive, but absolutely critical
-for the security model. However, the existence of Let's Encrypt as a trivial
-way to replace the old self-signed certificates with valid CA-signed ones helps
-smooth things over massively, especially as Synapse can now automate Let's
-Encrypt certificate generation if needed.
-
-### Can I manage my own certificates rather than having Synapse renew certificates itself?
-
-Yes, you are welcome to manage your certificates yourself. Synapse will only
-attempt to obtain certificates from Let's Encrypt if you configure it to do
-so.The only requirement is that there is a valid TLS cert present for
-federation end points.
-
-### Do you still recommend against using a reverse proxy on the federation port?
-
-We no longer actively recommend against using a reverse proxy. Many admins will
-find it easier to direct federation traffic to a reverse proxy and manage their
-own TLS certificates, and this is a supported configuration.
-
-See [reverse_proxy.rst](reverse_proxy.rst) for information on setting up a
-reverse proxy.
-
-### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy?
-
-Practically speaking, this is no longer necessary.
-
-If you are using a reverse proxy for all of your TLS traffic, then you can set
-`no_tls: True`. In that case, the only reason Synapse needs the certificate is
-to populate a legacy 'tls_fingerprints' field in the federation API. This is
-ignored by Synapse 0.99.0 and later, and the only time pre-0.99 Synapses will
-check it is when attempting to fetch the server keys - and generally this is
-delegated via `matrix.org`, which is on 0.99.0.
-
-However, there is a bug in Synapse 0.99.0
-[4554](<https://github.com/matrix-org/synapse/issues/4554>) which prevents
-Synapse from starting if you do not give it a TLS certificate. To work around
-this, you can give it any TLS certificate at all. This will be fixed soon.
-
-### Do I need the same certificate for the client and federation port?
-
-No. There is nothing stopping you from using different certificates,
-particularly if you are using a reverse proxy. However, Synapse will use the
-same certificate on any ports where TLS is configured.
-
-### How do I tell Synapse to reload my keys/certificates after I replace them?
-
-Synapse will reload the keys and certificates when it receives a SIGHUP - for
-example `kill -HUP $(cat homeserver.pid)`. Alternatively, simply restart
-Synapse, though this will result in downtime while it restarts.

docs/reverse_proxy.rst

@@ -1,112 +0,0 @@
-Using a reverse proxy with Synapse
-==================================
-
-It is recommended to put a reverse proxy such as
-`nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
-`Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_,
-`Caddy <https://caddyserver.com/docs/proxy>`_ or
-`HAProxy <https://www.haproxy.org/>`_ in front of Synapse. One advantage of
-doing so is that it means that you can expose the default https port (443) to
-Matrix clients without needing to run Synapse with root privileges.
-
-**NOTE**: Your reverse proxy must not 'canonicalise' or 'normalise' the
-requested URI in any way (for example, by decoding ``%xx`` escapes). Beware
-that Apache *will* canonicalise URIs unless you specifify ``nocanon``.
-
-When setting up a reverse proxy, remember that Matrix clients and other Matrix
-servers do not necessarily need to connect to your server via the same server
-name or port. Indeed, clients will use port 443 by default, whereas servers
-default to port 8448. Where these are different, we refer to the 'client port'
-and the 'federation port'. See `Setting up federation
-<../README.rst#setting-up-federation>`_ for more details of the algorithm used for
-federation connections.
-
-Let's assume that we expect clients to connect to our server at
-``https://matrix.example.com``, and other servers to connect at
-``https://example.com:8448``. Here are some example configurations:
-
-* nginx::
-
-      server {
-          listen 443 ssl;
-          listen [::]:443 ssl;
-          server_name matrix.example.com;
-
-          location /_matrix {
-              proxy_pass http://localhost:8008;
-              proxy_set_header X-Forwarded-For $remote_addr;
-          }
-      }
-
-      server {
-          listen 8448 ssl default_server;
-          listen [::]:8448 ssl default_server;
-          server_name example.com;
-
-          location / {
-              proxy_pass http://localhost:8008;
-              proxy_set_header X-Forwarded-For $remote_addr;
-          }
-      }
-
-* Caddy::
-
-      matrix.example.com {
-        proxy /_matrix http://localhost:8008 {
-          transparent
-        }
-      }
-
-      example.com:8448 {
-        proxy / http://localhost:8008 {
-          transparent
-        }
-      }
-
-* Apache (note the ``nocanon`` options here!)::
-
-      <VirtualHost *:443>
-          SSLEngine on
-          ServerName matrix.example.com;
-
-          <Location /_matrix>
-              ProxyPass http://127.0.0.1:8008/_matrix nocanon
-              ProxyPassReverse http://127.0.0.1:8008/_matrix
-          </Location>
-      </VirtualHost>
-
-      <VirtualHost *:8448>
-          SSLEngine on
-          ServerName example.com;
-
-          <Location /_matrix>
-              ProxyPass http://127.0.0.1:8008/_matrix nocanon
-              ProxyPassReverse http://127.0.0.1:8008/_matrix
-          </Location>
-      </VirtualHost>
-
-* HAProxy::
-
-      frontend https
-        bind 0.0.0.0:443 v4v6 ssl crt /etc/ssl/haproxy/ strict-sni alpn h2,http/1.1
-        bind :::443 ssl crt /etc/ssl/haproxy/ strict-sni alpn h2,http/1.1
-        
-        # Matrix client traffic
-        acl matrix hdr(host) -i matrix.example.com
-        use_backend matrix if matrix
-        
-      frontend matrix-federation
-        bind 0.0.0.0:8448 v4v6 ssl crt /etc/ssl/haproxy/synapse.pem alpn h2,http/1.1
-        bind :::8448 ssl crt /etc/ssl/haproxy/synapse.pem alpn h2,http/1.1
-        default_backend matrix
-        
-      backend matrix
-        server matrix 127.0.0.1:8008
-
-You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true``
-for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are
-recorded correctly.
-
-Having done so, you can then use ``https://matrix.example.com`` (instead of
-``https://matrix.example.com:8448``) as the "Custom server" when connecting to
-Synapse from a client.

docs/tcp_replication.rst

@@ -137,6 +137,7 @@ for each stream so that on reconneciton it can start streaming from the correct
 place. Note: not all RDATA have valid tokens due to batching. See
 ``RdataCommand`` for more details.
 
+
 Example
 ~~~~~~~
 
@@ -220,28 +221,3 @@ SYNC (S, C)
 
 See ``synapse/replication/tcp/commands.py`` for a detailed description and the
 format of each command.
-
-
-Cache Invalidation Stream
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The cache invalidation stream is used to inform workers when they need to
-invalidate any of their caches in the data store. This is done by streaming all
-cache invalidations done on master down to the workers, assuming that any caches
-on the workers also exist on the master.
-
-Each individual cache invalidation results in a row being sent down replication,
-which includes the cache name (the name of the function) and they key to
-invalidate. For example::
-
-    > RDATA caches 550953771 ["get_user_by_id", ["@bob:example.com"], 1550574873251]
-
-However, there are times when a number of caches need to be invalidated at the
-same time with the same key. To reduce traffic we batch those invalidations into
-a single poke by defining a special cache name that workers understand to mean
-to expand to invalidate the correct caches.
-
-Currently the special cache names are declared in ``synapse/storage/_base.py``
-and are:
-
-1. ``cs_cache_fake`` ─ invalidates caches that depend on the current state

docs/workers.rst

@@ -26,8 +26,9 @@ Configuration
 To make effective use of the workers, you will need to configure an HTTP
 reverse-proxy such as nginx or haproxy, which will direct incoming requests to
 the correct worker, or to the main synapse instance. Note that this includes
-requests made to the federation port. See `<reverse_proxy.rst>`_ for
-information on setting up a reverse proxy.
+requests made to the federation port. The caveats regarding running a
+reverse-proxy on the federation port still apply (see
+https://github.com/matrix-org/synapse/blob/master/README.rst#reverse-proxying-the-federation-port).
 
 To enable workers, you need to add two replication listeners to the master
 synapse, e.g.::
@@ -222,13 +223,6 @@ following regular expressions::
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/context/.*$
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$
-    ^/_matrix/client/(api/v1|r0|unstable)/login$
-
-Additionally, the following REST endpoints can be handled, but all requests must
-be routed to the same instance::
-
-    ^/_matrix/client/(r0|unstable)/register$
-
 
 ``synapse.app.user_dir``
 ~~~~~~~~~~~~~~~~~~~~~~~~

scripts-dev/build_debian_packages

@@ -1,154 +0,0 @@
-#!/usr/bin/env python3
-
-# Build the Debian packages using Docker images.
-#
-# This script builds the Docker images and then executes them sequentially, each
-# one building a Debian package for the targeted operating system. It is
-# designed to be a "single command" to produce all the images.
-#
-# By default, builds for all known distributions, but a list of distributions
-# can be passed on the commandline for debugging.
-
-import argparse
-import os
-import signal
-import subprocess
-import sys
-import threading
-from concurrent.futures import ThreadPoolExecutor
-
-DISTS = (
-    "debian:stretch",
-    "debian:buster",
-    "debian:sid",
-    "ubuntu:xenial",
-    "ubuntu:bionic",
-    "ubuntu:cosmic",
-)
-
-DESC = '''\
-Builds .debs for synapse, using a Docker image for the build environment.
-
-By default, builds for all known distributions, but a list of distributions
-can be passed on the commandline for debugging.
-'''
-
-
-class Builder(object):
-    def __init__(self, redirect_stdout=False):
-        self.redirect_stdout = redirect_stdout
-        self.active_containers = set()
-        self._lock = threading.Lock()
-        self._failed = False
-
-    def run_build(self, dist):
-        """Build deb for a single distribution"""
-
-        if self._failed:
-            print("not building %s due to earlier failure" % (dist, ))
-            raise Exception("failed")
-
-        try:
-            self._inner_build(dist)
-        except Exception as e:
-            print("build of %s failed: %s" % (dist, e), file=sys.stderr)
-            self._failed = True
-            raise
-
-    def _inner_build(self, dist):
-        projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-        os.chdir(projdir)
-
-        tag = dist.split(":", 1)[1]
-
-        # Make the dir where the debs will live.
-        #
-        # Note that we deliberately put this outside the source tree, otherwise
-        # we tend to get source packages which are full of debs. (We could hack
-        # around that with more magic in the build_debian.sh script, but that
-        # doesn't solve the problem for natively-run dpkg-buildpakage).
-        debsdir = os.path.join(projdir, '../debs')
-        os.makedirs(debsdir, exist_ok=True)
-
-        if self.redirect_stdout:
-            logfile = os.path.join(debsdir, "%s.buildlog" % (tag, ))
-            print("building %s: directing output to %s" % (dist, logfile))
-            stdout = open(logfile, "w")
-        else:
-            stdout = None
-
-        # first build a docker image for the build environment
-        subprocess.check_call([
-            "docker", "build",
-            "--tag", "dh-venv-builder:" + tag,
-            "--build-arg", "distro=" + dist,
-            "-f", "docker/Dockerfile-dhvirtualenv",
-            "docker",
-        ], stdout=stdout, stderr=subprocess.STDOUT)
-
-        container_name = "synapse_build_" + tag
-        with self._lock:
-            self.active_containers.add(container_name)
-
-        # then run the build itself
-        subprocess.check_call([
-            "docker", "run",
-            "--rm",
-            "--name", container_name,
-            "--volume=" + projdir + ":/synapse/source:ro",
-            "--volume=" + debsdir + ":/debs",
-            "-e", "TARGET_USERID=%i" % (os.getuid(), ),
-            "-e", "TARGET_GROUPID=%i" % (os.getgid(), ),
-            "dh-venv-builder:" + tag,
-        ], stdout=stdout, stderr=subprocess.STDOUT)
-
-        with self._lock:
-            self.active_containers.remove(container_name)
-
-        if stdout is not None:
-            stdout.close()
-            print("Completed build of %s" % (dist, ))
-
-    def kill_containers(self):
-        with self._lock:
-            active = list(self.active_containers)
-
-        for c in active:
-            print("killing container %s" % (c,))
-            subprocess.run([
-                "docker", "kill", c,
-            ], stdout=subprocess.DEVNULL)
-            with self._lock:
-                self.active_containers.remove(c)
-
-
-def run_builds(dists, jobs=1):
-    builder = Builder(redirect_stdout=(jobs > 1))
-
-    def sig(signum, _frame):
-        print("Caught SIGINT")
-        builder.kill_containers()
-    signal.signal(signal.SIGINT, sig)
-
-    with ThreadPoolExecutor(max_workers=jobs) as e:
-        res = e.map(builder.run_build, dists)
-
-    # make sure we consume the iterable so that exceptions are raised.
-    for r in res:
-        pass
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(
-        description=DESC,
-    )
-    parser.add_argument(
-        '-j', '--jobs', type=int, default=1,
-        help='specify the number of builds to run in parallel',
-    )
-    parser.add_argument(
-        'dist', nargs='*', default=DISTS,
-        help='a list of distributions to build for. Default: %(default)s',
-    )
-    args = parser.parse_args()
-    run_builds(dists=args.dist, jobs=args.jobs)

scripts-dev/check-newsfragment

@@ -1,41 +0,0 @@
-#!/bin/bash
-#
-# A script which checks that an appropriate news file has been added on this
-# branch.
-
-set -e
-
-# make sure that origin/develop is up to date
-git remote set-branches --add origin develop
-git fetch --depth=1 origin develop
-
-UPSTREAM=origin/develop
-
-# if there are changes in the debian directory, check that the debian changelog
-# has been updated
-if ! git diff --quiet $UPSTREAM... -- debian; then
-    if git diff --quiet $UPSTREAM... -- debian/changelog; then
-        echo "Updates to debian directory, but no update to the changelog." >&2
-        exit 1
-    fi
-fi
-
-# if there are changes *outside* the debian directory, check that the
-# newsfragments have been updated.
-if git diff --name-only $UPSTREAM... | grep -qv '^develop/'; then
-    tox -e check-newsfragment
-fi
-
-echo
-echo "--------------------------"
-echo
-
-# check that any new newsfiles on this branch end with a full stop.
-for f in `git diff --name-only $UPSTREAM... -- changelog.d`; do
-    lastchar=`tr -d '\n' < $f | tail -c 1`
-    if [ $lastchar != '.' ]; then
-        echo -e "\e[31mERROR: newsfragment $f does not end with a '.'\e[39m" >&2
-        exit 1
-    fi
-done
-

scripts/synapse_port_db

@@ -53,7 +53,6 @@ BOOLEAN_COLUMNS = {
     "group_summary_users": ["is_public"],
     "group_roles": ["is_public"],
     "local_group_membership": ["is_publicised", "is_admin"],
-    "e2e_room_keys": ["is_verified"],
 }
 
 

synapse/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2018-9 New Vector Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,4 +27,4 @@ try:
 except ImportError:
     pass
 
-__version__ = "0.99.2"
+__version__ = "0.34.1.1"

synapse/_scripts/register_new_matrix_user.py

@@ -46,7 +46,7 @@ def request_registration(
     # Get the nonce
     r = requests.get(url, verify=False)
 
-    if r.status_code != 200:
+    if r.status_code is not 200:
         _print("ERROR! Received %d %s" % (r.status_code, r.reason))
         if 400 <= r.status_code < 500:
             try:
@@ -84,7 +84,7 @@ def request_registration(
     _print("Sending registration request...")
     r = requests.post(url, json=data, verify=False)
 
-    if r.status_code != 200:
+    if r.status_code is not 200:
         _print("ERROR! Received %d %s" % (r.status_code, r.reason))
         if 400 <= r.status_code < 500:
             try:

synapse/api/auth.py

@@ -65,7 +65,7 @@ class Auth(object):
         register_cache("cache", "token_cache", self.token_cache)
 
     @defer.inlineCallbacks
-    def check_from_context(self, room_version, event, context, do_sig_check=True):
+    def check_from_context(self, event, context, do_sig_check=True):
         prev_state_ids = yield context.get_prev_state_ids(self.store)
         auth_events_ids = yield self.compute_auth_events(
             event, prev_state_ids, for_verification=True,
@@ -74,16 +74,12 @@ class Auth(object):
         auth_events = {
             (e.type, e.state_key): e for e in itervalues(auth_events)
         }
-        self.check(
-            room_version, event,
-            auth_events=auth_events, do_sig_check=do_sig_check,
-        )
+        self.check(event, auth_events=auth_events, do_sig_check=do_sig_check)
 
-    def check(self, room_version, event, auth_events, do_sig_check=True):
+    def check(self, event, auth_events, do_sig_check=True):
         """ Checks if this event is correctly authed.
 
         Args:
-            room_version (str): version of the room
             event: the event being checked.
             auth_events (dict: event-key -> event): the existing room state.
 
@@ -92,9 +88,7 @@ class Auth(object):
             True if the auth checks pass.
         """
         with Measure(self.clock, "auth.check"):
-            event_auth.check(
-                room_version, event, auth_events, do_sig_check=do_sig_check
-            )
+            event_auth.check(event, auth_events, do_sig_check=do_sig_check)
 
     @defer.inlineCallbacks
     def check_joined_room(self, room_id, user_id, current_state=None):
@@ -550,6 +544,17 @@ class Auth(object):
         """
         return self.store.is_server_admin(user)
 
+    @defer.inlineCallbacks
+    def add_auth_events(self, builder, context):
+        prev_state_ids = yield context.get_prev_state_ids(self.store)
+        auth_ids = yield self.compute_auth_events(builder, prev_state_ids)
+
+        auth_events_entries = yield self.store.add_event_hashes(
+            auth_ids
+        )
+
+        builder.auth_events = auth_events_entries
+
     @defer.inlineCallbacks
     def compute_auth_events(self, event, current_state_ids, for_verification=False):
         if event.type == EventTypes.Create:
@@ -566,7 +571,7 @@ class Auth(object):
         key = (EventTypes.JoinRules, "", )
         join_rule_event_id = current_state_ids.get(key)
 
-        key = (EventTypes.Member, event.sender, )
+        key = (EventTypes.Member, event.user_id, )
         member_event_id = current_state_ids.get(key)
 
         key = (EventTypes.Create, "", )
@@ -616,7 +621,7 @@ class Auth(object):
 
         defer.returnValue(auth_ids)
 
-    def check_redaction(self, room_version, event, auth_events):
+    def check_redaction(self, event, auth_events):
         """Check whether the event sender is allowed to redact the target event.
 
         Returns:
@@ -629,7 +634,7 @@ class Auth(object):
             AuthError if the event sender is definitely not allowed to redact
             the target event.
         """
-        return event_auth.check_redaction(room_version, event, auth_events)
+        return event_auth.check_redaction(event, auth_events)
 
     @defer.inlineCallbacks
     def check_can_change_room_list(self, room_id, user):
@@ -814,9 +819,7 @@ class Auth(object):
             elif threepid:
                 # If the user does not exist yet, but is signing up with a
                 # reserved threepid then pass auth check
-                if is_threepid_reserved(
-                    self.hs.config.mau_limits_reserved_threepids, threepid
-                ):
+                if is_threepid_reserved(self.hs.config, threepid):
                     return
             # Else if there is no room in the MAU bucket, bail
             current_mau = yield self.store.get_monthly_active_count()

synapse/api/constants.py

@@ -68,12 +68,10 @@ class EventTypes(object):
     Aliases = "m.room.aliases"
     Redaction = "m.room.redaction"
     ThirdPartyInvite = "m.room.third_party_invite"
-    Encryption = "m.room.encryption"
 
     RoomHistoryVisibility = "m.room.history_visibility"
     CanonicalAlias = "m.room.canonical_alias"
     RoomAvatar = "m.room.avatar"
-    RoomEncryption = "m.room.encryption"
     GuestAccess = "m.room.guest_access"
 
     # These are used for validation
@@ -105,15 +103,10 @@ class ThirdPartyEntityKind(object):
 class RoomVersions(object):
     V1 = "1"
     V2 = "2"
-    V3 = "3"
+    VDH_TEST = "vdh-test-version"
     STATE_V2_TEST = "state-v2-test"
 
 
-class RoomDisposition(object):
-    STABLE = "stable"
-    UNSTABLE = "unstable"
-
-
 # the version we will give rooms which are created on this server
 DEFAULT_ROOM_VERSION = RoomVersions.V1
 
@@ -122,26 +115,10 @@ DEFAULT_ROOM_VERSION = RoomVersions.V1
 KNOWN_ROOM_VERSIONS = {
     RoomVersions.V1,
     RoomVersions.V2,
-    RoomVersions.V3,
+    RoomVersions.VDH_TEST,
     RoomVersions.STATE_V2_TEST,
-    RoomVersions.V3,
 }
 
-
-class EventFormatVersions(object):
-    """This is an internal enum for tracking the version of the event format,
-    independently from the room version.
-    """
-    V1 = 1
-    V2 = 2
-
-
-KNOWN_EVENT_FORMAT_VERSIONS = {
-    EventFormatVersions.V1,
-    EventFormatVersions.V2,
-}
-
-
 ServerNoticeMsgType = "m.server_notice"
 ServerNoticeLimitReached = "m.server_notice.usage_limit_reached"
 
@@ -151,4 +128,4 @@ class UserTypes(object):
     'admin' and 'guest' users should also be UserTypes. Normal users are type None
     """
     SUPPORT = "support"
-    ALL_USER_TYPES = (SUPPORT,)
+    ALL_USER_TYPES = (SUPPORT)

synapse/api/filtering.py

@@ -444,20 +444,6 @@ class Filter(object):
     def include_redundant_members(self):
         return self.filter_json.get("include_redundant_members", False)
 
-    def with_room_ids(self, room_ids):
-        """Returns a new filter with the given room IDs appended.
-
-        Args:
-            room_ids (iterable[unicode]): The room_ids to add
-
-        Returns:
-            filter: A new filter including the given rooms and the old
-                    filter's rooms.
-        """
-        newFilter = Filter(self.filter_json)
-        newFilter.rooms += room_ids
-        return newFilter
-
 
 def _matches_wildcard(actual_value, filter_value):
     if filter_value.endswith("*"):

synapse/api/urls.py

@@ -24,9 +24,7 @@ from synapse.config import ConfigError
 
 CLIENT_PREFIX = "/_matrix/client/api/v1"
 CLIENT_V2_ALPHA_PREFIX = "/_matrix/client/v2_alpha"
-FEDERATION_PREFIX = "/_matrix/federation"
-FEDERATION_V1_PREFIX = FEDERATION_PREFIX + "/v1"
-FEDERATION_V2_PREFIX = FEDERATION_PREFIX + "/v2"
+FEDERATION_PREFIX = "/_matrix/federation/v1"
 STATIC_PREFIX = "/_matrix/static"
 WEB_CLIENT_PREFIX = "/_matrix/client"
 CONTENT_REPO_PREFIX = "/_matrix/content"

synapse/app/__init__.py

@@ -12,38 +12,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import logging
+
 import sys
 
 from synapse import python_dependencies  # noqa: E402
 
 sys.dont_write_bytecode = True
 
-logger = logging.getLogger(__name__)
-
 try:
     python_dependencies.check_requirements()
 except python_dependencies.DependencyException as e:
     sys.stderr.writelines(e.message)
     sys.exit(1)
-
-
-def check_bind_error(e, address, bind_addresses):
-    """
-    This method checks an exception occurred while binding on 0.0.0.0.
-    If :: is specified in the bind addresses a warning is shown.
-    The exception is still raised otherwise.
-
-    Binding on both 0.0.0.0 and :: causes an exception on Linux and macOS
-    because :: binds on both IPv4 and IPv6 (as per RFC 3493).
-    When binding on 0.0.0.0 after :: this can safely be ignored.
-
-    Args:
-        e (Exception): Exception that was caught.
-        address (str): Address on which binding was attempted.
-        bind_addresses (list): Addresses on which the service listens.
-    """
-    if address == '0.0.0.0' and '::' in bind_addresses:
-        logger.warn('Failed to listen on 0.0.0.0, continuing because listening on [::]')
-    else:
-        raise e

synapse/app/_base.py

@@ -15,38 +15,18 @@
 
 import gc
 import logging
-import signal
 import sys
-import traceback
 
 import psutil
 from daemonize import Daemonize
 
 from twisted.internet import error, reactor
-from twisted.protocols.tls import TLSMemoryBIOFactory
 
-import synapse
-from synapse.app import check_bind_error
-from synapse.crypto import context_factory
 from synapse.util import PreserveLoggingContext
 from synapse.util.rlimit import change_resource_limit
-from synapse.util.versionstring import get_version_string
 
 logger = logging.getLogger(__name__)
 
-_sighup_callbacks = []
-
-
-def register_sighup(func):
-    """
-    Register a function to be called when a SIGHUP occurs.
-
-    Args:
-        func (function): Function to be called when sent a SIGHUP signal.
-            Will be called with a single argument, the homeserver.
-    """
-    _sighup_callbacks.append(func)
-
 
 def start_worker_reactor(appname, config):
     """ Run the reactor in the main process
@@ -155,154 +135,62 @@ def listen_metrics(bind_addresses, port):
     from prometheus_client import start_http_server
 
     for host in bind_addresses:
-        logger.info("Starting metrics listener on %s:%d", host, port)
-        start_http_server(port, addr=host, registry=RegistryProxy)
+        reactor.callInThread(start_http_server, int(port),
+                             addr=host, registry=RegistryProxy)
+        logger.info("Metrics now reporting on %s:%d", host, port)
 
 
 def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50):
     """
     Create a TCP socket for a port and several addresses
-
-    Returns:
-        list[twisted.internet.tcp.Port]: listening for TCP connections
     """
-    r = []
     for address in bind_addresses:
         try:
-            r.append(
-                reactor.listenTCP(
-                    port,
-                    factory,
-                    backlog,
-                    address
-                )
+            reactor.listenTCP(
+                port,
+                factory,
+                backlog,
+                address
             )
         except error.CannotListenError as e:
             check_bind_error(e, address, bind_addresses)
 
-    return r
-
 
 def listen_ssl(
     bind_addresses, port, factory, context_factory, reactor=reactor, backlog=50
 ):
     """
-    Create an TLS-over-TCP socket for a port and several addresses
-
-    Returns:
-        list of twisted.internet.tcp.Port listening for TLS connections
+    Create an SSL socket for a port and several addresses
     """
-    r = []
     for address in bind_addresses:
         try:
-            r.append(
-                reactor.listenSSL(
-                    port,
-                    factory,
-                    context_factory,
-                    backlog,
-                    address
-                )
+            reactor.listenSSL(
+                port,
+                factory,
+                context_factory,
+                backlog,
+                address
             )
         except error.CannotListenError as e:
             check_bind_error(e, address, bind_addresses)
 
-    return r
 
-
-def refresh_certificate(hs):
-    """
-    Refresh the TLS certificates that Synapse is using by re-reading them from
-    disk and updating the TLS context factories to use them.
+def check_bind_error(e, address, bind_addresses):
     """
+    This method checks an exception occurred while binding on 0.0.0.0.
+    If :: is specified in the bind addresses a warning is shown.
+    The exception is still raised otherwise.
 
-    if not hs.config.has_tls_listener():
-        # attempt to reload the certs for the good of the tls_fingerprints
-        hs.config.read_certificate_from_disk(require_cert_and_key=False)
-        return
-
-    hs.config.read_certificate_from_disk(require_cert_and_key=True)
-    hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config)
-
-    if hs._listening_services:
-        logger.info("Updating context factories...")
-        for i in hs._listening_services:
-            # When you listenSSL, it doesn't make an SSL port but a TCP one with
-            # a TLS wrapping factory around the factory you actually want to get
-            # requests. This factory attribute is public but missing from
-            # Twisted's documentation.
-            if isinstance(i.factory, TLSMemoryBIOFactory):
-                addr = i.getHost()
-                logger.info(
-                    "Replacing TLS context factory on [%s]:%i", addr.host, addr.port,
-                )
-                # We want to replace TLS factories with a new one, with the new
-                # TLS configuration. We do this by reaching in and pulling out
-                # the wrappedFactory, and then re-wrapping it.
-                i.factory = TLSMemoryBIOFactory(
-                    hs.tls_server_context_factory,
-                    False,
-                    i.factory.wrappedFactory
-                )
-        logger.info("Context factories updated.")
-
-
-def start(hs, listeners=None):
-    """
-    Start a Synapse server or worker.
+    Binding on both 0.0.0.0 and :: causes an exception on Linux and macOS
+    because :: binds on both IPv4 and IPv6 (as per RFC 3493).
+    When binding on 0.0.0.0 after :: this can safely be ignored.
 
     Args:
-        hs (synapse.server.HomeServer)
-        listeners (list[dict]): Listener configuration ('listeners' in homeserver.yaml)
+        e (Exception): Exception that was caught.
+        address (str): Address on which binding was attempted.
+        bind_addresses (list): Addresses on which the service listens.
     """
-    try:
-        # Set up the SIGHUP machinery.
-        if hasattr(signal, "SIGHUP"):
-            def handle_sighup(*args, **kwargs):
-                for i in _sighup_callbacks:
-                    i(hs)
-
-            signal.signal(signal.SIGHUP, handle_sighup)
-
-            register_sighup(refresh_certificate)
-
-        # Load the certificate from disk.
-        refresh_certificate(hs)
-
-        # It is now safe to start your Synapse.
-        hs.start_listening(listeners)
-        hs.get_datastore().start_profiling()
-
-        setup_sentry(hs)
-    except Exception:
-        traceback.print_exc(file=sys.stderr)
-        reactor = hs.get_reactor()
-        if reactor.running:
-            reactor.stop()
-        sys.exit(1)
-
-
-def setup_sentry(hs):
-    """Enable sentry integration, if enabled in configuration
-
-    Args:
-        hs (synapse.server.HomeServer)
-    """
-
-    if not hs.config.sentry_enabled:
-        return
-
-    import sentry_sdk
-    sentry_sdk.init(
-        dsn=hs.config.sentry_dsn,
-        release=get_version_string(synapse),
-    )
-
-    # We set some default tags that give some context to this instance
-    with sentry_sdk.configure_scope() as scope:
-        scope.set_tag("matrix_server_name", hs.config.server_name)
-
-        app = hs.config.worker_app if hs.config.worker_app else "synapse.app.homeserver"
-        name = hs.config.worker_name if hs.config.worker_name else "master"
-        scope.set_tag("worker_app", app)
-        scope.set_tag("worker_name", name)
+    if address == '0.0.0.0' and '::' in bind_addresses:
+        logger.warn('Failed to listen on 0.0.0.0, continuing because listening on [::]')
+    else:
+        raise e

synapse/app/appservice.py

@@ -168,7 +168,12 @@ def start(config_options):
     )
 
     ps.setup()
-    reactor.callWhenRunning(_base.start, ps, config.worker_listeners)
+    ps.start_listening(config.worker_listeners)
+
+    def start():
+        ps.get_datastore().start_profiling()
+
+    reactor.callWhenRunning(start)
 
     _base.start_worker_reactor("synapse-appservice", config)
 

synapse/app/client_reader.py

@@ -25,6 +25,7 @@ from synapse.app import _base
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
+from synapse.crypto import context_factory
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
 from synapse.metrics import RegistryProxy
@@ -40,7 +41,6 @@ from synapse.replication.slave.storage.registration import SlavedRegistrationSto
 from synapse.replication.slave.storage.room import RoomStore
 from synapse.replication.slave.storage.transactions import SlavedTransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
-from synapse.rest.client.v1.login import LoginRestServlet
 from synapse.rest.client.v1.room import (
     JoinedRoomMemberListRestServlet,
     PublicRoomListRestServlet,
@@ -48,7 +48,6 @@ from synapse.rest.client.v1.room import (
     RoomMemberListRestServlet,
     RoomStateRestServlet,
 )
-from synapse.rest.client.v2_alpha.register import RegisterRestServlet
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
 from synapse.util.httpresourcetree import create_resource_tree
@@ -94,8 +93,6 @@ class ClientReaderServer(HomeServer):
                     JoinedRoomMemberListRestServlet(self).register(resource)
                     RoomStateRestServlet(self).register(resource)
                     RoomEventContextServlet(self).register(resource)
-                    RegisterRestServlet(self).register(resource)
-                    LoginRestServlet(self).register(resource)
 
                     resources.update({
                         "/_matrix/client/r0": resource,
@@ -167,16 +164,26 @@ def start(config_options):
 
     database_engine = create_engine(config.database_config)
 
+    tls_server_context_factory = context_factory.ServerContextFactory(config)
+    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
+
     ss = ClientReaderServer(
         config.server_name,
         db_config=config.database_config,
+        tls_server_context_factory=tls_server_context_factory,
+        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
     )
 
     ss.setup()
-    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
+    ss.start_listening(config.worker_listeners)
+
+    def start():
+        ss.get_datastore().start_profiling()
+
+    reactor.callWhenRunning(start)
 
     _base.start_worker_reactor("synapse-client-reader", config)
 

synapse/app/event_creator.py

@@ -25,6 +25,7 @@ from synapse.app import _base
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
+from synapse.crypto import context_factory
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
 from synapse.metrics import RegistryProxy
@@ -184,16 +185,26 @@ def start(config_options):
 
     database_engine = create_engine(config.database_config)
 
+    tls_server_context_factory = context_factory.ServerContextFactory(config)
+    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
+
     ss = EventCreatorServer(
         config.server_name,
         db_config=config.database_config,
+        tls_server_context_factory=tls_server_context_factory,
+        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
     )
 
     ss.setup()
-    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
+    ss.start_listening(config.worker_listeners)
+
+    def start():
+        ss.get_datastore().start_profiling()
+
+    reactor.callWhenRunning(start)
 
     _base.start_worker_reactor("synapse-event-creator", config)
 

synapse/app/federation_reader.py

@@ -26,6 +26,7 @@ from synapse.app import _base
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
+from synapse.crypto import context_factory
 from synapse.federation.transport.server import TransportLayerServer
 from synapse.http.site import SynapseSite
 from synapse.metrics import RegistryProxy
@@ -40,7 +41,6 @@ from synapse.replication.slave.storage.profile import SlavedProfileStore
 from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
 from synapse.replication.slave.storage.pushers import SlavedPusherStore
 from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
-from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.replication.slave.storage.room import RoomStore
 from synapse.replication.slave.storage.transactions import SlavedTransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
@@ -63,7 +63,6 @@ class FederationReaderSlavedStore(
     SlavedReceiptsStore,
     SlavedEventStore,
     SlavedKeyStore,
-    SlavedRegistrationStore,
     RoomStore,
     DirectoryStore,
     SlavedTransactionStore,
@@ -88,16 +87,6 @@ class FederationReaderServer(HomeServer):
                     resources.update({
                         FEDERATION_PREFIX: TransportLayerServer(self),
                     })
-                if name == "openid" and "federation" not in res["names"]:
-                    # Only load the openid resource separately if federation resource
-                    # is not specified since federation resource includes openid
-                    # resource.
-                    resources.update({
-                        FEDERATION_PREFIX: TransportLayerServer(
-                            self,
-                            servlet_groups=["openid"],
-                        ),
-                    })
 
         root_resource = create_resource_tree(resources, NoResource())
 
@@ -110,8 +99,7 @@ class FederationReaderServer(HomeServer):
                 listener_config,
                 root_resource,
                 self.version_string,
-            ),
-            reactor=self.get_reactor()
+            )
         )
 
         logger.info("Synapse federation reader now listening on port %d", port)
@@ -163,16 +151,26 @@ def start(config_options):
 
     database_engine = create_engine(config.database_config)
 
+    tls_server_context_factory = context_factory.ServerContextFactory(config)
+    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
+
     ss = FederationReaderServer(
         config.server_name,
         db_config=config.database_config,
+        tls_server_context_factory=tls_server_context_factory,
+        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
     )
 
     ss.setup()
-    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
+    ss.start_listening(config.worker_listeners)
+
+    def start():
+        ss.get_datastore().start_profiling()
+
+    reactor.callWhenRunning(start)
 
     _base.start_worker_reactor("synapse-federation-reader", config)
 

synapse/app/federation_sender.py

@@ -25,6 +25,7 @@ from synapse.app import _base
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
+from synapse.crypto import context_factory
 from synapse.federation import send_queue
 from synapse.http.site import SynapseSite
 from synapse.metrics import RegistryProxy
@@ -182,17 +183,26 @@ def start(config_options):
     # Force the pushers to start since they will be disabled in the main config
     config.send_federation = True
 
-    ss = FederationSenderServer(
+    tls_server_context_factory = context_factory.ServerContextFactory(config)
+    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
+
+    ps = FederationSenderServer(
         config.server_name,
         db_config=config.database_config,
+        tls_server_context_factory=tls_server_context_factory,
+        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
     )
 
-    ss.setup()
-    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
+    ps.setup()
+    ps.start_listening(config.worker_listeners)
 
+    def start():
+        ps.get_datastore().start_profiling()
+
+    reactor.callWhenRunning(start)
     _base.start_worker_reactor("synapse-federation-sender", config)
 
 

synapse/app/frontend_proxy.py

@@ -21,11 +21,12 @@ from twisted.web.resource import NoResource
 
 import synapse
 from synapse import events
-from synapse.api.errors import HttpResponseException, SynapseError
+from synapse.api.errors import SynapseError
 from synapse.app import _base
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
+from synapse.crypto import context_factory
 from synapse.http.server import JsonResource
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
 from synapse.http.site import SynapseSite
@@ -66,15 +67,10 @@ class PresenceStatusStubServlet(ClientV1RestServlet):
         headers = {
             "Authorization": auth_headers,
         }
-
-        try:
-            result = yield self.http_client.get_json(
-                self.main_uri + request.uri.decode('ascii'),
-                headers=headers,
-            )
-        except HttpResponseException as e:
-            raise e.to_synapse_error()
-
+        result = yield self.http_client.get_json(
+            self.main_uri + request.uri.decode('ascii'),
+            headers=headers,
+        )
         defer.returnValue((200, result))
 
     @defer.inlineCallbacks
@@ -245,16 +241,26 @@ def start(config_options):
 
     database_engine = create_engine(config.database_config)
 
+    tls_server_context_factory = context_factory.ServerContextFactory(config)
+    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
+
     ss = FrontendProxyServer(
         config.server_name,
         db_config=config.database_config,
+        tls_server_context_factory=tls_server_context_factory,
+        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
     )
 
     ss.setup()
-    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
+    ss.start_listening(config.worker_listeners)
+
+    def start():
+        ss.get_datastore().start_profiling()
+
+    reactor.callWhenRunning(start)
 
     _base.start_worker_reactor("synapse-frontend-proxy", config)
 

synapse/app/homeserver.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,9 +13,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from __future__ import print_function
-
 import gc
 import logging
 import os
@@ -29,7 +25,6 @@ from prometheus_client import Gauge
 
 from twisted.application import service
 from twisted.internet import defer, reactor
-from twisted.python.failure import Failure
 from twisted.web.resource import EncodingResourceWrapper, NoResource
 from twisted.web.server import GzipEncoderFactory
 from twisted.web.static import File
@@ -50,6 +45,7 @@ from synapse.app import _base
 from synapse.app._base import listen_ssl, listen_tcp, quit_with_error
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
+from synapse.crypto import context_factory
 from synapse.federation.transport.server import TransportLayerServer
 from synapse.http.additional_resource import AdditionalResource
 from synapse.http.server import RootRedirect
@@ -93,13 +89,12 @@ class SynapseHomeServer(HomeServer):
         tls = listener_config.get("tls", False)
         site_tag = listener_config.get("tag", port)
 
+        if tls and config.no_tls:
+            return
+
         resources = {}
         for res in listener_config["resources"]:
             for name in res["names"]:
-                if name == "openid" and "federation" in res["names"]:
-                    # Skip loading openid resource if federation is defined
-                    # since federation resource will include openid
-                    continue
                 resources.update(self._configure_named_resource(
                     name, res.get("compress", False),
                 ))
@@ -124,7 +119,7 @@ class SynapseHomeServer(HomeServer):
         root_resource = create_resource_tree(resources, root_resource)
 
         if tls:
-            ports = listen_ssl(
+            listen_ssl(
                 bind_addresses,
                 port,
                 SynapseSite(
@@ -135,12 +130,10 @@ class SynapseHomeServer(HomeServer):
                     self.version_string,
                 ),
                 self.tls_server_context_factory,
-                reactor=self.get_reactor(),
             )
-            logger.info("Synapse now listening on TCP port %d (TLS)", port)
 
         else:
-            ports = listen_tcp(
+            listen_tcp(
                 bind_addresses,
                 port,
                 SynapseSite(
@@ -149,12 +142,9 @@ class SynapseHomeServer(HomeServer):
                     listener_config,
                     root_resource,
                     self.version_string,
-                ),
-                reactor=self.get_reactor(),
+                )
             )
-            logger.info("Synapse now listening on TCP port %d", port)
-
-        return ports
+        logger.info("Synapse now listening on port %d", port)
 
     def _configure_named_resource(self, name, compress=False):
         """Build a resource map for a named resource
@@ -200,11 +190,6 @@ class SynapseHomeServer(HomeServer):
                 FEDERATION_PREFIX: TransportLayerServer(self),
             })
 
-        if name == "openid":
-            resources.update({
-                FEDERATION_PREFIX: TransportLayerServer(self, servlet_groups=["openid"]),
-            })
-
         if name in ["static", "client"]:
             resources.update({
                 STATIC_PREFIX: File(
@@ -250,14 +235,12 @@ class SynapseHomeServer(HomeServer):
 
         return resources
 
-    def start_listening(self, listeners):
+    def start_listening(self):
         config = self.get_config()
 
-        for listener in listeners:
+        for listener in config.listeners:
             if listener["type"] == "http":
-                self._listening_services.extend(
-                    self._listener_http(config, listener)
-                )
+                self._listener_http(config, listener)
             elif listener["type"] == "manhole":
                 listen_tcp(
                     listener["bind_addresses"],
@@ -269,14 +252,14 @@ class SynapseHomeServer(HomeServer):
                     )
                 )
             elif listener["type"] == "replication":
-                services = listen_tcp(
-                    listener["bind_addresses"],
-                    listener["port"],
-                    ReplicationStreamProtocolFactory(self),
-                )
-                for s in services:
+                bind_addresses = listener["bind_addresses"]
+                for address in bind_addresses:
+                    factory = ReplicationStreamProtocolFactory(self)
+                    server_listener = reactor.listenTCP(
+                        listener["port"], factory, interface=address
+                    )
                     reactor.addSystemEventTrigger(
-                        "before", "shutdown", s.stopListening,
+                        "before", "shutdown", server_listener.stopListening,
                     )
             elif listener["type"] == "metrics":
                 if not self.get_config().enable_metrics:
@@ -337,19 +320,21 @@ def setup(config_options):
         # generating config files and shouldn't try to continue.
         sys.exit(0)
 
-    synapse.config.logger.setup_logging(
-        config,
-        use_worker_options=False
-    )
+    synapse.config.logger.setup_logging(config, use_worker_options=False)
 
     events.USE_FROZEN_DICTS = config.use_frozen_dicts
 
+    tls_server_context_factory = context_factory.ServerContextFactory(config)
+    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
+
     database_engine = create_engine(config.database_config)
     config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection
 
     hs = SynapseHomeServer(
         config.server_name,
         db_config=config.database_config,
+        tls_server_context_factory=tls_server_context_factory,
+        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
@@ -376,79 +361,12 @@ def setup(config_options):
     logger.info("Database prepared in %s.", config.database_config['name'])
 
     hs.setup()
+    hs.start_listening()
 
-    @defer.inlineCallbacks
-    def do_acme():
-        """
-        Reprovision an ACME certificate, if it's required.
-
-        Returns:
-            Deferred[bool]: Whether the cert has been updated.
-        """
-        acme = hs.get_acme_handler()
-
-        # Check how long the certificate is active for.
-        cert_days_remaining = hs.config.is_disk_cert_valid(
-            allow_self_signed=False
-        )
-
-        # We want to reprovision if cert_days_remaining is None (meaning no
-        # certificate exists), or the days remaining number it returns
-        # is less than our re-registration threshold.
-        provision = False
-
-        if (
-            cert_days_remaining is None or
-            cert_days_remaining < hs.config.acme_reprovision_threshold
-        ):
-            provision = True
-
-        if provision:
-            yield acme.provision_certificate()
-
-        defer.returnValue(provision)
-
-    @defer.inlineCallbacks
-    def reprovision_acme():
-        """
-        Provision a certificate from ACME, if required, and reload the TLS
-        certificate if it's renewed.
-        """
-        reprovisioned = yield do_acme()
-        if reprovisioned:
-            _base.refresh_certificate(hs)
-
-    @defer.inlineCallbacks
     def start():
-        try:
-            # Run the ACME provisioning code, if it's enabled.
-            if hs.config.acme_enabled:
-                acme = hs.get_acme_handler()
-                # Start up the webservices which we will respond to ACME
-                # challenges with, and then provision.
-                yield acme.start_listening()
-                yield do_acme()
-
-                # Check if it needs to be reprovisioned every day.
-                hs.get_clock().looping_call(
-                    reprovision_acme,
-                    24 * 60 * 60 * 1000
-                )
-
-            _base.start(hs, config.listeners)
-
-            hs.get_pusherpool().start()
-            hs.get_datastore().start_doing_background_updates()
-        except Exception:
-            # Print the exception and bail out.
-            print("Error during startup:", file=sys.stderr)
-
-            # this gives better tracebacks than traceback.print_exc()
-            Failure().printTraceback(file=sys.stderr)
-
-            if reactor.running:
-                reactor.stop()
-            sys.exit(1)
+        hs.get_pusherpool().start()
+        hs.get_datastore().start_profiling()
+        hs.get_datastore().start_doing_background_updates()
 
     reactor.callWhenRunning(start)
 
@@ -456,8 +374,7 @@ def setup(config_options):
 
 
 class SynapseService(service.Service):
-    """
-    A twisted Service class that will start synapse. Used to run synapse
+    """A twisted Service class that will start synapse. Used to run synapse
     via twistd and a .tac.
     """
     def __init__(self, config):
@@ -555,9 +472,6 @@ def run(hs):
                 stats["memory_rss"] += process.memory_info().rss
                 stats["cpu_average"] += int(process.cpu_percent(interval=None))
 
-        stats["database_engine"] = hs.get_datastore().database_engine_name
-        stats["database_server_version"] = hs.get_datastore().get_server_version()
-
         logger.info("Reporting stats to matrix.org: %s" % (stats,))
         try:
             yield hs.get_simple_http_client().put_json(

synapse/app/media_repository.py

@@ -26,6 +26,7 @@ from synapse.app import _base
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
+from synapse.crypto import context_factory
 from synapse.http.site import SynapseSite
 from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
@@ -150,16 +151,26 @@ def start(config_options):
 
     database_engine = create_engine(config.database_config)
 
+    tls_server_context_factory = context_factory.ServerContextFactory(config)
+    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
+
     ss = MediaRepositoryServer(
         config.server_name,
         db_config=config.database_config,
+        tls_server_context_factory=tls_server_context_factory,
+        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
     )
 
     ss.setup()
-    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
+    ss.start_listening(config.worker_listeners)
+
+    def start():
+        ss.get_datastore().start_profiling()
+
+    reactor.callWhenRunning(start)
 
     _base.start_worker_reactor("synapse-media-repository", config)
 

synapse/app/pusher.py

@@ -224,10 +224,11 @@ def start(config_options):
     )
 
     ps.setup()
+    ps.start_listening(config.worker_listeners)
 
     def start():
-        _base.start(ps, config.worker_listeners)
         ps.get_pusherpool().start()
+        ps.get_datastore().start_profiling()
 
     reactor.callWhenRunning(start)
 

synapse/app/synchrotron.py

@@ -445,7 +445,12 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
+    ss.start_listening(config.worker_listeners)
+
+    def start():
+        ss.get_datastore().start_profiling()
+
+    reactor.callWhenRunning(start)
 
     _base.start_worker_reactor("synapse-synchrotron", config)
 

synapse/app/user_dir.py

@@ -26,6 +26,7 @@ from synapse.app import _base
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
+from synapse.crypto import context_factory
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
 from synapse.metrics import RegistryProxy
@@ -210,16 +211,26 @@ def start(config_options):
     # Force the pushers to start since they will be disabled in the main config
     config.update_user_directory = True
 
-    ss = UserDirectoryServer(
+    tls_server_context_factory = context_factory.ServerContextFactory(config)
+    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
+
+    ps = UserDirectoryServer(
         config.server_name,
         db_config=config.database_config,
+        tls_server_context_factory=tls_server_context_factory,
+        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
     )
 
-    ss.setup()
-    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
+    ps.setup()
+    ps.start_listening(config.worker_listeners)
+
+    def start():
+        ps.get_datastore().start_profiling()
+
+    reactor.callWhenRunning(start)
 
     _base.start_worker_reactor("synapse-user-dir", config)
 

synapse/config/_base.py

@@ -257,7 +257,7 @@ class Config(object):
             "--keys-directory",
             metavar="DIRECTORY",
             help="Used with 'generate-*' options to specify where files such as"
-            " signing keys should be stored, unless explicitly"
+            " certs and signing keys should be stored in, unless explicitly"
             " specified in the config.",
         )
         config_parser.add_argument(
@@ -313,11 +313,16 @@ class Config(object):
                 print(
                     (
                         "A config file has been generated in %r for server name"
-                        " %r. Please review this file and customise it"
+                        " %r with corresponding SSL keys and self-signed"
+                        " certificates. Please review this file and customise it"
                         " to your needs."
                     )
                     % (config_path, server_name)
                 )
+                print(
+                    "If this server name is incorrect, you will need to"
+                    " regenerate the SSL certificates"
+                )
                 return
             else:
                 print(
@@ -362,7 +367,7 @@ class Config(object):
         if not keys_directory:
             keys_directory = os.path.dirname(config_files[-1])
 
-        self.config_dir_path = os.path.abspath(keys_directory)
+        config_dir_path = os.path.abspath(keys_directory)
 
         specified_config = {}
         for config_file in config_files:
@@ -374,7 +379,7 @@ class Config(object):
 
         server_name = specified_config["server_name"]
         config_string = self.generate_config(
-            config_dir_path=self.config_dir_path,
+            config_dir_path=config_dir_path,
             data_dir_path=os.getcwd(),
             server_name=server_name,
             generate_secrets=False,

synapse/config/api.py

@@ -24,7 +24,6 @@ class ApiConfig(Config):
             EventTypes.JoinRules,
             EventTypes.CanonicalAlias,
             EventTypes.RoomAvatar,
-            EventTypes.RoomEncryption,
             EventTypes.Name,
         ])
 
@@ -33,11 +32,9 @@ class ApiConfig(Config):
         ## API Configuration ##
 
         # A list of event types that will be included in the room_invite_state
-        #
         room_invite_state_types:
             - "{JoinRules}"
             - "{CanonicalAlias}"
             - "{RoomAvatar}"
-            - "{RoomEncryption}"
             - "{Name}"
         """.format(**vars(EventTypes))

synapse/config/appservice.py

@@ -38,12 +38,10 @@ class AppServiceConfig(Config):
     def default_config(cls, **kwargs):
         return """\
         # A list of application service config file to use
-        #
         app_service_config_files: []
 
         # Whether or not to track application service IP addresses. Implicitly
         # enables MAU tracking for application service users.
-        #
         track_appservice_user_ips: False
         """
 

synapse/config/captcha.py

@@ -30,22 +30,19 @@ class CaptchaConfig(Config):
         # See docs/CAPTCHA_SETUP for full details of configuring this.
 
         # This Home Server's ReCAPTCHA public key.
-        #
         recaptcha_public_key: "YOUR_PUBLIC_KEY"
 
         # This Home Server's ReCAPTCHA private key.
-        #
         recaptcha_private_key: "YOUR_PRIVATE_KEY"
 
         # Enables ReCaptcha checks when registering, preventing signup
         # unless a captcha is answered. Requires a valid ReCaptcha
         # public/private key.
-        #
         enable_registration_captcha: False
 
         # A secret key used to bypass the captcha test entirely.
         #captcha_bypass_secret: "YOUR_SECRET_HERE"
 
         # The API endpoint to use for verifying m.login.recaptcha responses.
-        recaptcha_siteverify_api: "https://www.recaptcha.net/recaptcha/api/siteverify"
+        recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify"
         """

synapse/config/cas.py

@@ -38,7 +38,6 @@ class CasConfig(Config):
     def default_config(self, config_dir_path, server_name, **kwargs):
         return """
         # Enable CAS for registration and login.
-        #
         #cas_config:
         #   enabled: true
         #   server_url: "https://cas-server.com"

synapse/config/consent_config.py

@@ -13,10 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from os import path
-
-from synapse.config import ConfigError
-
 from ._base import Config
 
 DEFAULT_CONFIG = """\
@@ -54,20 +50,20 @@ DEFAULT_CONFIG = """\
 # for an account. Has no effect unless `require_at_registration` is enabled.
 # Defaults to "Privacy Policy".
 #
-#user_consent:
-#  template_dir: res/templates/privacy
-#  version: 1.0
-#  server_notice_content:
-#    msgtype: m.text
-#    body: >-
-#      To continue using this homeserver you must review and agree to the
-#      terms and conditions at %(consent_uri)s
-#  send_server_notice_to_guests: True
-#  block_events_error: >-
-#    To continue using this homeserver you must review and agree to the
-#    terms and conditions at %(consent_uri)s
-#  require_at_registration: False
-#  policy_name: Privacy Policy
+# user_consent:
+#   template_dir: res/templates/privacy
+#   version: 1.0
+#   server_notice_content:
+#     msgtype: m.text
+#     body: >-
+#       To continue using this homeserver you must review and agree to the
+#       terms and conditions at %(consent_uri)s
+#   send_server_notice_to_guests: True
+#   block_events_error: >-
+#     To continue using this homeserver you must review and agree to the
+#     terms and conditions at %(consent_uri)s
+#   require_at_registration: False
+#   policy_name: Privacy Policy
 #
 """
 
@@ -89,15 +85,7 @@ class ConsentConfig(Config):
         if consent_config is None:
             return
         self.user_consent_version = str(consent_config["version"])
-        self.user_consent_template_dir = self.abspath(
-            consent_config["template_dir"]
-        )
-        if not path.isdir(self.user_consent_template_dir):
-            raise ConfigError(
-                "Could not find template directory '%s'" % (
-                    self.user_consent_template_dir,
-                ),
-            )
+        self.user_consent_template_dir = consent_config["template_dir"]
         self.user_consent_server_notice_content = consent_config.get(
             "server_notice_content",
         )

synapse/config/groups.py

@@ -24,11 +24,9 @@ class GroupsConfig(Config):
     def default_config(self, **kwargs):
         return """\
         # Whether to allow non server admins to create groups on this server
-        #
         enable_group_creation: false
 
         # If enabled, non server admins can only create groups with local parts
         # starting with this prefix
-        #
-        #group_creation_prefix: "unofficial/"
+        # group_creation_prefix: "unofficial/"
         """

synapse/config/homeserver.py

@@ -42,7 +42,7 @@ from .voip import VoipConfig
 from .workers import WorkerConfig
 
 
-class HomeServerConfig(ServerConfig, TlsConfig, DatabaseConfig, LoggingConfig,
+class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig,
                        RatelimitConfig, ContentRepositoryConfig, CaptchaConfig,
                        VoipConfig, RegistrationConfig, MetricsConfig, ApiConfig,
                        AppServiceConfig, KeyConfig, SAML2Config, CasConfig,

synapse/config/jwt_config.py

@@ -46,8 +46,8 @@ class JWTConfig(Config):
         return """\
         # The JWT needs to contain a globally unique "sub" (subject) claim.
         #
-        #jwt_config:
-        #   enabled: true
-        #   secret: "a secret"
-        #   algorithm: "HS256"
+        # jwt_config:
+        #    enabled: true
+        #    secret: "a secret"
+        #    algorithm: "HS256"
         """

synapse/config/key.py

@@ -40,7 +40,7 @@ class KeyConfig(Config):
     def read_config(self, config):
         self.signing_key = self.read_signing_key(config["signing_key_path"])
         self.old_signing_keys = self.read_old_signing_keys(
-            config.get("old_signing_keys", {})
+            config["old_signing_keys"]
         )
         self.key_refresh_interval = self.parse_duration(
             config["key_refresh_interval"]
@@ -56,7 +56,7 @@ class KeyConfig(Config):
         if not self.macaroon_secret_key:
             # Unfortunately, there are people out there that don't have this
             # set. Lets just be "nice" and derive one from their secret key.
-            logger.warn("Config is missing macaroon_secret_key")
+            logger.warn("Config is missing missing macaroon_secret_key")
             seed = bytes(self.signing_key[0])
             self.macaroon_secret_key = hashlib.sha256(seed).digest()
 
@@ -84,28 +84,26 @@ class KeyConfig(Config):
         # the registration_shared_secret is used, if one is given; otherwise,
         # a secret key is derived from the signing key.
         #
+        # Note that changing this will invalidate any active access tokens, so
+        # all clients will have to log back in.
         %(macaroon_secret_key)s
 
         # Used to enable access token expiration.
-        #
         expire_access_token: False
 
         # a secret which is used to calculate HMACs for form values, to stop
         # falsification of values. Must be specified for the User Consent
         # forms to work.
-        #
         %(form_secret)s
 
         ## Signing Keys ##
 
         # Path to the signing key to sign messages with
-        #
         signing_key_path: "%(base_key_name)s.signing.key"
 
         # The keys that the server used to sign messages with but won't use
         # to sign new messages. E.g. it has lost its private key
-        #
-        #old_signing_keys:
+        old_signing_keys: {}
         #  "ed25519:auto":
         #    # Base64 encoded public key
         #    key: "The public part of your old signing key."
@@ -116,11 +114,9 @@ class KeyConfig(Config):
         # Used to set the valid_until_ts in /key/v2 APIs.
         # Determines how quickly servers will query to check which keys
         # are still valid.
-        #
         key_refresh_interval: "1d" # 1 Day.
 
         # The trusted servers to download signing keys from.
-        #
         perspectives:
           servers:
             "matrix.org":

synapse/config/logger.py

@@ -15,6 +15,7 @@
 import logging
 import logging.config
 import os
+import signal
 import sys
 from string import Template
 
@@ -23,7 +24,6 @@ import yaml
 from twisted.logger import STDLibLogObserver, globalLogBeginner
 
 import synapse
-from synapse.app import _base as appbase
 from synapse.util.logcontext import LoggingContextFilter
 from synapse.util.versionstring import get_version_string
 
@@ -83,7 +83,6 @@ class LoggingConfig(Config):
         log_config = os.path.join(config_dir_path, server_name + ".log.config")
         return """
         # A yaml python logging config file
-        #
         log_config: "%(log_config)s"
         """ % locals()
 
@@ -137,9 +136,6 @@ def setup_logging(config, use_worker_options=False):
 
         use_worker_options (bool): True to use 'worker_log_config' and
             'worker_log_file' options instead of 'log_config' and 'log_file'.
-
-        register_sighup (func | None): Function to call to register a
-            sighup handler.
     """
     log_config = (config.worker_log_config if use_worker_options
                   else config.log_config)
@@ -182,7 +178,7 @@ def setup_logging(config, use_worker_options=False):
         else:
             handler = logging.StreamHandler()
 
-            def sighup(*args):
+            def sighup(signum, stack):
                 pass
 
         handler.setFormatter(formatter)
@@ -195,14 +191,20 @@ def setup_logging(config, use_worker_options=False):
             with open(log_config, 'r') as f:
                 logging.config.dictConfig(yaml.load(f))
 
-        def sighup(*args):
+        def sighup(signum, stack):
             # it might be better to use a file watcher or something for this.
             load_log_config()
             logging.info("Reloaded log config from %s due to SIGHUP", log_config)
 
         load_log_config()
 
-    appbase.register_sighup(sighup)
+    # TODO(paul): obviously this is a terrible mechanism for
+    #   stealing SIGHUP, because it means no other part of synapse
+    #   can use it instead. If we want to catch SIGHUP anywhere
+    #   else as well, I'd suggest we find a nicer way to broadcast
+    #   it around.
+    if getattr(signal, "SIGHUP"):
+        signal.signal(signal.SIGHUP, sighup)
 
     # make sure that the first thing we log is a thing we can grep backwards
     # for
@@ -243,5 +245,3 @@ def setup_logging(config, use_worker_options=False):
         [_log],
         redirectStandardIO=not config.no_redirect_stdio,
     )
-    if not config.no_redirect_stdio:
-        print("Redirected stdout/stderr to logs")

synapse/config/metrics.py

@@ -13,13 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._base import Config, ConfigError
-
-MISSING_SENTRY = (
-    """Missing sentry-sdk library. This is required to enable sentry
-    integration.
-    """
-)
+from ._base import Config
 
 
 class MetricsConfig(Config):
@@ -29,38 +23,12 @@ class MetricsConfig(Config):
         self.metrics_port = config.get("metrics_port")
         self.metrics_bind_host = config.get("metrics_bind_host", "127.0.0.1")
 
-        self.sentry_enabled = "sentry" in config
-        if self.sentry_enabled:
-            try:
-                import sentry_sdk  # noqa F401
-            except ImportError:
-                raise ConfigError(MISSING_SENTRY)
-
-            self.sentry_dsn = config["sentry"].get("dsn")
-            if not self.sentry_dsn:
-                raise ConfigError(
-                    "sentry.dsn field is required when sentry integration is enabled",
-                )
-
     def default_config(self, report_stats=None, **kwargs):
         res = """\
         ## Metrics ###
 
         # Enable collection and rendering of performance metrics
-        #
         enable_metrics: False
-
-        # Enable sentry integration
-        # NOTE: While attempts are made to ensure that the logs don't contain
-        # any sensitive information, this cannot be guaranteed. By enabling
-        # this option the sentry server may therefore receive sensitive
-        # information, and it in turn may then diseminate sensitive information
-        # through insecure notification channels if so configured.
-        #
-        #sentry:
-        #    dsn: "..."
-
-        # Whether or not to report anonymized homeserver usage statistics.
         """
 
         if report_stats is None:

synapse/config/password.py

@@ -28,7 +28,6 @@ class PasswordConfig(Config):
     def default_config(self, config_dir_path, server_name, **kwargs):
         return """
         # Enable password for login.
-        #
         password_config:
            enabled: true
            # Uncomment and change to a secret random string for extra security.

synapse/config/password_auth_providers.py

@@ -52,18 +52,18 @@ class PasswordAuthProviderConfig(Config):
 
     def default_config(self, **kwargs):
         return """\
-        #password_providers:
-        #    - module: "ldap_auth_provider.LdapAuthProvider"
-        #      config:
-        #        enabled: true
-        #        uri: "ldap://ldap.example.com:389"
-        #        start_tls: true
-        #        base: "ou=users,dc=example,dc=com"
-        #        attributes:
-        #           uid: "cn"
-        #           mail: "email"
-        #           name: "givenName"
-        #        #bind_dn:
-        #        #bind_password:
-        #        #filter: "(objectClass=posixAccount)"
+        # password_providers:
+        #     - module: "ldap_auth_provider.LdapAuthProvider"
+        #       config:
+        #         enabled: true
+        #         uri: "ldap://ldap.example.com:389"
+        #         start_tls: true
+        #         base: "ou=users,dc=example,dc=com"
+        #         attributes:
+        #            uid: "cn"
+        #            mail: "email"
+        #            name: "givenName"
+        #         #bind_dn:
+        #         #bind_password:
+        #         #filter: "(objectClass=posixAccount)"
         """

synapse/config/push.py

@@ -51,11 +51,11 @@ class PushConfig(Config):
         # notification request includes the content of the event (other details
         # like the sender are still included). For `event_id_only` push, it
         # has no effect.
-        #
+
         # For modern android devices the notification content will still appear
         # because it is loaded by the app. iPhone, however will send a
         # notification saying only that a message arrived and who it came from.
         #
         #push:
-        #  include_content: true
+        #   include_content: true
         """

synapse/config/ratelimiting.py

@@ -32,34 +32,27 @@ class RatelimitConfig(Config):
         ## Ratelimiting ##
 
         # Number of messages a client can send per second
-        #
         rc_messages_per_second: 0.2
 
         # Number of message a client can send before being throttled
-        #
         rc_message_burst_count: 10.0
 
         # The federation window size in milliseconds
-        #
         federation_rc_window_size: 1000
 
         # The number of federation requests from a single server in a window
         # before the server will delay processing the request.
-        #
         federation_rc_sleep_limit: 10
 
         # The duration in milliseconds to delay processing events from
         # remote servers by if they go over the sleep limit.
-        #
         federation_rc_sleep_delay: 500
 
         # The maximum number of concurrent federation requests allowed
         # from a single server
-        #
         federation_rc_reject_limit: 50
 
         # The number of federation requests to concurrently process from a
         # single server
-        #
         federation_rc_concurrent: 3
         """

synapse/config/registration.py

@@ -50,10 +50,6 @@ class RegistrationConfig(Config):
                 raise ConfigError('Invalid auto_join_rooms entry %s' % (room_alias,))
         self.autocreate_auto_join_rooms = config.get("autocreate_auto_join_rooms", True)
 
-        self.disable_msisdn_registration = (
-            config.get("disable_msisdn_registration", False)
-        )
-
     def default_config(self, generate_secrets=False, **kwargs):
         if generate_secrets:
             registration_shared_secret = 'registration_shared_secret: "%s"' % (
@@ -70,29 +66,23 @@ class RegistrationConfig(Config):
 
         # The user must provide all of the below types of 3PID when registering.
         #
-        #registrations_require_3pid:
-        #  - email
-        #  - msisdn
-
-        # Explicitly disable asking for MSISDNs from the registration
-        # flow (overrides registrations_require_3pid if MSISDNs are set as required)
-        #
-        #disable_msisdn_registration: True
+        # registrations_require_3pid:
+        #     - email
+        #     - msisdn
 
         # Mandate that users are only allowed to associate certain formats of
         # 3PIDs with accounts on this server.
         #
-        #allowed_local_3pids:
-        #  - medium: email
-        #    pattern: '.*@matrix\\.org'
-        #  - medium: email
-        #    pattern: '.*@vector\\.im'
-        #  - medium: msisdn
-        #    pattern: '\\+44'
+        # allowed_local_3pids:
+        #     - medium: email
+        #       pattern: ".*@matrix\\.org"
+        #     - medium: email
+        #       pattern: ".*@vector\\.im"
+        #     - medium: msisdn
+        #       pattern: "\\+44"
 
         # If set, allows registration by anyone who also has the shared
         # secret, even if registration is otherwise disabled.
-        #
         %(registration_shared_secret)s
 
         # Set the number of bcrypt rounds used to generate password hash.
@@ -100,13 +90,11 @@ class RegistrationConfig(Config):
         # The default number is 12 (which equates to 2^12 rounds).
         # N.B. that increasing this will exponentially increase the time required
         # to register or login - e.g. 24 => 2^24 rounds which will take >20 mins.
-        #
         bcrypt_rounds: 12
 
         # Allows users to register as guests without a password/email/etc, and
         # participate in rooms hosted on this server which have been made
         # accessible to anonymous users.
-        #
         allow_guest_access: False
 
         # The identity server which we suggest that clients should use when users log
@@ -115,30 +103,27 @@ class RegistrationConfig(Config):
         # (By default, no suggestion is made, so it is left up to the client.
         # This setting is ignored unless public_baseurl is also set.)
         #
-        #default_identity_server: https://matrix.org
+        # default_identity_server: https://matrix.org
 
         # The list of identity servers trusted to verify third party
         # identifiers by this server.
         #
         # Also defines the ID server which will be called when an account is
         # deactivated (one will be picked arbitrarily).
-        #
         trusted_third_party_id_servers:
-          - matrix.org
-          - vector.im
+            - matrix.org
+            - vector.im
 
         # Users who register on this homeserver will automatically be joined
         # to these rooms
-        #
         #auto_join_rooms:
-        #  - "#example:example.com"
+        #    - "#example:example.com"
 
         # Where auto_join_rooms are specified, setting this flag ensures that the
         # the rooms exist by creating them when the first user on the
         # homeserver registers.
         # Setting to false means that if the rooms are not manually created,
         # users cannot be auto-joined since they do not exist.
-        #
         autocreate_auto_join_rooms: true
         """ % locals()
 

synapse/config/repository.py

@@ -180,34 +180,29 @@ class ContentRepositoryConfig(Config):
         uploads_path = os.path.join(data_dir_path, "uploads")
         return r"""
         # Directory where uploaded images and attachments are stored.
-        #
         media_store_path: "%(media_store)s"
 
         # Media storage providers allow media to be stored in different
         # locations.
-        #
-        #media_storage_providers:
-        #  - module: file_system
-        #    # Whether to write new local files.
-        #    store_local: false
-        #    # Whether to write new remote media
-        #    store_remote: false
-        #    # Whether to block upload requests waiting for write to this
-        #    # provider to complete
-        #    store_synchronous: false
-        #    config:
-        #       directory: /mnt/some/other/directory
+        # media_storage_providers:
+        # - module: file_system
+        #   # Whether to write new local files.
+        #   store_local: false
+        #   # Whether to write new remote media
+        #   store_remote: false
+        #   # Whether to block upload requests waiting for write to this
+        #   # provider to complete
+        #   store_synchronous: false
+        #   config:
+        #     directory: /mnt/some/other/directory
 
         # Directory where in-progress uploads are stored.
-        #
         uploads_path: "%(uploads_path)s"
 
         # The largest allowed upload size in bytes
-        #
         max_upload_size: "10M"
 
         # Maximum number of pixels that will be thumbnailed
-        #
         max_image_pixels: "32M"
 
         # Whether to generate new thumbnails on the fly to precisely match
@@ -215,11 +210,9 @@ class ContentRepositoryConfig(Config):
         # a new resolution is requested by the client the server will
         # generate a new thumbnail. If false the server will pick a thumbnail
         # from a precalculated list.
-        #
         dynamic_thumbnails: false
 
-        # List of thumbnails to precalculate when an image is uploaded.
-        #
+        # List of thumbnail to precalculate when an image is uploaded.
         thumbnail_sizes:
         - width: 32
           height: 32
@@ -240,7 +233,6 @@ class ContentRepositoryConfig(Config):
         # Is the preview URL API enabled?  If enabled, you *must* specify
         # an explicit url_preview_ip_range_blacklist of IPs that the spider is
         # denied from accessing.
-        #
         url_preview_enabled: False
 
         # List of IP address CIDR ranges that the URL preview spider is denied
@@ -251,16 +243,16 @@ class ContentRepositoryConfig(Config):
         # synapse to issue arbitrary GET requests to your internal services,
         # causing serious security issues.
         #
-        #url_preview_ip_range_blacklist:
-        #  - '127.0.0.0/8'
-        #  - '10.0.0.0/8'
-        #  - '172.16.0.0/12'
-        #  - '192.168.0.0/16'
-        #  - '100.64.0.0/10'
-        #  - '169.254.0.0/16'
-        #  - '::1/128'
-        #  - 'fe80::/64'
-        #  - 'fc00::/7'
+        # url_preview_ip_range_blacklist:
+        # - '127.0.0.0/8'
+        # - '10.0.0.0/8'
+        # - '172.16.0.0/12'
+        # - '192.168.0.0/16'
+        # - '100.64.0.0/10'
+        # - '169.254.0.0/16'
+        # - '::1/128'
+        # - 'fe80::/64'
+        # - 'fc00::/7'
         #
         # List of IP address CIDR ranges that the URL preview spider is allowed
         # to access even if they are specified in url_preview_ip_range_blacklist.
@@ -268,8 +260,8 @@ class ContentRepositoryConfig(Config):
         # target IP ranges - e.g. for enabling URL previews for a specific private
         # website only visible in your network.
         #
-        #url_preview_ip_range_whitelist:
-        #   - '192.168.1.1'
+        # url_preview_ip_range_whitelist:
+        # - '192.168.1.1'
 
         # Optional list of URL matches that the URL preview spider is
         # denied from accessing.  You should use url_preview_ip_range_blacklist
@@ -287,25 +279,26 @@ class ContentRepositoryConfig(Config):
         # specified component matches for a given list item succeed, the URL is
         # blacklisted.
         #
-        #url_preview_url_blacklist:
-        #  # blacklist any URL with a username in its URI
-        #  - username: '*'
+        # url_preview_url_blacklist:
+        # # blacklist any URL with a username in its URI
+        # - username: '*'
         #
-        #  # blacklist all *.google.com URLs
-        #  - netloc: 'google.com'
-        #  - netloc: '*.google.com'
+        # # blacklist all *.google.com URLs
+        # - netloc: 'google.com'
+        # - netloc: '*.google.com'
         #
-        #  # blacklist all plain HTTP URLs
-        #  - scheme: 'http'
+        # # blacklist all plain HTTP URLs
+        # - scheme: 'http'
         #
-        #  # blacklist http(s)://www.acme.com/foo
-        #  - netloc: 'www.acme.com'
-        #    path: '/foo'
+        # # blacklist http(s)://www.acme.com/foo
+        # - netloc: 'www.acme.com'
+        #   path: '/foo'
         #
-        #  # blacklist any URL with a literal IPv4 address
-        #  - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'
+        # # blacklist any URL with a literal IPv4 address
+        # - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'
 
         # The largest allowed URL preview spidering size in bytes
         max_spider_size: "10M"
 
+
         """ % locals()

synapse/config/room_directory.py

@@ -20,37 +20,12 @@ from ._base import Config, ConfigError
 
 class RoomDirectoryConfig(Config):
     def read_config(self, config):
-        alias_creation_rules = config.get("alias_creation_rules")
+        alias_creation_rules = config["alias_creation_rules"]
 
-        if alias_creation_rules is not None:
-            self._alias_creation_rules = [
-                _RoomDirectoryRule("alias_creation_rules", rule)
-                for rule in alias_creation_rules
-            ]
-        else:
-            self._alias_creation_rules = [
-                _RoomDirectoryRule(
-                    "alias_creation_rules", {
-                        "action": "allow",
-                    }
-                )
-            ]
-
-        room_list_publication_rules = config.get("room_list_publication_rules")
-
-        if room_list_publication_rules is not None:
-            self._room_list_publication_rules = [
-                _RoomDirectoryRule("room_list_publication_rules", rule)
-                for rule in room_list_publication_rules
-            ]
-        else:
-            self._room_list_publication_rules = [
-                _RoomDirectoryRule(
-                    "room_list_publication_rules", {
-                        "action": "allow",
-                    }
-                )
-            ]
+        self._alias_creation_rules = [
+            _AliasRule(rule)
+            for rule in alias_creation_rules
+        ]
 
     def default_config(self, config_dir_path, server_name, **kwargs):
         return """
@@ -58,138 +33,60 @@ class RoomDirectoryConfig(Config):
         # on this server.
         #
         # The format of this option is a list of rules that contain globs that
-        # match against user_id, room_id and the new alias (fully qualified with
-        # server name). The action in the first rule that matches is taken,
-        # which can currently either be "allow" or "deny".
+        # match against user_id and the new alias (fully qualified with server
+        # name). The action in the first rule that matches is taken, which can
+        # currently either be "allow" or "deny".
         #
-        # Missing user_id/room_id/alias fields default to "*".
-        #
-        # If no rules match the request is denied. An empty list means no one
-        # can create aliases.
-        #
-        # Options for the rules include:
-        #
-        #   user_id: Matches against the creator of the alias
-        #   alias: Matches against the alias being created
-        #   room_id: Matches against the room ID the alias is being pointed at
-        #   action: Whether to "allow" or "deny" the request if the rule matches
-        #
-        # The default is:
-        #
-        #alias_creation_rules:
-        #  - user_id: "*"
-        #    alias: "*"
-        #    room_id: "*"
-        #    action: allow
-
-        # The `room_list_publication_rules` option controls who can publish and
-        # which rooms can be published in the public room list.
-        #
-        # The format of this option is the same as that for
-        # `alias_creation_rules`.
-        #
-        # If the room has one or more aliases associated with it, only one of
-        # the aliases needs to match the alias rule. If there are no aliases
-        # then only rules with `alias: *` match.
-        #
-        # If no rules match the request is denied. An empty list means no one
-        # can publish rooms.
-        #
-        # Options for the rules include:
-        #
-        #   user_id: Matches agaisnt the creator of the alias
-        #   room_id: Matches against the room ID being published
-        #   alias: Matches against any current local or canonical aliases
-        #            associated with the room
-        #   action: Whether to "allow" or "deny" the request if the rule matches
-        #
-        # The default is:
-        #
-        #room_list_publication_rules:
-        #  - user_id: "*"
-        #    alias: "*"
-        #    room_id: "*"
-        #    action: allow
+        # If no rules match the request is denied.
+        alias_creation_rules:
+            - user_id: "*"
+              alias: "*"
+              action: allow
         """
 
-    def is_alias_creation_allowed(self, user_id, room_id, alias):
+    def is_alias_creation_allowed(self, user_id, alias):
         """Checks if the given user is allowed to create the given alias
 
         Args:
             user_id (str)
-            room_id (str)
             alias (str)
 
         Returns:
             boolean: True if user is allowed to crate the alias
         """
         for rule in self._alias_creation_rules:
-            if rule.matches(user_id, room_id, [alias]):
-                return rule.action == "allow"
-
-        return False
-
-    def is_publishing_room_allowed(self, user_id, room_id, aliases):
-        """Checks if the given user is allowed to publish the room
-
-        Args:
-            user_id (str)
-            room_id (str)
-            aliases (list[str]): any local aliases associated with the room
-
-        Returns:
-            boolean: True if user can publish room
-        """
-        for rule in self._room_list_publication_rules:
-            if rule.matches(user_id, room_id, aliases):
+            if rule.matches(user_id, alias):
                 return rule.action == "allow"
 
         return False
 
 
-class _RoomDirectoryRule(object):
-    """Helper class to test whether a room directory action is allowed, like
-    creating an alias or publishing a room.
-    """
-
-    def __init__(self, option_name, rule):
-        """
-        Args:
-            option_name (str): Name of the config option this rule belongs to
-            rule (dict): The rule as specified in the config
-        """
-
+class _AliasRule(object):
+    def __init__(self, rule):
         action = rule["action"]
-        user_id = rule.get("user_id", "*")
-        room_id = rule.get("room_id", "*")
-        alias = rule.get("alias", "*")
+        user_id = rule["user_id"]
+        alias = rule["alias"]
 
         if action in ("allow", "deny"):
             self.action = action
         else:
             raise ConfigError(
-                "%s rules can only have action of 'allow'"
-                " or 'deny'" % (option_name,)
+                "alias_creation_rules rules can only have action of 'allow'"
+                " or 'deny'"
             )
 
-        self._alias_matches_all = alias == "*"
-
         try:
             self._user_id_regex = glob_to_regex(user_id)
             self._alias_regex = glob_to_regex(alias)
-            self._room_id_regex = glob_to_regex(room_id)
         except Exception as e:
             raise ConfigError("Failed to parse glob into regex: %s", e)
 
-    def matches(self, user_id, room_id, aliases):
-        """Tests if this rule matches the given user_id, room_id and aliases.
+    def matches(self, user_id, alias):
+        """Tests if this rule matches the given user_id and alias.
 
         Args:
             user_id (str)
-            room_id (str)
-            aliases (list[str]): The associated aliases to the room. Will be a
-                single element for testing alias creation, and can be empty for
-                testing room publishing.
+            alias (str)
 
         Returns:
             boolean
@@ -199,22 +96,7 @@ class _RoomDirectoryRule(object):
         if not self._user_id_regex.match(user_id):
             return False
 
-        if not self._room_id_regex.match(room_id):
+        if not self._alias_regex.match(alias):
             return False
 
-        # We only have alias checks left, so we can short circuit if the alias
-        # rule matches everything.
-        if self._alias_matches_all:
-            return True
-
-        # If we are not given any aliases then this rule only matches if the
-        # alias glob matches all aliases, which we checked above.
-        if not aliases:
-            return False
-
-        # Otherwise, we just need one alias to match
-        for alias in aliases:
-            if self._alias_regex.match(alias):
-                return True
-
-        return False
+        return True

synapse/config/saml2_config.py

@@ -67,43 +67,44 @@ class SAML2Config(Config):
         return """
         # Enable SAML2 for registration and login. Uses pysaml2.
         #
-        # `sp_config` is the configuration for the pysaml2 Service Provider.
-        # See pysaml2 docs for format of config.
+        # saml2_config:
         #
-        # Default values will be used for the 'entityid' and 'service' settings,
-        # so it is not normally necessary to specify them unless you need to
-        # override them.
+        #   # The following is the configuration for the pysaml2 Service Provider.
+        #   # See pysaml2 docs for format of config.
+        #   #
+        #   # Default values will be used for the 'entityid' and 'service' settings,
+        #   # so it is not normally necessary to specify them unless you need to
+        #   # override them.
         #
-        #saml2_config:
-        #  sp_config:
-        #    # point this to the IdP's metadata. You can use either a local file or
-        #    # (preferably) a URL.
-        #    metadata:
-        #      #local: ["saml2/idp.xml"]
-        #      remote:
-        #        - url: https://our_idp/metadata.xml
+        #   sp_config:
+        #     # point this to the IdP's metadata. You can use either a local file or
+        #     # (preferably) a URL.
+        #     metadata:
+        #       # local: ["saml2/idp.xml"]
+        #       remote:
+        #         - url: https://our_idp/metadata.xml
         #
-        #    # The rest of sp_config is just used to generate our metadata xml, and you
-        #    # may well not need it, depending on your setup. Alternatively you
-        #    # may need a whole lot more detail - see the pysaml2 docs!
+        #     # The following is just used to generate our metadata xml, and you
+        #     # may well not need it, depending on your setup. Alternatively you
+        #     # may need a whole lot more detail - see the pysaml2 docs!
         #
-        #    description: ["My awesome SP", "en"]
-        #    name: ["Test SP", "en"]
+        #     description: ["My awesome SP", "en"]
+        #     name: ["Test SP", "en"]
         #
-        #    organization:
-        #      name: Example com
-        #      display_name:
-        #        - ["Example co", "en"]
-        #      url: "http://example.com"
+        #     organization:
+        #       name: Example com
+        #       display_name:
+        #         - ["Example co", "en"]
+        #       url: "http://example.com"
         #
-        #    contact_person:
-        #      - given_name: Bob
-        #        sur_name: "the Sysadmin"
-        #        email_address": ["admin@example.com"]
-        #        contact_type": technical
+        #     contact_person:
+        #       - given_name: Bob
+        #         sur_name: "the Sysadmin"
+        #         email_address": ["admin@example.com"]
+        #         contact_type": technical
         #
-        #  # Instead of putting the config inline as above, you can specify a
-        #  # separate pysaml2 configuration file:
-        #  #
-        #  config_path: "%(config_dir_path)s/sp_conf.py"
+        #   # Instead of putting the config inline as above, you can specify a
+        #   # separate pysaml2 configuration file:
+        #   #
+        #   # config_path: "%(config_dir_path)s/sp_conf.py"
         """ % {"config_dir_path": config_dir_path}

synapse/config/server.py

@@ -24,14 +24,6 @@ from ._base import Config, ConfigError
 
 logger = logging.Logger(__name__)
 
-# by default, we attempt to listen on both '::' *and* '0.0.0.0' because some OSes
-# (Windows, macOS, other BSD/Linux where net.ipv6.bindv6only is set) will only listen
-# on IPv6 when '::' is set.
-#
-# We later check for errors when binding to 0.0.0.0 and ignore them if :: is also in
-# in the list.
-DEFAULT_BIND_ADDRESSES = ['::', '0.0.0.0']
-
 
 class ServerConfig(Config):
 
@@ -115,49 +107,38 @@ class ServerConfig(Config):
         federation_domain_whitelist = config.get(
             "federation_domain_whitelist", None
         )
+
         # turn the whitelist into a hash for speed of lookup
         if federation_domain_whitelist is not None:
             self.federation_domain_whitelist = {}
             for domain in federation_domain_whitelist:
                 self.federation_domain_whitelist[domain] = True
 
+        # Optional proxy address for federation traffic
+        self.federation_request_gateway_addr = config.get(
+            "federation_request_gateway_addr", None
+        )
+
+        if self.federation_request_gateway_addr is not None:
+            # Ensure proxy address is correctly formatted
+            if len(self.federation_request_gateway_addr.split(':')) != 2:
+                self.federation_request_gateway_addr = None
+
         if self.public_baseurl is not None:
             if self.public_baseurl[-1] != '/':
                 self.public_baseurl += '/'
         self.start_pushers = config.get("start_pushers", True)
 
-        self.listeners = []
-        for listener in config.get("listeners", []):
-            if not isinstance(listener.get("port", None), int):
-                raise ConfigError(
-                    "Listener configuration is lacking a valid 'port' option"
-                )
-
-            if listener.setdefault("tls", False):
-                # no_tls is not really supported any more, but let's grandfather it in
-                # here.
-                if config.get("no_tls", False):
-                    logger.info(
-                        "Ignoring TLS-enabled listener on port %i due to no_tls"
-                    )
-                    continue
+        self.listeners = config.get("listeners", [])
 
+        for listener in self.listeners:
             bind_address = listener.pop("bind_address", None)
             bind_addresses = listener.setdefault("bind_addresses", [])
 
-            # if bind_address was specified, add it to the list of addresses
             if bind_address:
                 bind_addresses.append(bind_address)
-
-            # if we still have an empty list of addresses, use the default list
-            if not bind_addresses:
-                if listener['type'] == 'metrics':
-                    # the metrics listener doesn't support IPv6
-                    bind_addresses.append('0.0.0.0')
-                else:
-                    bind_addresses.extend(DEFAULT_BIND_ADDRESSES)
-
-            self.listeners.append(listener)
+            elif not bind_addresses:
+                bind_addresses.append('')
 
         if not self.web_client_location:
             _warn_if_webclient_configured(self.listeners)
@@ -166,9 +147,6 @@ class ServerConfig(Config):
 
         bind_port = config.get("bind_port")
         if bind_port:
-            if config.get("no_tls", False):
-                raise ConfigError("no_tls is incompatible with bind_port")
-
             self.listeners = []
             bind_host = config.get("bind_host", "")
             gzip_responses = config.get("gzip_responses", True)
@@ -215,7 +193,6 @@ class ServerConfig(Config):
                 "port": manhole,
                 "bind_addresses": ["127.0.0.1"],
                 "type": "manhole",
-                "tls": False,
             })
 
         metrics_port = config.get("metrics_port")
@@ -241,9 +218,6 @@ class ServerConfig(Config):
 
         _check_resource_config(self.listeners)
 
-    def has_tls_listener(self):
-        return any(l["tls"] for l in self.listeners)
-
     def default_config(self, server_name, data_dir_path, **kwargs):
         _, bind_port = parse_and_validate_server_name(server_name)
         if bind_port is not None:
@@ -286,20 +260,15 @@ class ServerConfig(Config):
         #
         # This setting requires the affinity package to be installed!
         #
-        #cpu_affinity: 0xFFFFFFFF
+        # cpu_affinity: 0xFFFFFFFF
 
         # The path to the web client which will be served at /_matrix/client/
         # if 'webclient' is configured under the 'listeners' configuration.
         #
-        #web_client_location: "/path/to/web/root"
+        # web_client_location: "/path/to/web/root"
 
-        # The public-facing base URL that clients use to access this HS
-        # (not including _matrix/...). This is the same URL a user would
-        # enter into the 'custom HS URL' field on their client. If you
-        # use synapse with a reverse proxy, this should be the URL to reach
-        # synapse via the proxy.
-        #
-        #public_baseurl: https://example.com/
+        # The public-facing base URL for the client API (not including _matrix/...)
+        # public_baseurl: https://example.com:8448/
 
         # Set the soft limit on the number of file descriptors synapse can use
         # Zero is used to indicate synapse should set the soft limit to the
@@ -310,25 +279,15 @@ class ServerConfig(Config):
         use_presence: true
 
         # The GC threshold parameters to pass to `gc.set_threshold`, if defined
-        #
-        #gc_thresholds: [700, 10, 10]
+        # gc_thresholds: [700, 10, 10]
 
         # Set the limit on the returned events in the timeline in the get
         # and sync operations. The default value is -1, means no upper limit.
-        #
-        #filter_timeline_limit: 5000
+        # filter_timeline_limit: 5000
 
         # Whether room invites to users on this server should be blocked
         # (except those sent by local server admins). The default is False.
-        #
-        #block_non_admin_invites: True
-
-        # Room searching
-        #
-        # If disabled, new messages will not be indexed for searching and users
-        # will receive errors when searching for messages. Defaults to enabled.
-        #
-        #enable_search: false
+        # block_non_admin_invites: True
 
         # Restrict federation to the following whitelist of domains.
         # N.B. we recommend also firewalling your federation listener to limit
@@ -336,145 +295,122 @@ class ServerConfig(Config):
         # purely on this application-layer restriction.  If not specified, the
         # default is to whitelist everything.
         #
-        #federation_domain_whitelist:
+        # federation_domain_whitelist:
         #  - lon.example.com
         #  - nyc.example.com
         #  - syd.example.com
 
+        # Send outbound federation requests through a seperate traffic gateway.
+        # Not intended to be used with a SOCKS proxy, but rather a relay for
+        # HTTP traffic.
+        # federation_request_gateway_addr: localhost:1234
+
         # List of ports that Synapse should listen on, their purpose and their
         # configuration.
-        #
-        # Options for each listener include:
-        #
-        #   port: the TCP port to bind to
-        #
-        #   bind_addresses: a list of local addresses to listen on. The default is
-        #       'all local interfaces'.
-        #
-        #   type: the type of listener. Normally 'http', but other valid options are:
-        #       'manhole' (see docs/manhole.md),
-        #       'metrics' (see docs/metrics-howto.rst),
-        #       'replication' (see docs/workers.rst).
-        #
-        #   tls: set to true to enable TLS for this listener. Will use the TLS
-        #       key/cert specified in tls_private_key_path / tls_certificate_path.
-        #
-        #   x_forwarded: Only valid for an 'http' listener. Set to true to use the
-        #       X-Forwarded-For header as the client IP. Useful when Synapse is
-        #       behind a reverse-proxy.
-        #
-        #   resources: Only valid for an 'http' listener. A list of resources to host
-        #       on this port. Options for each resource are:
-        #
-        #       names: a list of names of HTTP resources. See below for a list of
-        #           valid resource names.
-        #
-        #       compress: set to true to enable HTTP comression for this resource.
-        #
-        #   additional_resources: Only valid for an 'http' listener. A map of
-        #        additional endpoints which should be loaded via dynamic modules.
-        #
-        # Valid resource names are:
-        #
-        #   client: the client-server API (/_matrix/client). Also implies 'media' and
-        #       'static'.
-        #
-        #   consent: user consent forms (/_matrix/consent). See
-        #       docs/consent_tracking.md.
-        #
-        #   federation: the server-server API (/_matrix/federation). Also implies
-        #       'media', 'keys', 'openid'
-        #
-        #   keys: the key discovery API (/_matrix/keys).
-        #
-        #   media: the media API (/_matrix/media).
-        #
-        #   metrics: the metrics interface. See docs/metrics-howto.rst.
-        #
-        #   openid: OpenID authentication.
-        #
-        #   replication: the HTTP replication API (/_synapse/replication). See
-        #       docs/workers.rst.
-        #
-        #   static: static resources under synapse/static (/_matrix/static). (Mostly
-        #       useful for 'fallback authentication'.)
-        #
-        #   webclient: A web client. Requires web_client_location to be set.
-        #
         listeners:
-          # TLS-enabled listener: for when matrix traffic is sent directly to synapse.
-          #
-          # Disabled by default. To enable it, uncomment the following. (Note that you
-          # will also need to give Synapse a TLS key and certificate: see the TLS section
-          # below.)
-          #
-          #- port: %(bind_port)s
-          #  type: http
-          #  tls: true
-          #  resources:
-          #    - names: [client, federation]
+          # Main HTTPS listener
+          # For when matrix traffic is sent directly to synapse.
+          -
+            # The port to listen for HTTPS requests on.
+            port: %(bind_port)s
 
-          # Unsecure HTTP listener: for when matrix traffic passes through a reverse proxy
-          # that unwraps TLS.
-          #
-          # If you plan to use a reverse proxy, please see
-          # https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.rst.
-          #
-          - port: %(unsecure_port)s
-            tls: false
-            bind_addresses: ['::1', '127.0.0.1']
+            # Local addresses to listen on.
+            # On Linux and Mac OS, `::` will listen on all IPv4 and IPv6
+            # addresses by default. For most other OSes, this will only listen
+            # on IPv6.
+            bind_addresses:
+              - '::'
+              - '0.0.0.0'
+
+            # This is a 'http' listener, allows us to specify 'resources'.
             type: http
-            x_forwarded: true
 
+            tls: true
+
+            # Use the X-Forwarded-For (XFF) header as the client IP and not the
+            # actual client IP.
+            x_forwarded: false
+
+            # List of HTTP resources to serve on this listener.
             resources:
-              - names: [client, federation]
+              -
+                # List of resources to host on this listener.
+                names:
+                  - client       # The client-server APIs, both v1 and v2
+                  # - webclient  # A web client. Requires web_client_location to be set.
+
+                # Should synapse compress HTTP responses to clients that support it?
+                # This should be disabled if running synapse behind a load balancer
+                # that can do automatic compression.
+                compress: true
+
+              - names: [federation]  # Federation APIs
                 compress: false
 
-            # example additonal_resources:
-            #
-            #additional_resources:
-            #  "/_matrix/my/custom/endpoint":
-            #    module: my_module.CustomRequestHandler
-            #    config: {}
+            # optional list of additional endpoints which can be loaded via
+            # dynamic modules
+            # additional_resources:
+            #   "/_matrix/my/custom/endpoint":
+            #     module: my_module.CustomRequestHandler
+            #     config: {}
+
+          # Unsecure HTTP listener,
+          # For when matrix traffic passes through loadbalancer that unwraps TLS.
+          - port: %(unsecure_port)s
+            tls: false
+            bind_addresses: ['::', '0.0.0.0']
+            type: http
+
+            x_forwarded: false
+
+            resources:
+              - names: [client]
+                compress: true
+              - names: [federation]
+                compress: false
 
           # Turn on the twisted ssh manhole service on localhost on the given
           # port.
-          #
-          #- port: 9000
-          #  bind_addresses: ['::1', '127.0.0.1']
-          #  type: manhole
+          # - port: 9000
+          #   bind_addresses: ['::1', '127.0.0.1']
+          #   type: manhole
 
 
-        ## Homeserver blocking ##
-
+        # Homeserver blocking
+        #
         # How to reach the server admin, used in ResourceLimitError
+        # admin_contact: 'mailto:admin@server.com'
         #
-        #admin_contact: 'mailto:admin@server.com'
-
-        # Global blocking
+        # Global block config
+        #
+        # hs_disabled: False
+        # hs_disabled_message: 'Human readable reason for why the HS is blocked'
+        # hs_disabled_limit_type: 'error code(str), to help clients decode reason'
         #
-        #hs_disabled: False
-        #hs_disabled_message: 'Human readable reason for why the HS is blocked'
-        #hs_disabled_limit_type: 'error code(str), to help clients decode reason'
-
         # Monthly Active User Blocking
         #
-        #limit_usage_by_mau: False
-        #max_mau_value: 50
-        #mau_trial_days: 2
-
+        # Enables monthly active user checking
+        # limit_usage_by_mau: False
+        # max_mau_value: 50
+        # mau_trial_days: 2
+        #
         # If enabled, the metrics for the number of monthly active users will
         # be populated, however no one will be limited. If limit_usage_by_mau
         # is true, this is implied to be true.
+        # mau_stats_only: False
         #
-        #mau_stats_only: False
-
         # Sometimes the server admin will want to ensure certain accounts are
         # never blocked by mau checking. These accounts are specified here.
         #
-        #mau_limit_reserved_threepids:
-        #  - medium: 'email'
-        #    address: 'reserved_user@example.com'
+        # mau_limit_reserved_threepids:
+        # - medium: 'email'
+        #   address: 'reserved_user@example.com'
+        #
+        # Room searching
+        #
+        # If disabled, new messages will not be indexed for searching and users
+        # will receive errors when searching for messages. Defaults to enabled.
+        # enable_search: true
         """ % locals()
 
     def read_arguments(self, args):
@@ -500,18 +436,19 @@ class ServerConfig(Config):
                                   " service on the given port.")
 
 
-def is_threepid_reserved(reserved_threepids, threepid):
+def is_threepid_reserved(config, threepid):
     """Check the threepid against the reserved threepid config
     Args:
-        reserved_threepids([dict]) - list of reserved threepids
+        config(ServerConfig) - to access server config attributes
         threepid(dict) - The threepid to test for
 
     Returns:
         boolean Is the threepid undertest reserved_user
     """
 
-    for tp in reserved_threepids:
-        if (threepid['medium'] == tp['medium'] and threepid['address'] == tp['address']):
+    for tp in config.mau_limits_reserved_threepids:
+        if (threepid['medium'] == tp['medium']
+                and threepid['address'] == tp['address']):
             return True
     return False
 
@@ -556,7 +493,6 @@ KNOWN_RESOURCES = (
     'keys',
     'media',
     'metrics',
-    'openid',
     'replication',
     'static',
     'webclient',

synapse/config/server_notices_config.py

@@ -30,11 +30,11 @@ DEFAULT_CONFIG = """\
 # It's also possible to override the room name, the display name of the
 # "notices" user, and the avatar for the user.
 #
-#server_notices:
-#  system_mxid_localpart: notices
-#  system_mxid_display_name: "Server Notices"
-#  system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
-#  room_name: "Server Notices"
+# server_notices:
+#   system_mxid_localpart: notices
+#   system_mxid_display_name: "Server Notices"
+#   system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
+#   room_name: "Server Notices"
 """
 
 

synapse/config/spam_checker.py

@@ -28,8 +28,8 @@ class SpamCheckerConfig(Config):
 
     def default_config(self, **kwargs):
         return """\
-        #spam_checker:
-        #  module: "my_custom_project.SuperSpamChecker"
-        #  config:
-        #    example_option: 'things'
+        # spam_checker:
+        #     module: "my_custom_project.SuperSpamChecker"
+        #     config:
+        #         example_option: 'things'
         """

synapse/config/tls.py

@@ -13,62 +13,51 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import logging
 import os
-import warnings
-from datetime import datetime
+import subprocess
 from hashlib import sha256
 
-import six
-
 from unpaddedbase64 import encode_base64
 
 from OpenSSL import crypto
 
-from synapse.config._base import Config, ConfigError
+from ._base import Config
 
-logger = logging.getLogger(__name__)
+GENERATE_DH_PARAMS = False
 
 
 class TlsConfig(Config):
     def read_config(self, config):
+        self.tls_certificate = self.read_tls_certificate(
+            config.get("tls_certificate_path")
+        )
+        self.tls_certificate_file = config.get("tls_certificate_path")
 
-        acme_config = config.get("acme", None)
-        if acme_config is None:
-            acme_config = {}
+        self.no_tls = config.get("no_tls", False)
 
-        self.acme_enabled = acme_config.get("enabled", False)
+        if self.no_tls:
+            self.tls_private_key = None
+        else:
+            self.tls_private_key = self.read_tls_private_key(
+                config.get("tls_private_key_path")
+            )
 
-        # hyperlink complains on py2 if this is not a Unicode
-        self.acme_url = six.text_type(acme_config.get(
-            "url", u"https://acme-v01.api.letsencrypt.org/directory"
-        ))
-        self.acme_port = acme_config.get("port", 80)
-        self.acme_bind_addresses = acme_config.get("bind_addresses", ['::', '0.0.0.0'])
-        self.acme_reprovision_threshold = acme_config.get("reprovision_threshold", 30)
-        self.acme_domain = acme_config.get("domain", config.get("server_name"))
+        self.tls_dh_params_path = self.check_file(
+            config.get("tls_dh_params_path"), "tls_dh_params"
+        )
 
-        self.tls_certificate_file = self.abspath(config.get("tls_certificate_path"))
-        self.tls_private_key_file = self.abspath(config.get("tls_private_key_path"))
+        self.tls_fingerprints = config["tls_fingerprints"]
 
-        if self.has_tls_listener():
-            if not self.tls_certificate_file:
-                raise ConfigError(
-                    "tls_certificate_path must be specified if TLS-enabled listeners are "
-                    "configured."
-                )
-            if not self.tls_private_key_file:
-                raise ConfigError(
-                    "tls_private_key_path must be specified if TLS-enabled listeners are "
-                    "configured."
-                )
-
-        self._original_tls_fingerprints = config.get("tls_fingerprints", [])
-
-        if self._original_tls_fingerprints is None:
-            self._original_tls_fingerprints = []
-
-        self.tls_fingerprints = list(self._original_tls_fingerprints)
+        # Check that our own certificate is included in the list of fingerprints
+        # and include it if it is not.
+        x509_certificate_bytes = crypto.dump_certificate(
+            crypto.FILETYPE_ASN1,
+            self.tls_certificate
+        )
+        sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest())
+        sha256_fingerprints = set(f["sha256"] for f in self.tls_fingerprints)
+        if sha256_fingerprint not in sha256_fingerprints:
+            self.tls_fingerprints.append({u"sha256": sha256_fingerprint})
 
         # This config option applies to non-federation HTTP clients
         # (e.g. for talking to recaptcha, identity servers, and such)
@@ -78,176 +67,29 @@ class TlsConfig(Config):
             "use_insecure_ssl_client_just_for_testing_do_not_use"
         )
 
-        self.tls_certificate = None
-        self.tls_private_key = None
-
-    def is_disk_cert_valid(self, allow_self_signed=True):
-        """
-        Is the certificate we have on disk valid, and if so, for how long?
-
-        Args:
-            allow_self_signed (bool): Should we allow the certificate we
-                read to be self signed?
-
-        Returns:
-            int: Days remaining of certificate validity.
-            None: No certificate exists.
-        """
-        if not os.path.exists(self.tls_certificate_file):
-            return None
-
-        try:
-            with open(self.tls_certificate_file, 'rb') as f:
-                cert_pem = f.read()
-        except Exception:
-            logger.exception("Failed to read existing certificate off disk!")
-            raise
-
-        try:
-            tls_certificate = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)
-        except Exception:
-            logger.exception("Failed to parse existing certificate off disk!")
-            raise
-
-        if not allow_self_signed:
-            if tls_certificate.get_subject() == tls_certificate.get_issuer():
-                raise ValueError(
-                    "TLS Certificate is self signed, and this is not permitted"
-                )
-
-        # YYYYMMDDhhmmssZ -- in UTC
-        expires_on = datetime.strptime(
-            tls_certificate.get_notAfter().decode('ascii'), "%Y%m%d%H%M%SZ"
-        )
-        now = datetime.utcnow()
-        days_remaining = (expires_on - now).days
-        return days_remaining
-
-    def read_certificate_from_disk(self, require_cert_and_key):
-        """
-        Read the certificates and private key from disk.
-
-        Args:
-            require_cert_and_key (bool): set to True to throw an error if the certificate
-                and key file are not given
-        """
-        if require_cert_and_key:
-            self.tls_private_key = self.read_tls_private_key()
-            self.tls_certificate = self.read_tls_certificate()
-        elif self.tls_certificate_file:
-            # we only need the certificate for the tls_fingerprints. Reload it if we
-            # can, but it's not a fatal error if we can't.
-            try:
-                self.tls_certificate = self.read_tls_certificate()
-            except Exception as e:
-                logger.info(
-                    "Unable to read TLS certificate (%s). Ignoring as no "
-                    "tls listeners enabled.", e,
-                )
-
-        self.tls_fingerprints = list(self._original_tls_fingerprints)
-
-        if self.tls_certificate:
-            # Check that our own certificate is included in the list of fingerprints
-            # and include it if it is not.
-            x509_certificate_bytes = crypto.dump_certificate(
-                crypto.FILETYPE_ASN1, self.tls_certificate
-            )
-            sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest())
-            sha256_fingerprints = set(f["sha256"] for f in self.tls_fingerprints)
-            if sha256_fingerprint not in sha256_fingerprints:
-                self.tls_fingerprints.append({u"sha256": sha256_fingerprint})
-
     def default_config(self, config_dir_path, server_name, **kwargs):
         base_key_name = os.path.join(config_dir_path, server_name)
 
         tls_certificate_path = base_key_name + ".tls.crt"
         tls_private_key_path = base_key_name + ".tls.key"
+        tls_dh_params_path = base_key_name + ".tls.dh"
 
-        # this is to avoid the max line length. Sorrynotsorry
-        proxypassline = (
-            'ProxyPass /.well-known/acme-challenge '
-            'http://localhost:8009/.well-known/acme-challenge'
-        )
+        return """\
+        # PEM encoded X509 certificate for TLS.
+        # You can replace the self-signed certificate that synapse
+        # autogenerates on launch with your own SSL certificate + key pair
+        # if you like.  Any required intermediary certificates can be
+        # appended after the primary certificate in hierarchical order.
+        tls_certificate_path: "%(tls_certificate_path)s"
 
-        return (
-            """\
-        ## TLS ##
+        # PEM encoded private key for TLS
+        tls_private_key_path: "%(tls_private_key_path)s"
 
-        # PEM-encoded X509 certificate for TLS.
-        # This certificate, as of Synapse 1.0, will need to be a valid and verifiable
-        # certificate, signed by a recognised Certificate Authority.
-        #
-        # See 'ACME support' below to enable auto-provisioning this certificate via
-        # Let's Encrypt.
-        #
-        #tls_certificate_path: "%(tls_certificate_path)s"
+        # PEM dh parameters for ephemeral keys
+        tls_dh_params_path: "%(tls_dh_params_path)s"
 
-        # PEM-encoded private key for TLS
-        #
-        #tls_private_key_path: "%(tls_private_key_path)s"
-
-        # ACME support: This will configure Synapse to request a valid TLS certificate
-        # for your configured `server_name` via Let's Encrypt.
-        #
-        # Note that provisioning a certificate in this way requires port 80 to be
-        # routed to Synapse so that it can complete the http-01 ACME challenge.
-        # By default, if you enable ACME support, Synapse will attempt to listen on
-        # port 80 for incoming http-01 challenges - however, this will likely fail
-        # with 'Permission denied' or a similar error.
-        #
-        # There are a couple of potential solutions to this:
-        #
-        #  * If you already have an Apache, Nginx, or similar listening on port 80,
-        #    you can configure Synapse to use an alternate port, and have your web
-        #    server forward the requests. For example, assuming you set 'port: 8009'
-        #    below, on Apache, you would write:
-        #
-        #    %(proxypassline)s
-        #
-        #  * Alternatively, you can use something like `authbind` to give Synapse
-        #    permission to listen on port 80.
-        #
-        acme:
-            # ACME support is disabled by default. Uncomment the following line
-            # (and tls_certificate_path and tls_private_key_path above) to enable it.
-            #
-            #enabled: true
-
-            # Endpoint to use to request certificates. If you only want to test,
-            # use Let's Encrypt's staging url:
-            #     https://acme-staging.api.letsencrypt.org/directory
-            #
-            #url: https://acme-v01.api.letsencrypt.org/directory
-
-            # Port number to listen on for the HTTP-01 challenge. Change this if
-            # you are forwarding connections through Apache/Nginx/etc.
-            #
-            #port: 80
-
-            # Local addresses to listen on for incoming connections.
-            # Again, you may want to change this if you are forwarding connections
-            # through Apache/Nginx/etc.
-            #
-            #bind_addresses: ['::', '0.0.0.0']
-
-            # How many days remaining on a certificate before it is renewed.
-            #
-            #reprovision_threshold: 30
-
-            # The domain that the certificate should be for. Normally this
-            # should be the same as your Matrix domain (i.e., 'server_name'), but,
-            # by putting a file at 'https://<server_name>/.well-known/matrix/server',
-            # you can delegate incoming traffic to another server. If you do that,
-            # you should give the target of the delegation here.
-            #
-            # For example: if your 'server_name' is 'example.com', but
-            # 'https://example.com/.well-known/matrix/server' delegates to
-            # 'matrix.example.com', you should put 'matrix.example.com' here.
-            #
-            # If not set, defaults to your 'server_name'.
-            #
-            #domain: matrix.example.com
+        # Don't bind to the https port
+        no_tls: False
 
         # List of allowed TLS fingerprints for this server to publish along
         # with the signing keys for this server. Other matrix servers that
@@ -274,44 +116,80 @@ class TlsConfig(Config):
         #   openssl x509 -outform DER | openssl sha256 -binary | base64 | tr -d '='
         # or by checking matrix.org/federationtester/api/report?server_name=$host
         #
-        #tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
+        tls_fingerprints: []
+        # tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
+        """ % locals()
 
-        """
-            % locals()
-        )
+    def read_tls_certificate(self, cert_path):
+        cert_pem = self.read_file(cert_path, "tls_certificate")
+        return crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)
 
-    def read_tls_certificate(self):
-        """Reads the TLS certificate from the configured file, and returns it
-
-        Also checks if it is self-signed, and warns if so
-
-        Returns:
-            OpenSSL.crypto.X509: the certificate
-        """
-        cert_path = self.tls_certificate_file
-        logger.info("Loading TLS certificate from %s", cert_path)
-        cert_pem = self.read_file(cert_path, "tls_certificate_path")
-        cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)
-
-        # Check if it is self-signed, and issue a warning if so.
-        if cert.get_issuer() == cert.get_subject():
-            warnings.warn(
-                (
-                    "Self-signed TLS certificates will not be accepted by Synapse 1.0. "
-                    "Please either provide a valid certificate, or use Synapse's ACME "
-                    "support to provision one."
-                )
-            )
-
-        return cert
-
-    def read_tls_private_key(self):
-        """Reads the TLS private key from the configured file, and returns it
-
-        Returns:
-            OpenSSL.crypto.PKey: the private key
-        """
-        private_key_path = self.tls_private_key_file
-        logger.info("Loading TLS key from %s", private_key_path)
-        private_key_pem = self.read_file(private_key_path, "tls_private_key_path")
+    def read_tls_private_key(self, private_key_path):
+        private_key_pem = self.read_file(private_key_path, "tls_private_key")
         return crypto.load_privatekey(crypto.FILETYPE_PEM, private_key_pem)
+
+    def generate_files(self, config):
+        tls_certificate_path = config["tls_certificate_path"]
+        tls_private_key_path = config["tls_private_key_path"]
+        tls_dh_params_path = config["tls_dh_params_path"]
+
+        if not self.path_exists(tls_private_key_path):
+            with open(tls_private_key_path, "wb") as private_key_file:
+                tls_private_key = crypto.PKey()
+                tls_private_key.generate_key(crypto.TYPE_RSA, 2048)
+                private_key_pem = crypto.dump_privatekey(
+                    crypto.FILETYPE_PEM, tls_private_key
+                )
+                private_key_file.write(private_key_pem)
+        else:
+            with open(tls_private_key_path) as private_key_file:
+                private_key_pem = private_key_file.read()
+                tls_private_key = crypto.load_privatekey(
+                    crypto.FILETYPE_PEM, private_key_pem
+                )
+
+        if not self.path_exists(tls_certificate_path):
+            with open(tls_certificate_path, "wb") as certificate_file:
+                cert = crypto.X509()
+                subject = cert.get_subject()
+                subject.CN = config["server_name"]
+
+                cert.set_serial_number(1000)
+                cert.gmtime_adj_notBefore(0)
+                cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60)
+                cert.set_issuer(cert.get_subject())
+                cert.set_pubkey(tls_private_key)
+
+                cert.sign(tls_private_key, 'sha256')
+
+                cert_pem = crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
+
+                certificate_file.write(cert_pem)
+
+        if not self.path_exists(tls_dh_params_path):
+            if GENERATE_DH_PARAMS:
+                subprocess.check_call([
+                    "openssl", "dhparam",
+                    "-outform", "PEM",
+                    "-out", tls_dh_params_path,
+                    "2048"
+                ])
+            else:
+                with open(tls_dh_params_path, "w") as dh_params_file:
+                    dh_params_file.write(
+                        "2048-bit DH parameters taken from rfc3526\n"
+                        "-----BEGIN DH PARAMETERS-----\n"
+                        "MIIBCAKCAQEA///////////JD9qiIWjC"
+                        "NMTGYouA3BzRKQJOCIpnzHQCC76mOxOb\n"
+                        "IlFKCHmONATd75UZs806QxswKwpt8l8U"
+                        "N0/hNW1tUcJF5IW1dmJefsb0TELppjft\n"
+                        "awv/XLb0Brft7jhr+1qJn6WunyQRfEsf"
+                        "5kkoZlHs5Fs9wgB8uKFjvwWY2kg2HFXT\n"
+                        "mmkWP6j9JM9fg2VdI9yjrZYcYvNWIIVS"
+                        "u57VKQdwlpZtZww1Tkq8mATxdGwIyhgh\n"
+                        "fDKQXkYuNs474553LBgOhgObJ4Oi7Aei"
+                        "j7XFXfBvTFLJ3ivL9pVYFxg5lUl86pVq\n"
+                        "5RXSJhiY+gUQFXKOWoqsqmj/////////"
+                        "/wIBAg==\n"
+                        "-----END DH PARAMETERS-----\n"
+                    )

synapse/config/user_directory.py

@@ -40,5 +40,5 @@ class UserDirectoryConfig(Config):
         # on your database to tell it to rebuild the user_directory search indexes.
         #
         #user_directory:
-        #  search_all_users: false
+        #   search_all_users: false
         """

synapse/config/voip.py

@@ -27,24 +27,20 @@ class VoipConfig(Config):
 
     def default_config(self, **kwargs):
         return """\
-        ## TURN ##
+        ## Turn ##
 
         # The public URIs of the TURN server to give to clients
-        #
         #turn_uris: []
 
         # The shared secret used to compute passwords for the TURN server
-        #
         #turn_shared_secret: "YOUR_SHARED_SECRET"
 
         # The Username and password if the TURN server needs them and
         # does not use a token
-        #
         #turn_username: "TURNSERVER_USERNAME"
         #turn_password: "TURNSERVER_PASSWORD"
 
         # How long generated TURN credentials last
-        #
         turn_user_lifetime: "1h"
 
         # Whether guests should be allowed to use the TURN server.
@@ -52,6 +48,5 @@ class VoipConfig(Config):
         # However, it does introduce a slight security risk as it allows users to
         # connect to arbitrary endpoints without having first signed up for a
         # valid account (e.g. by passing a CAPTCHA).
-        #
         turn_allow_guests: True
         """

synapse/crypto/context_factory.py

@@ -1,5 +1,4 @@
 # Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,14 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import logging
 
 from zope.interface import implementer
 
 from OpenSSL import SSL, crypto
 from twisted.internet._sslverify import _defaultCurveName
-from twisted.internet.abstract import isIPAddress, isIPv6Address
 from twisted.internet.interfaces import IOpenSSLClientConnectionCreator
 from twisted.internet.ssl import CertificateOptions, ContextFactory
 from twisted.python.failure import Failure
@@ -45,12 +42,12 @@ class ServerContextFactory(ContextFactory):
             logger.exception("Failed to enable elliptic curve for TLS")
         context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)
         context.use_certificate_chain_file(config.tls_certificate_file)
-        context.use_privatekey(config.tls_private_key)
 
-        # https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
-        context.set_cipher_list(
-            "ECDH+AESGCM:ECDH+CHACHA20:ECDH+AES256:ECDH+AES128:!aNULL:!SHA1"
-        )
+        if not config.no_tls:
+            context.use_privatekey(config.tls_private_key)
+
+        context.load_tmp_dh(config.tls_dh_params_path)
+        context.set_cipher_list("!ADH:HIGH+kEDH:!AECDH:HIGH+kEECDH")
 
     def getContext(self):
         return self._context
@@ -99,15 +96,11 @@ class ClientTLSOptions(object):
 
     def __init__(self, hostname, ctx):
         self._ctx = ctx
-
-        if isIPAddress(hostname) or isIPv6Address(hostname):
-            self._hostnameBytes = hostname.encode('ascii')
-            self._sendSNI = False
-        else:
-            self._hostnameBytes = _idnaBytes(hostname)
-            self._sendSNI = True
-
-        ctx.set_info_callback(_tolerateErrors(self._identityVerifyingInfoCallback))
+        self._hostname = hostname
+        self._hostnameBytes = _idnaBytes(hostname)
+        ctx.set_info_callback(
+            _tolerateErrors(self._identityVerifyingInfoCallback)
+        )
 
     def clientConnectionForTLS(self, tlsProtocol):
         context = self._ctx
@@ -116,9 +109,7 @@ class ClientTLSOptions(object):
         return connection
 
     def _identityVerifyingInfoCallback(self, connection, where, ret):
-        # Literal IPv4 and IPv6 addresses are not permitted
-        # as host names according to the RFCs
-        if where & SSL.SSL_CB_HANDSHAKE_START and self._sendSNI:
+        if where & SSL.SSL_CB_HANDSHAKE_START:
             connection.set_tlsext_host_name(self._hostnameBytes)
 
 
@@ -128,8 +119,10 @@ class ClientTLSOptionsFactory(object):
 
     def __init__(self, config):
         # We don't use config options yet
-        self._options = CertificateOptions(verify=False)
+        pass
 
     def get_options(self, host):
-        # Use _makeContext so that we get a fresh OpenSSL CTX each time.
-        return ClientTLSOptions(host, self._options._makeContext())
+        return ClientTLSOptions(
+            host,
+            CertificateOptions(verify=False).getContext()
+        )

synapse/crypto/event_signing.py

@@ -23,14 +23,14 @@ from signedjson.sign import sign_json
 from unpaddedbase64 import decode_base64, encode_base64
 
 from synapse.api.errors import Codes, SynapseError
-from synapse.events.utils import prune_event, prune_event_dict
+from synapse.events.utils import prune_event
 
 logger = logging.getLogger(__name__)
 
 
 def check_event_content_hash(event, hash_algorithm=hashlib.sha256):
     """Check whether the hash for this PDU matches the contents"""
-    name, expected_hash = compute_content_hash(event.get_pdu_json(), hash_algorithm)
+    name, expected_hash = compute_content_hash(event, hash_algorithm)
     logger.debug("Expecting hash: %s", encode_base64(expected_hash))
 
     # some malformed events lack a 'hashes'. Protect against it being missing
@@ -59,70 +59,35 @@ def check_event_content_hash(event, hash_algorithm=hashlib.sha256):
     return message_hash_bytes == expected_hash
 
 
-def compute_content_hash(event_dict, hash_algorithm):
-    """Compute the content hash of an event, which is the hash of the
-    unredacted event.
+def compute_content_hash(event, hash_algorithm):
+    event_json = event.get_pdu_json()
+    event_json.pop("age_ts", None)
+    event_json.pop("unsigned", None)
+    event_json.pop("signatures", None)
+    event_json.pop("hashes", None)
+    event_json.pop("outlier", None)
+    event_json.pop("destinations", None)
 
-    Args:
-        event_dict (dict): The unredacted event as a dict
-        hash_algorithm: A hasher from `hashlib`, e.g. hashlib.sha256, to use
-            to hash the event
-
-    Returns:
-        tuple[str, bytes]: A tuple of the name of hash and the hash as raw
-        bytes.
-    """
-    event_dict = dict(event_dict)
-    event_dict.pop("age_ts", None)
-    event_dict.pop("unsigned", None)
-    event_dict.pop("signatures", None)
-    event_dict.pop("hashes", None)
-    event_dict.pop("outlier", None)
-    event_dict.pop("destinations", None)
-
-    event_json_bytes = encode_canonical_json(event_dict)
+    event_json_bytes = encode_canonical_json(event_json)
 
     hashed = hash_algorithm(event_json_bytes)
     return (hashed.name, hashed.digest())
 
 
 def compute_event_reference_hash(event, hash_algorithm=hashlib.sha256):
-    """Computes the event reference hash. This is the hash of the redacted
-    event.
-
-    Args:
-        event (FrozenEvent)
-        hash_algorithm: A hasher from `hashlib`, e.g. hashlib.sha256, to use
-            to hash the event
-
-    Returns:
-        tuple[str, bytes]: A tuple of the name of hash and the hash as raw
-        bytes.
-    """
     tmp_event = prune_event(event)
-    event_dict = tmp_event.get_pdu_json()
-    event_dict.pop("signatures", None)
-    event_dict.pop("age_ts", None)
-    event_dict.pop("unsigned", None)
-    event_json_bytes = encode_canonical_json(event_dict)
+    event_json = tmp_event.get_pdu_json()
+    event_json.pop("signatures", None)
+    event_json.pop("age_ts", None)
+    event_json.pop("unsigned", None)
+    event_json_bytes = encode_canonical_json(event_json)
     hashed = hash_algorithm(event_json_bytes)
     return (hashed.name, hashed.digest())
 
 
-def compute_event_signature(event_dict, signature_name, signing_key):
-    """Compute the signature of the event for the given name and key.
-
-    Args:
-        event_dict (dict): The event as a dict
-        signature_name (str): The name of the entity signing the event
-            (typically the server's hostname).
-        signing_key (syutil.crypto.SigningKey): The key to sign with
-
-    Returns:
-        dict[str, dict[str, str]]: Returns a dictionary in the same format of
-        an event's signatures field.
-    """
-    redact_json = prune_event_dict(event_dict)
+def compute_event_signature(event, signature_name, signing_key):
+    tmp_event = prune_event(event)
+    redact_json = tmp_event.get_pdu_json()
     redact_json.pop("age_ts", None)
     redact_json.pop("unsigned", None)
     logger.debug("Signing event: %s", encode_canonical_json(redact_json))
@@ -131,25 +96,25 @@ def compute_event_signature(event_dict, signature_name, signing_key):
     return redact_json["signatures"]
 
 
-def add_hashes_and_signatures(event_dict, signature_name, signing_key,
+def add_hashes_and_signatures(event, signature_name, signing_key,
                               hash_algorithm=hashlib.sha256):
-    """Add content hash and sign the event
+    # if hasattr(event, "old_state_events"):
+    #     state_json_bytes = encode_canonical_json(
+    #         [e.event_id for e in event.old_state_events.values()]
+    #     )
+    #     hashed = hash_algorithm(state_json_bytes)
+    #     event.state_hash = {
+    #         hashed.name: encode_base64(hashed.digest())
+    #     }
 
-    Args:
-        event_dict (dict): The event to add hashes to and sign
-        signature_name (str): The name of the entity signing the event
-            (typically the server's hostname).
-        signing_key (syutil.crypto.SigningKey): The key to sign with
-        hash_algorithm: A hasher from `hashlib`, e.g. hashlib.sha256, to use
-            to hash the event
-    """
+    name, digest = compute_content_hash(event, hash_algorithm=hash_algorithm)
 
-    name, digest = compute_content_hash(event_dict, hash_algorithm=hash_algorithm)
+    if not hasattr(event, "hashes"):
+        event.hashes = {}
+    event.hashes[name] = encode_base64(digest)
 
-    event_dict.setdefault("hashes", {})[name] = encode_base64(digest)
-
-    event_dict["signatures"] = compute_event_signature(
-        event_dict,
+    event.signatures = compute_event_signature(
+        event,
         signature_name=signature_name,
         signing_key=signing_key,
     )

synapse/crypto/keyclient.py

@@ -0,0 +1,149 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014-2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from six.moves import urllib
+
+from canonicaljson import json
+
+from twisted.internet import defer, reactor
+from twisted.internet.error import ConnectError
+from twisted.internet.protocol import Factory
+from twisted.names.error import DomainError
+from twisted.web.http import HTTPClient
+
+from synapse.http.endpoint import matrix_federation_endpoint
+from synapse.util import logcontext
+
+logger = logging.getLogger(__name__)
+
+KEY_API_V2 = "/_matrix/key/v2/server/%s"
+
+
+@defer.inlineCallbacks
+def fetch_server_key(server_name, tls_client_options_factory, key_id):
+    """Fetch the keys for a remote server."""
+
+    factory = SynapseKeyClientFactory()
+    factory.path = KEY_API_V2 % (urllib.parse.quote(key_id), )
+    factory.host = server_name
+    endpoint = matrix_federation_endpoint(
+        reactor, server_name, tls_client_options_factory, timeout=30
+    )
+
+    for i in range(5):
+        try:
+            with logcontext.PreserveLoggingContext():
+                protocol = yield endpoint.connect(factory)
+                server_response, server_certificate = yield protocol.remote_key
+                defer.returnValue((server_response, server_certificate))
+        except SynapseKeyClientError as e:
+            logger.warn("Error getting key for %r: %s", server_name, e)
+            if e.status.startswith(b"4"):
+                # Don't retry for 4xx responses.
+                raise IOError("Cannot get key for %r" % server_name)
+        except (ConnectError, DomainError) as e:
+            logger.warn("Error getting key for %r: %s", server_name, e)
+        except Exception:
+            logger.exception("Error getting key for %r", server_name)
+    raise IOError("Cannot get key for %r" % server_name)
+
+
+class SynapseKeyClientError(Exception):
+    """The key wasn't retrieved from the remote server."""
+    status = None
+    pass
+
+
+class SynapseKeyClientProtocol(HTTPClient):
+    """Low level HTTPS client which retrieves an application/json response from
+    the server and extracts the X.509 certificate for the remote peer from the
+    SSL connection."""
+
+    timeout = 30
+
+    def __init__(self):
+        self.remote_key = defer.Deferred()
+        self.host = None
+        self._peer = None
+
+    def connectionMade(self):
+        self._peer = self.transport.getPeer()
+        logger.debug("Connected to %s", self._peer)
+
+        if not isinstance(self.path, bytes):
+            self.path = self.path.encode('ascii')
+
+        if not isinstance(self.host, bytes):
+            self.host = self.host.encode('ascii')
+
+        self.sendCommand(b"GET", self.path)
+        if self.host:
+            self.sendHeader(b"Host", self.host)
+        self.endHeaders()
+        self.timer = reactor.callLater(
+            self.timeout,
+            self.on_timeout
+        )
+
+    def errback(self, error):
+        if not self.remote_key.called:
+            self.remote_key.errback(error)
+
+    def callback(self, result):
+        if not self.remote_key.called:
+            self.remote_key.callback(result)
+
+    def handleStatus(self, version, status, message):
+        if status != b"200":
+            # logger.info("Non-200 response from %s: %s %s",
+            #            self.transport.getHost(), status, message)
+            error = SynapseKeyClientError(
+                "Non-200 response %r from %r" % (status, self.host)
+            )
+            error.status = status
+            self.errback(error)
+            self.transport.abortConnection()
+
+    def handleResponse(self, response_body_bytes):
+        try:
+            json_response = json.loads(response_body_bytes)
+        except ValueError:
+            # logger.info("Invalid JSON response from %s",
+            #            self.transport.getHost())
+            self.transport.abortConnection()
+            return
+
+        certificate = self.transport.getPeerCertificate()
+        self.callback((json_response, certificate))
+        self.transport.abortConnection()
+        self.timer.cancel()
+
+    def on_timeout(self):
+        logger.debug(
+            "Timeout waiting for response from %s: %s",
+            self.host, self._peer,
+        )
+        self.errback(IOError("Timeout waiting for response"))
+        self.transport.abortConnection()
+
+
+class SynapseKeyClientFactory(Factory):
+    def protocol(self):
+        protocol = SynapseKeyClientProtocol()
+        protocol.path = self.path
+        protocol.host = self.host
+        return protocol

synapse/crypto/keyring.py

@@ -14,12 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import hashlib
 import logging
 from collections import namedtuple
 
-from six import raise_from
-from six.moves import urllib
-
 from signedjson.key import (
     decode_verify_key_bytes,
     encode_verify_key_base64,
@@ -32,16 +30,13 @@ from signedjson.sign import (
     signature_ids,
     verify_signed_json,
 )
-from unpaddedbase64 import decode_base64
+from unpaddedbase64 import decode_base64, encode_base64
 
+from OpenSSL import crypto
 from twisted.internet import defer
 
-from synapse.api.errors import (
-    Codes,
-    HttpResponseException,
-    RequestSendFailed,
-    SynapseError,
-)
+from synapse.api.errors import Codes, SynapseError
+from synapse.crypto.keyclient import fetch_server_key
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.logcontext import (
     LoggingContext,
@@ -50,7 +45,6 @@ from synapse.util.logcontext import (
     run_in_background,
 )
 from synapse.util.metrics import Measure
-from synapse.util.retryutils import NotRetryingDestination
 
 logger = logging.getLogger(__name__)
 
@@ -374,18 +368,13 @@ class Keyring(object):
                     server_name_and_key_ids, perspective_name, perspective_keys
                 )
                 defer.returnValue(result)
-            except KeyLookupError as e:
-                logger.warning(
-                    "Key lookup failed from %r: %s", perspective_name, e,
-                )
             except Exception as e:
                 logger.exception(
                     "Unable to get key from %r: %s %s",
                     perspective_name,
                     type(e).__name__, str(e),
                 )
-
-            defer.returnValue({})
+                defer.returnValue({})
 
         results = yield logcontext.make_deferred_yieldable(defer.gatherResults(
             [
@@ -433,30 +422,21 @@ class Keyring(object):
         # TODO(mark): Set the minimum_valid_until_ts to that needed by
         # the events being validated or the current time if validating
         # an incoming request.
-        try:
-            query_response = yield self.client.post_json(
-                destination=perspective_name,
-                path="/_matrix/key/v2/query",
-                data={
-                    u"server_keys": {
-                        server_name: {
-                            key_id: {
-                                u"minimum_valid_until_ts": 0
-                            } for key_id in key_ids
-                        }
-                        for server_name, key_ids in server_names_and_key_ids
+        query_response = yield self.client.post_json(
+            destination=perspective_name,
+            path="/_matrix/key/v2/query",
+            data={
+                u"server_keys": {
+                    server_name: {
+                        key_id: {
+                            u"minimum_valid_until_ts": 0
+                        } for key_id in key_ids
                     }
-                },
-                long_retries=True,
-            )
-        except (NotRetryingDestination, RequestSendFailed) as e:
-            raise_from(
-                KeyLookupError("Failed to connect to remote server"), e,
-            )
-        except HttpResponseException as e:
-            raise_from(
-                KeyLookupError("Remote server returned an error"), e,
-            )
+                    for server_name, key_ids in server_names_and_key_ids
+                }
+            },
+            long_retries=True,
+        )
 
         keys = {}
 
@@ -523,25 +503,31 @@ class Keyring(object):
             if requested_key_id in keys:
                 continue
 
-            try:
-                response = yield self.client.get_json(
-                    destination=server_name,
-                    path="/_matrix/key/v2/server/" + urllib.parse.quote(requested_key_id),
-                    ignore_backoff=True,
-                )
-            except (NotRetryingDestination, RequestSendFailed) as e:
-                raise_from(
-                    KeyLookupError("Failed to connect to remote server"), e,
-                )
-            except HttpResponseException as e:
-                raise_from(
-                    KeyLookupError("Remote server returned an error"), e,
-                )
+            (response, tls_certificate) = yield fetch_server_key(
+                server_name, self.hs.tls_client_options_factory, requested_key_id
+            )
 
             if (u"signatures" not in response
                     or server_name not in response[u"signatures"]):
                 raise KeyLookupError("Key response not signed by remote server")
 
+            if "tls_fingerprints" not in response:
+                raise KeyLookupError("Key response missing TLS fingerprints")
+
+            certificate_bytes = crypto.dump_certificate(
+                crypto.FILETYPE_ASN1, tls_certificate
+            )
+            sha256_fingerprint = hashlib.sha256(certificate_bytes).digest()
+            sha256_fingerprint_b64 = encode_base64(sha256_fingerprint)
+
+            response_sha256_fingerprints = set()
+            for fingerprint in response[u"tls_fingerprints"]:
+                if u"sha256" in fingerprint:
+                    response_sha256_fingerprints.add(fingerprint[u"sha256"])
+
+            if sha256_fingerprint_b64 not in response_sha256_fingerprints:
+                raise KeyLookupError("TLS certificate not allowed by fingerprints")
+
             response_keys = yield self.process_v2_response(
                 from_server=server_name,
                 requested_ids=[requested_key_id],
@@ -686,7 +672,7 @@ def _handle_key_deferred(verify_request):
     try:
         with PreserveLoggingContext():
             _, key_id, verify_key = yield verify_request.deferred
-    except (IOError, RequestSendFailed) as e:
+    except IOError as e:
         logger.warn(
             "Got IOError when downloading keys for %s: %s %s",
             server_name, type(e).__name__, str(e),

synapse/event_auth.py

@@ -20,25 +20,17 @@ from signedjson.key import decode_verify_key_bytes
 from signedjson.sign import SignatureVerifyException, verify_signed_json
 from unpaddedbase64 import decode_base64
 
-from synapse.api.constants import (
-    KNOWN_ROOM_VERSIONS,
-    EventFormatVersions,
-    EventTypes,
-    JoinRules,
-    Membership,
-    RoomVersions,
-)
+from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventTypes, JoinRules, Membership
 from synapse.api.errors import AuthError, EventSizeError, SynapseError
 from synapse.types import UserID, get_domain_from_id
 
 logger = logging.getLogger(__name__)
 
 
-def check(room_version, event, auth_events, do_sig_check=True, do_size_check=True):
+def check(event, auth_events, do_sig_check=True, do_size_check=True):
     """ Checks if this event is correctly authed.
 
     Args:
-        room_version (str): the version of the room
         event: the event being checked.
         auth_events (dict: event-key -> event): the existing room state.
 
@@ -56,6 +48,7 @@ def check(room_version, event, auth_events, do_sig_check=True, do_size_check=Tru
 
     if do_sig_check:
         sender_domain = get_domain_from_id(event.sender)
+        event_id_domain = get_domain_from_id(event.event_id)
 
         is_invite_via_3pid = (
             event.type == EventTypes.Member
@@ -72,13 +65,9 @@ def check(room_version, event, auth_events, do_sig_check=True, do_size_check=Tru
             if not is_invite_via_3pid:
                 raise AuthError(403, "Event not signed by sender's server")
 
-        if event.format_version in (EventFormatVersions.V1,):
-            # Only older room versions have event IDs to check.
-            event_id_domain = get_domain_from_id(event.event_id)
-
-            # Check the origin domain has signed the event
-            if not event.signatures.get(event_id_domain):
-                raise AuthError(403, "Event not signed by sending server")
+        # Check the event_id's domain has signed the event
+        if not event.signatures.get(event_id_domain):
+            raise AuthError(403, "Event not signed by sending server")
 
     if auth_events is None:
         # Oh, we don't know what the state of the room was, so we
@@ -178,7 +167,7 @@ def check(room_version, event, auth_events, do_sig_check=True, do_size_check=Tru
         _check_power_levels(event, auth_events)
 
     if event.type == EventTypes.Redaction:
-        check_redaction(room_version, event, auth_events)
+        check_redaction(event, auth_events)
 
     logger.debug("Allowing! %s", event)
 
@@ -432,7 +421,7 @@ def _can_send_event(event, auth_events):
     return True
 
 
-def check_redaction(room_version, event, auth_events):
+def check_redaction(event, auth_events):
     """Check whether the event sender is allowed to redact the target event.
 
     Returns:
@@ -452,16 +441,10 @@ def check_redaction(room_version, event, auth_events):
     if user_level >= redact_level:
         return False
 
-    if room_version in (RoomVersions.V1, RoomVersions.V2,):
-        redacter_domain = get_domain_from_id(event.event_id)
-        redactee_domain = get_domain_from_id(event.redacts)
-        if redacter_domain == redactee_domain:
-            return True
-    elif room_version == RoomVersions.V3:
-        event.internal_metadata.recheck_redaction = True
+    redacter_domain = get_domain_from_id(event.event_id)
+    redactee_domain = get_domain_from_id(event.redacts)
+    if redacter_domain == redactee_domain:
         return True
-    else:
-        raise RuntimeError("Unrecognized room version %r" % (room_version,))
 
     raise AuthError(
         403,

synapse/events/__init__.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,9 +18,6 @@ from distutils.util import strtobool
 
 import six
 
-from unpaddedbase64 import encode_base64
-
-from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventFormatVersions, RoomVersions
 from synapse.util.caches import intern_dict
 from synapse.util.frozenutils import freeze
 
@@ -45,13 +41,8 @@ class _EventInternalMetadata(object):
     def is_outlier(self):
         return getattr(self, "outlier", False)
 
-    def is_out_of_band_membership(self):
-        """Whether this is an out of band membership, like an invite or an invite
-        rejection. This is needed as those events are marked as outliers, but
-        they still need to be processed as if they're new events (e.g. updating
-        invite state in the database, relaying to clients, etc).
-        """
-        return getattr(self, "out_of_band_membership", False)
+    def is_invite_from_remote(self):
+        return getattr(self, "invite_from_remote", False)
 
     def get_send_on_behalf_of(self):
         """Whether this server should send the event on behalf of another server.
@@ -62,21 +53,6 @@ class _EventInternalMetadata(object):
         """
         return getattr(self, "send_on_behalf_of", None)
 
-    def need_to_check_redaction(self):
-        """Whether the redaction event needs to be rechecked when fetching
-        from the database.
-
-        Starting in room v3 redaction events are accepted up front, and later
-        checked to see if the redacter and redactee's domains match.
-
-        If the sender of the redaction event is allowed to redact any event
-        due to auth rules, then this will always return false.
-
-        Returns:
-            bool
-        """
-        return getattr(self, "recheck_redaction", False)
-
 
 def _event_dict_property(key):
     # We want to be able to use hasattr with the event dict properties.
@@ -203,8 +179,6 @@ class EventBase(object):
 
 
 class FrozenEvent(EventBase):
-    format_version = EventFormatVersions.V1  # All events of this type are V1
-
     def __init__(self, event_dict, internal_metadata_dict={}, rejected_reason=None):
         event_dict = dict(event_dict)
 
@@ -239,6 +213,16 @@ class FrozenEvent(EventBase):
             rejected_reason=rejected_reason,
         )
 
+    @staticmethod
+    def from_event(event):
+        e = FrozenEvent(
+            event.get_pdu_json()
+        )
+
+        e.internal_metadata = event.internal_metadata
+
+        return e
+
     def __str__(self):
         return self.__repr__()
 
@@ -248,127 +232,3 @@ class FrozenEvent(EventBase):
             self.get("type", None),
             self.get("state_key", None),
         )
-
-
-class FrozenEventV2(EventBase):
-    format_version = EventFormatVersions.V2  # All events of this type are V2
-
-    def __init__(self, event_dict, internal_metadata_dict={}, rejected_reason=None):
-        event_dict = dict(event_dict)
-
-        # Signatures is a dict of dicts, and this is faster than doing a
-        # copy.deepcopy
-        signatures = {
-            name: {sig_id: sig for sig_id, sig in sigs.items()}
-            for name, sigs in event_dict.pop("signatures", {}).items()
-        }
-
-        assert "event_id" not in event_dict
-
-        unsigned = dict(event_dict.pop("unsigned", {}))
-
-        # We intern these strings because they turn up a lot (especially when
-        # caching).
-        event_dict = intern_dict(event_dict)
-
-        if USE_FROZEN_DICTS:
-            frozen_dict = freeze(event_dict)
-        else:
-            frozen_dict = event_dict
-
-        self._event_id = None
-        self.type = event_dict["type"]
-        if "state_key" in event_dict:
-            self.state_key = event_dict["state_key"]
-
-        super(FrozenEventV2, self).__init__(
-            frozen_dict,
-            signatures=signatures,
-            unsigned=unsigned,
-            internal_metadata_dict=internal_metadata_dict,
-            rejected_reason=rejected_reason,
-        )
-
-    @property
-    def event_id(self):
-        # We have to import this here as otherwise we get an import loop which
-        # is hard to break.
-        from synapse.crypto.event_signing import compute_event_reference_hash
-
-        if self._event_id:
-            return self._event_id
-        self._event_id = "$" + encode_base64(compute_event_reference_hash(self)[1])
-        return self._event_id
-
-    def prev_event_ids(self):
-        """Returns the list of prev event IDs. The order matches the order
-        specified in the event, though there is no meaning to it.
-
-        Returns:
-            list[str]: The list of event IDs of this event's prev_events
-        """
-        return self.prev_events
-
-    def auth_event_ids(self):
-        """Returns the list of auth event IDs. The order matches the order
-        specified in the event, though there is no meaning to it.
-
-        Returns:
-            list[str]: The list of event IDs of this event's auth_events
-        """
-        return self.auth_events
-
-    def __str__(self):
-        return self.__repr__()
-
-    def __repr__(self):
-        return "<FrozenEventV2 event_id='%s', type='%s', state_key='%s'>" % (
-            self.event_id,
-            self.get("type", None),
-            self.get("state_key", None),
-        )
-
-
-def room_version_to_event_format(room_version):
-    """Converts a room version string to the event format
-
-    Args:
-        room_version (str)
-
-    Returns:
-        int
-    """
-    if room_version not in KNOWN_ROOM_VERSIONS:
-        # We should have already checked version, so this should not happen
-        raise RuntimeError("Unrecognized room version %s" % (room_version,))
-
-    if room_version in (
-        RoomVersions.V1, RoomVersions.V2, RoomVersions.STATE_V2_TEST,
-    ):
-        return EventFormatVersions.V1
-    elif room_version in (RoomVersions.V3,):
-        return EventFormatVersions.V2
-    else:
-        raise RuntimeError("Unrecognized room version %s" % (room_version,))
-
-
-def event_type_from_format_version(format_version):
-    """Returns the python type to use to construct an Event object for the
-    given event format version.
-
-    Args:
-        format_version (int): The event format version
-
-    Returns:
-        type: A type that can be initialized as per the initializer of
-        `FrozenEvent`
-    """
-
-    if format_version == EventFormatVersions.V1:
-        return FrozenEvent
-    elif format_version == EventFormatVersions.V2:
-        return FrozenEventV2
-    else:
-        raise Exception(
-            "No event format %r" % (format_version,)
-        )

synapse/events/builder.py

@@ -13,270 +13,63 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import attr
+import copy
 
-from twisted.internet import defer
-
-from synapse.api.constants import (
-    KNOWN_EVENT_FORMAT_VERSIONS,
-    KNOWN_ROOM_VERSIONS,
-    MAX_DEPTH,
-    EventFormatVersions,
-)
-from synapse.crypto.event_signing import add_hashes_and_signatures
 from synapse.types import EventID
 from synapse.util.stringutils import random_string
 
-from . import (
-    _EventInternalMetadata,
-    event_type_from_format_version,
-    room_version_to_event_format,
-)
+from . import EventBase, FrozenEvent, _event_dict_property
 
 
-@attr.s(slots=True, cmp=False, frozen=True)
-class EventBuilder(object):
-    """A format independent event builder used to build up the event content
-    before signing the event.
+class EventBuilder(EventBase):
+    def __init__(self, key_values={}, internal_metadata_dict={}):
+        signatures = copy.deepcopy(key_values.pop("signatures", {}))
+        unsigned = copy.deepcopy(key_values.pop("unsigned", {}))
 
-    (Note that while objects of this class are frozen, the
-    content/unsigned/internal_metadata fields are still mutable)
-
-    Attributes:
-        format_version (int): Event format version
-        room_id (str)
-        type (str)
-        sender (str)
-        content (dict)
-        unsigned (dict)
-        internal_metadata (_EventInternalMetadata)
-
-        _state (StateHandler)
-        _auth (synapse.api.Auth)
-        _store (DataStore)
-        _clock (Clock)
-        _hostname (str): The hostname of the server creating the event
-        _signing_key: The signing key to use to sign the event as the server
-    """
-
-    _state = attr.ib()
-    _auth = attr.ib()
-    _store = attr.ib()
-    _clock = attr.ib()
-    _hostname = attr.ib()
-    _signing_key = attr.ib()
-
-    format_version = attr.ib()
-
-    room_id = attr.ib()
-    type = attr.ib()
-    sender = attr.ib()
-
-    content = attr.ib(default=attr.Factory(dict))
-    unsigned = attr.ib(default=attr.Factory(dict))
-
-    # These only exist on a subset of events, so they raise AttributeError if
-    # someone tries to get them when they don't exist.
-    _state_key = attr.ib(default=None)
-    _redacts = attr.ib(default=None)
-
-    internal_metadata = attr.ib(default=attr.Factory(lambda: _EventInternalMetadata({})))
-
-    @property
-    def state_key(self):
-        if self._state_key is not None:
-            return self._state_key
-
-        raise AttributeError("state_key")
-
-    def is_state(self):
-        return self._state_key is not None
-
-    @defer.inlineCallbacks
-    def build(self, prev_event_ids):
-        """Transform into a fully signed and hashed event
-
-        Args:
-            prev_event_ids (list[str]): The event IDs to use as the prev events
-
-        Returns:
-            Deferred[FrozenEvent]
-        """
-
-        state_ids = yield self._state.get_current_state_ids(
-            self.room_id, prev_event_ids,
-        )
-        auth_ids = yield self._auth.compute_auth_events(
-            self, state_ids,
+        super(EventBuilder, self).__init__(
+            key_values,
+            signatures=signatures,
+            unsigned=unsigned,
+            internal_metadata_dict=internal_metadata_dict,
         )
 
-        if self.format_version == EventFormatVersions.V1:
-            auth_events = yield self._store.add_event_hashes(auth_ids)
-            prev_events = yield self._store.add_event_hashes(prev_event_ids)
-        else:
-            auth_events = auth_ids
-            prev_events = prev_event_ids
+    event_id = _event_dict_property("event_id")
+    state_key = _event_dict_property("state_key")
+    type = _event_dict_property("type")
 
-        old_depth = yield self._store.get_max_depth_of(
-            prev_event_ids,
-        )
-        depth = old_depth + 1
-
-        # we cap depth of generated events, to ensure that they are not
-        # rejected by other servers (and so that they can be persisted in
-        # the db)
-        depth = min(depth, MAX_DEPTH)
-
-        event_dict = {
-            "auth_events": auth_events,
-            "prev_events": prev_events,
-            "type": self.type,
-            "room_id": self.room_id,
-            "sender": self.sender,
-            "content": self.content,
-            "unsigned": self.unsigned,
-            "depth": depth,
-            "prev_state": [],
-        }
-
-        if self.is_state():
-            event_dict["state_key"] = self._state_key
-
-        if self._redacts is not None:
-            event_dict["redacts"] = self._redacts
-
-        defer.returnValue(
-            create_local_event_from_event_dict(
-                clock=self._clock,
-                hostname=self._hostname,
-                signing_key=self._signing_key,
-                format_version=self.format_version,
-                event_dict=event_dict,
-                internal_metadata_dict=self.internal_metadata.get_dict(),
-            )
-        )
+    def build(self):
+        return FrozenEvent.from_event(self)
 
 
 class EventBuilderFactory(object):
-    def __init__(self, hs):
-        self.clock = hs.get_clock()
-        self.hostname = hs.hostname
-        self.signing_key = hs.config.signing_key[0]
+    def __init__(self, clock, hostname):
+        self.clock = clock
+        self.hostname = hostname
 
-        self.store = hs.get_datastore()
-        self.state = hs.get_state_handler()
-        self.auth = hs.get_auth()
+        self.event_id_count = 0
 
-    def new(self, room_version, key_values):
-        """Generate an event builder appropriate for the given room version
+    def create_event_id(self):
+        i = str(self.event_id_count)
+        self.event_id_count += 1
 
-        Args:
-            room_version (str): Version of the room that we're creating an
-                event builder for
-            key_values (dict): Fields used as the basis of the new event
+        local_part = str(int(self.clock.time())) + i + random_string(5)
 
-        Returns:
-            EventBuilder
-        """
+        e_id = EventID(local_part, self.hostname)
 
-        # There's currently only the one event version defined
-        if room_version not in KNOWN_ROOM_VERSIONS:
-            raise Exception(
-                "No event format defined for version %r" % (room_version,)
-            )
+        return e_id.to_string()
 
-        return EventBuilder(
-            store=self.store,
-            state=self.state,
-            auth=self.auth,
-            clock=self.clock,
-            hostname=self.hostname,
-            signing_key=self.signing_key,
-            format_version=room_version_to_event_format(room_version),
-            type=key_values["type"],
-            state_key=key_values.get("state_key"),
-            room_id=key_values["room_id"],
-            sender=key_values["sender"],
-            content=key_values.get("content", {}),
-            unsigned=key_values.get("unsigned", {}),
-            redacts=key_values.get("redacts", None),
-        )
+    def new(self, key_values={}):
+        key_values["event_id"] = self.create_event_id()
 
+        time_now = int(self.clock.time_msec())
 
-def create_local_event_from_event_dict(clock, hostname, signing_key,
-                                       format_version, event_dict,
-                                       internal_metadata_dict=None):
-    """Takes a fully formed event dict, ensuring that fields like `origin`
-    and `origin_server_ts` have correct values for a locally produced event,
-    then signs and hashes it.
+        key_values.setdefault("origin", self.hostname)
+        key_values.setdefault("origin_server_ts", time_now)
 
-    Args:
-        clock (Clock)
-        hostname (str)
-        signing_key
-        format_version (int)
-        event_dict (dict)
-        internal_metadata_dict (dict|None)
+        key_values.setdefault("unsigned", {})
+        age = key_values["unsigned"].pop("age", 0)
+        key_values["unsigned"].setdefault("age_ts", time_now - age)
 
-    Returns:
-        FrozenEvent
-    """
+        key_values["signatures"] = {}
 
-    # There's currently only the one event version defined
-    if format_version not in KNOWN_EVENT_FORMAT_VERSIONS:
-        raise Exception(
-            "No event format defined for version %r" % (format_version,)
-        )
-
-    if internal_metadata_dict is None:
-        internal_metadata_dict = {}
-
-    time_now = int(clock.time_msec())
-
-    if format_version == EventFormatVersions.V1:
-        event_dict["event_id"] = _create_event_id(clock, hostname)
-
-    event_dict["origin"] = hostname
-    event_dict["origin_server_ts"] = time_now
-
-    event_dict.setdefault("unsigned", {})
-    age = event_dict["unsigned"].pop("age", 0)
-    event_dict["unsigned"].setdefault("age_ts", time_now - age)
-
-    event_dict.setdefault("signatures", {})
-
-    add_hashes_and_signatures(
-        event_dict,
-        hostname,
-        signing_key,
-    )
-    return event_type_from_format_version(format_version)(
-        event_dict, internal_metadata_dict=internal_metadata_dict,
-    )
-
-
-# A counter used when generating new event IDs
-_event_id_counter = 0
-
-
-def _create_event_id(clock, hostname):
-    """Create a new event ID
-
-    Args:
-        clock (Clock)
-        hostname (str): The server name for the event ID
-
-    Returns:
-        str
-    """
-
-    global _event_id_counter
-
-    i = str(_event_id_counter)
-    _event_id_counter += 1
-
-    local_part = str(int(clock.time())) + i + random_string(5)
-
-    e_id = EventID(local_part, hostname)
-
-    return e_id.to_string()
+        return EventBuilder(key_values=key_values,)

synapse/events/utils.py

@@ -38,31 +38,8 @@ def prune_event(event):
     This is used when we "redact" an event. We want to remove all fields that
     the user has specified, but we do want to keep necessary information like
     type, state_key etc.
-
-    Args:
-        event (FrozenEvent)
-
-    Returns:
-        FrozenEvent
-    """
-    pruned_event_dict = prune_event_dict(event.get_dict())
-
-    from . import event_type_from_format_version
-    return event_type_from_format_version(event.format_version)(
-        pruned_event_dict, event.internal_metadata.get_dict()
-    )
-
-
-def prune_event_dict(event_dict):
-    """Redacts the event_dict in the same way as `prune_event`, except it
-    operates on dicts rather than event objects
-
-    Args:
-        event_dict (dict)
-
-    Returns:
-        dict: A copy of the pruned event dict
     """
+    event_type = event.type
 
     allowed_keys = [
         "event_id",
@@ -82,13 +59,13 @@ def prune_event_dict(event_dict):
         "membership",
     ]
 
-    event_type = event_dict["type"]
+    event_dict = event.get_dict()
 
     new_content = {}
 
     def add_fields(*fields):
         for field in fields:
-            if field in event_dict["content"]:
+            if field in event.content:
                 new_content[field] = event_dict["content"][field]
 
     if event_type == EventTypes.Member:
@@ -121,17 +98,17 @@ def prune_event_dict(event_dict):
 
     allowed_fields["content"] = new_content
 
-    unsigned = {}
-    allowed_fields["unsigned"] = unsigned
+    allowed_fields["unsigned"] = {}
 
-    event_unsigned = event_dict.get("unsigned", {})
+    if "age_ts" in event.unsigned:
+        allowed_fields["unsigned"]["age_ts"] = event.unsigned["age_ts"]
+    if "replaces_state" in event.unsigned:
+        allowed_fields["unsigned"]["replaces_state"] = event.unsigned["replaces_state"]
 
-    if "age_ts" in event_unsigned:
-        unsigned["age_ts"] = event_unsigned["age_ts"]
-    if "replaces_state" in event_unsigned:
-        unsigned["replaces_state"] = event_unsigned["replaces_state"]
-
-    return allowed_fields
+    return type(event)(
+        allowed_fields,
+        internal_metadata_dict=event.internal_metadata.get_dict()
+    )
 
 
 def _copy_field(src, dst, field):
@@ -267,7 +244,6 @@ def serialize_event(e, time_now_ms, as_client_event=True,
     Returns:
         dict
     """
-
     # FIXME(erikj): To handle the case of presence events and the like
     if not isinstance(e, EventBase):
         return e
@@ -277,8 +253,6 @@ def serialize_event(e, time_now_ms, as_client_event=True,
     # Should this strip out None's?
     d = {k: v for k, v in e.get_dict().items()}
 
-    d["event_id"] = e.event_id
-
     if "age_ts" in d["unsigned"]:
         d["unsigned"]["age"] = time_now_ms - d["unsigned"]["age_ts"]
         del d["unsigned"]["age_ts"]

synapse/events/validator.py

@@ -15,29 +15,23 @@
 
 from six import string_types
 
-from synapse.api.constants import EventFormatVersions, EventTypes, Membership
+from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import SynapseError
 from synapse.types import EventID, RoomID, UserID
 
 
 class EventValidator(object):
-    def validate_new(self, event):
-        """Validates the event has roughly the right format
 
-        Args:
-            event (FrozenEvent)
-        """
-        self.validate_builder(event)
-
-        if event.format_version == EventFormatVersions.V1:
-            EventID.from_string(event.event_id)
+    def validate(self, event):
+        EventID.from_string(event.event_id)
+        RoomID.from_string(event.room_id)
 
         required = [
-            "auth_events",
+            # "auth_events",
             "content",
-            "hashes",
+            # "hashes",
             "origin",
-            "prev_events",
+            # "prev_events",
             "sender",
             "type",
         ]
@@ -47,25 +41,8 @@ class EventValidator(object):
                 raise SynapseError(400, "Event does not have key %s" % (k,))
 
         # Check that the following keys have string values
-        event_strings = [
-            "origin",
-        ]
-
-        for s in event_strings:
-            if not isinstance(getattr(event, s), string_types):
-                raise SynapseError(400, "'%s' not a string type" % (s,))
-
-    def validate_builder(self, event):
-        """Validates that the builder/event has roughly the right format. Only
-        checks values that we expect a proto event to have, rather than all the
-        fields an event would have
-
-        Args:
-            event (EventBuilder|FrozenEvent)
-        """
-
         strings = [
-            "room_id",
+            "origin",
             "sender",
             "type",
         ]
@@ -77,7 +54,22 @@ class EventValidator(object):
             if not isinstance(getattr(event, s), string_types):
                 raise SynapseError(400, "Not '%s' a string type" % (s,))
 
-        RoomID.from_string(event.room_id)
+        if event.type == EventTypes.Member:
+            if "membership" not in event.content:
+                raise SynapseError(400, "Content has not membership key")
+
+            if event.content["membership"] not in Membership.LIST:
+                raise SynapseError(400, "Invalid membership key")
+
+        # Check that the following keys have dictionary values
+        # TODO
+
+        # Check that the following keys have the correct format for DAGs
+        # TODO
+
+    def validate_new(self, event):
+        self.validate(event)
+
         UserID.from_string(event.sender)
 
         if event.type == EventTypes.Message:
@@ -94,16 +86,9 @@ class EventValidator(object):
         elif event.type == EventTypes.Name:
             self._ensure_strings(event.content, ["name"])
 
-        elif event.type == EventTypes.Member:
-            if "membership" not in event.content:
-                raise SynapseError(400, "Content has not membership key")
-
-            if event.content["membership"] not in Membership.LIST:
-                raise SynapseError(400, "Invalid membership key")
-
     def _ensure_strings(self, d, keys):
         for s in keys:
             if s not in d:
                 raise SynapseError(400, "'%s' not in content" % (s,))
             if not isinstance(d[s], string_types):
-                raise SynapseError(400, "'%s' not a string type" % (s,))
+                raise SynapseError(400, "Not '%s' a string type" % (s,))

synapse/federation/federation_base.py

@@ -20,10 +20,10 @@ import six
 from twisted.internet import defer
 from twisted.internet.defer import DeferredList
 
-from synapse.api.constants import MAX_DEPTH, EventTypes, Membership, RoomVersions
+from synapse.api.constants import MAX_DEPTH, EventTypes, Membership
 from synapse.api.errors import Codes, SynapseError
 from synapse.crypto.event_signing import check_event_content_hash
-from synapse.events import event_type_from_format_version
+from synapse.events import FrozenEvent
 from synapse.events.utils import prune_event
 from synapse.http.servlet import assert_params_in_dict
 from synapse.types import get_domain_from_id
@@ -43,8 +43,8 @@ class FederationBase(object):
         self._clock = hs.get_clock()
 
     @defer.inlineCallbacks
-    def _check_sigs_and_hash_and_fetch(self, origin, pdus, room_version,
-                                       outlier=False, include_none=False):
+    def _check_sigs_and_hash_and_fetch(self, origin, pdus, outlier=False,
+                                       include_none=False):
         """Takes a list of PDUs and checks the signatures and hashs of each
         one. If a PDU fails its signature check then we check if we have it in
         the database and if not then request if from the originating server of
@@ -56,17 +56,13 @@ class FederationBase(object):
         a new list.
 
         Args:
-            origin (str)
             pdu (list)
-            room_version (str)
-            outlier (bool): Whether the events are outliers or not
-            include_none (str): Whether to include None in the returned list
-                for events that have failed their checks
+            outlier (bool)
 
         Returns:
             Deferred : A list of PDUs that have valid signatures and hashes.
         """
-        deferreds = self._check_sigs_and_hashes(room_version, pdus)
+        deferreds = self._check_sigs_and_hashes(pdus)
 
         @defer.inlineCallbacks
         def handle_check_result(pdu, deferred):
@@ -88,7 +84,6 @@ class FederationBase(object):
                     res = yield self.get_pdu(
                         destinations=[pdu.origin],
                         event_id=pdu.event_id,
-                        room_version=room_version,
                         outlier=outlier,
                         timeout=10000,
                     )
@@ -121,17 +116,16 @@ class FederationBase(object):
         else:
             defer.returnValue([p for p in valid_pdus if p])
 
-    def _check_sigs_and_hash(self, room_version, pdu):
+    def _check_sigs_and_hash(self, pdu):
         return logcontext.make_deferred_yieldable(
-            self._check_sigs_and_hashes(room_version, [pdu])[0],
+            self._check_sigs_and_hashes([pdu])[0],
         )
 
-    def _check_sigs_and_hashes(self, room_version, pdus):
+    def _check_sigs_and_hashes(self, pdus):
         """Checks that each of the received events is correctly signed by the
         sending server.
 
         Args:
-            room_version (str): The room version of the PDUs
             pdus (list[FrozenEvent]): the events to be checked
 
         Returns:
@@ -142,7 +136,7 @@ class FederationBase(object):
               * throws a SynapseError if the signature check failed.
             The deferreds run their callbacks in the sentinel logcontext.
         """
-        deferreds = _check_sigs_on_pdus(self.keyring, room_version, pdus)
+        deferreds = _check_sigs_on_pdus(self.keyring, pdus)
 
         ctx = logcontext.LoggingContext.current_context()
 
@@ -204,17 +198,16 @@ class FederationBase(object):
 
 
 class PduToCheckSig(namedtuple("PduToCheckSig", [
-    "pdu", "redacted_pdu_json", "sender_domain", "deferreds",
+    "pdu", "redacted_pdu_json", "event_id_domain", "sender_domain", "deferreds",
 ])):
     pass
 
 
-def _check_sigs_on_pdus(keyring, room_version, pdus):
+def _check_sigs_on_pdus(keyring, pdus):
     """Check that the given events are correctly signed
 
     Args:
         keyring (synapse.crypto.Keyring): keyring object to do the checks
-        room_version (str): the room version of the PDUs
         pdus (Collection[EventBase]): the events to be checked
 
     Returns:
@@ -227,7 +220,9 @@ def _check_sigs_on_pdus(keyring, room_version, pdus):
 
     # we want to check that the event is signed by:
     #
-    # (a) the sender's server
+    # (a) the server which created the event_id
+    #
+    # (b) the sender's server.
     #
     #     - except in the case of invites created from a 3pid invite, which are exempt
     #     from this check, because the sender has to match that of the original 3pid
@@ -241,26 +236,34 @@ def _check_sigs_on_pdus(keyring, room_version, pdus):
     #     and signatures are *supposed* to be valid whether or not an event has been
     #     redacted. But this isn't the worst of the ways that 3pid invites are broken.
     #
-    # (b) for V1 and V2 rooms, the server which created the event_id
-    #
     # let's start by getting the domain for each pdu, and flattening the event back
     # to JSON.
-
     pdus_to_check = [
         PduToCheckSig(
             pdu=p,
             redacted_pdu_json=prune_event(p).get_pdu_json(),
+            event_id_domain=get_domain_from_id(p.event_id),
             sender_domain=get_domain_from_id(p.sender),
             deferreds=[],
         )
         for p in pdus
     ]
 
-    # First we check that the sender event is signed by the sender's domain
-    # (except if its a 3pid invite, in which case it may be sent by any server)
+    # first make sure that the event is signed by the event_id's domain
+    deferreds = keyring.verify_json_objects_for_server([
+        (p.event_id_domain, p.redacted_pdu_json)
+        for p in pdus_to_check
+    ])
+
+    for p, d in zip(pdus_to_check, deferreds):
+        p.deferreds.append(d)
+
+    # now let's look for events where the sender's domain is different to the
+    # event id's domain (normally only the case for joins/leaves), and add additional
+    # checks.
     pdus_to_check_sender = [
         p for p in pdus_to_check
-        if not _is_invite_via_3pid(p.pdu)
+        if p.sender_domain != p.event_id_domain and not _is_invite_via_3pid(p.pdu)
     ]
 
     more_deferreds = keyring.verify_json_objects_for_server([
@@ -271,43 +274,19 @@ def _check_sigs_on_pdus(keyring, room_version, pdus):
     for p, d in zip(pdus_to_check_sender, more_deferreds):
         p.deferreds.append(d)
 
-    # now let's look for events where the sender's domain is different to the
-    # event id's domain (normally only the case for joins/leaves), and add additional
-    # checks. Only do this if the room version has a concept of event ID domain
-    if room_version in (
-        RoomVersions.V1, RoomVersions.V2, RoomVersions.STATE_V2_TEST,
-    ):
-        pdus_to_check_event_id = [
-            p for p in pdus_to_check
-            if p.sender_domain != get_domain_from_id(p.pdu.event_id)
-        ]
-
-        more_deferreds = keyring.verify_json_objects_for_server([
-            (get_domain_from_id(p.pdu.event_id), p.redacted_pdu_json)
-            for p in pdus_to_check_event_id
-        ])
-
-        for p, d in zip(pdus_to_check_event_id, more_deferreds):
-            p.deferreds.append(d)
-    elif room_version in (RoomVersions.V3,):
-        pass  # No further checks needed, as event IDs are hashes here
-    else:
-        raise RuntimeError("Unrecognized room version %s" % (room_version,))
-
     # replace lists of deferreds with single Deferreds
     return [_flatten_deferred_list(p.deferreds) for p in pdus_to_check]
 
 
 def _flatten_deferred_list(deferreds):
-    """Given a list of deferreds, either return the single deferred,
-    combine into a DeferredList, or return an already resolved deferred.
+    """Given a list of one or more deferreds, either return the single deferred, or
+    combine into a DeferredList.
     """
     if len(deferreds) > 1:
         return DeferredList(deferreds, fireOnOneErrback=True, consumeErrors=True)
-    elif len(deferreds) == 1:
-        return deferreds[0]
     else:
-        return defer.succeed(None)
+        assert len(deferreds) == 1
+        return deferreds[0]
 
 
 def _is_invite_via_3pid(event):
@@ -318,12 +297,11 @@ def _is_invite_via_3pid(event):
     )
 
 
-def event_from_pdu_json(pdu_json, event_format_version, outlier=False):
+def event_from_pdu_json(pdu_json, outlier=False):
     """Construct a FrozenEvent from an event json received over federation
 
     Args:
         pdu_json (object): pdu as received over federation
-        event_format_version (int): The event format version
         outlier (bool): True to mark this event as an outlier
 
     Returns:
@@ -335,7 +313,7 @@ def event_from_pdu_json(pdu_json, event_format_version, outlier=False):
     """
     # we could probably enforce a bunch of other fields here (room_id, sender,
     # origin, etc etc)
-    assert_params_in_dict(pdu_json, ('type', 'depth'))
+    assert_params_in_dict(pdu_json, ('event_id', 'type', 'depth'))
 
     depth = pdu_json['depth']
     if not isinstance(depth, six.integer_types):
@@ -347,8 +325,8 @@ def event_from_pdu_json(pdu_json, event_format_version, outlier=False):
     elif depth > MAX_DEPTH:
         raise SynapseError(400, "Depth too large", Codes.BAD_JSON)
 
-    event = event_type_from_format_version(event_format_version)(
-        pdu_json,
+    event = FrozenEvent(
+        pdu_json
     )
 
     event.internal_metadata.outlier = outlier

synapse/federation/federation_client.py

@@ -25,20 +25,14 @@ from prometheus_client import Counter
 
 from twisted.internet import defer
 
-from synapse.api.constants import (
-    KNOWN_ROOM_VERSIONS,
-    EventTypes,
-    Membership,
-    RoomVersions,
-)
+from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventTypes, Membership
 from synapse.api.errors import (
     CodeMessageException,
-    Codes,
     FederationDeniedError,
     HttpResponseException,
     SynapseError,
 )
-from synapse.events import builder, room_version_to_event_format
+from synapse.events import builder
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -72,9 +66,6 @@ class FederationClient(FederationBase):
         self.state = hs.get_state_handler()
         self.transport_layer = hs.get_federation_transport_client()
 
-        self.hostname = hs.hostname
-        self.signing_key = hs.config.signing_key[0]
-
         self._get_pdu_cache = ExpiringCache(
             cache_name="get_pdu_cache",
             clock=self._clock,
@@ -171,13 +162,13 @@ class FederationClient(FederationBase):
 
     @defer.inlineCallbacks
     @log_function
-    def backfill(self, dest, room_id, limit, extremities):
+    def backfill(self, dest, context, limit, extremities):
         """Requests some more historic PDUs for the given context from the
         given destination server.
 
         Args:
             dest (str): The remote home server to ask.
-            room_id (str): The room_id to backfill.
+            context (str): The context to backfill.
             limit (int): The maximum number of PDUs to return.
             extremities (list): List of PDU id and origins of the first pdus
                 we have seen from the context
@@ -192,21 +183,18 @@ class FederationClient(FederationBase):
             return
 
         transaction_data = yield self.transport_layer.backfill(
-            dest, room_id, extremities, limit)
+            dest, context, extremities, limit)
 
         logger.debug("backfill transaction_data=%s", repr(transaction_data))
 
-        room_version = yield self.store.get_room_version(room_id)
-        format_ver = room_version_to_event_format(room_version)
-
         pdus = [
-            event_from_pdu_json(p, format_ver, outlier=False)
+            event_from_pdu_json(p, outlier=False)
             for p in transaction_data["pdus"]
         ]
 
         # FIXME: We should handle signature failures more gracefully.
         pdus[:] = yield logcontext.make_deferred_yieldable(defer.gatherResults(
-            self._check_sigs_and_hashes(room_version, pdus),
+            self._check_sigs_and_hashes(pdus),
             consumeErrors=True,
         ).addErrback(unwrapFirstError))
 
@@ -214,8 +202,7 @@ class FederationClient(FederationBase):
 
     @defer.inlineCallbacks
     @log_function
-    def get_pdu(self, destinations, event_id, room_version, outlier=False,
-                timeout=None):
+    def get_pdu(self, destinations, event_id, outlier=False, timeout=None):
         """Requests the PDU with given origin and ID from the remote home
         servers.
 
@@ -225,7 +212,6 @@ class FederationClient(FederationBase):
         Args:
             destinations (list): Which home servers to query
             event_id (str): event to fetch
-            room_version (str): version of the room
             outlier (bool): Indicates whether the PDU is an `outlier`, i.e. if
                 it's from an arbitary point in the context as opposed to part
                 of the current block of PDUs. Defaults to `False`
@@ -244,8 +230,6 @@ class FederationClient(FederationBase):
 
         pdu_attempts = self.pdu_destination_tried.setdefault(event_id, {})
 
-        format_ver = room_version_to_event_format(room_version)
-
         signed_pdu = None
         for destination in destinations:
             now = self._clock.time_msec()
@@ -261,7 +245,7 @@ class FederationClient(FederationBase):
                 logger.debug("transaction_data %r", transaction_data)
 
                 pdu_list = [
-                    event_from_pdu_json(p, format_ver, outlier=outlier)
+                    event_from_pdu_json(p, outlier=outlier)
                     for p in transaction_data["pdus"]
                 ]
 
@@ -269,7 +253,7 @@ class FederationClient(FederationBase):
                     pdu = pdu_list[0]
 
                     # Check signatures are correct.
-                    signed_pdu = yield self._check_sigs_and_hash(room_version, pdu)
+                    signed_pdu = yield self._check_sigs_and_hash(pdu)
 
                     break
 
@@ -355,16 +339,12 @@ class FederationClient(FederationBase):
             destination, room_id, event_id=event_id,
         )
 
-        room_version = yield self.store.get_room_version(room_id)
-        format_ver = room_version_to_event_format(room_version)
-
         pdus = [
-            event_from_pdu_json(p, format_ver, outlier=True)
-            for p in result["pdus"]
+            event_from_pdu_json(p, outlier=True) for p in result["pdus"]
         ]
 
         auth_chain = [
-            event_from_pdu_json(p, format_ver, outlier=True)
+            event_from_pdu_json(p, outlier=True)
             for p in result.get("auth_chain", [])
         ]
 
@@ -375,8 +355,7 @@ class FederationClient(FederationBase):
         signed_pdus = yield self._check_sigs_and_hash_and_fetch(
             destination,
             [p for p in pdus if p.event_id not in seen_events],
-            outlier=True,
-            room_version=room_version,
+            outlier=True
         )
         signed_pdus.extend(
             seen_events[p.event_id] for p in pdus if p.event_id in seen_events
@@ -385,8 +364,7 @@ class FederationClient(FederationBase):
         signed_auth = yield self._check_sigs_and_hash_and_fetch(
             destination,
             [p for p in auth_chain if p.event_id not in seen_events],
-            outlier=True,
-            room_version=room_version,
+            outlier=True
         )
         signed_auth.extend(
             seen_events[p.event_id] for p in auth_chain if p.event_id in seen_events
@@ -433,8 +411,6 @@ class FederationClient(FederationBase):
             random.shuffle(srvs)
             return srvs
 
-        room_version = yield self.store.get_room_version(room_id)
-
         batch_size = 20
         missing_events = list(missing_events)
         for i in range(0, len(missing_events), batch_size):
@@ -445,7 +421,6 @@ class FederationClient(FederationBase):
                     self.get_pdu,
                     destinations=random_server_list(),
                     event_id=e_id,
-                    room_version=room_version,
                 )
                 for e_id in batch
             ]
@@ -470,17 +445,13 @@ class FederationClient(FederationBase):
             destination, room_id, event_id,
         )
 
-        room_version = yield self.store.get_room_version(room_id)
-        format_ver = room_version_to_event_format(room_version)
-
         auth_chain = [
-            event_from_pdu_json(p, format_ver, outlier=True)
+            event_from_pdu_json(p, outlier=True)
             for p in res["auth_chain"]
         ]
 
         signed_auth = yield self._check_sigs_and_hash_and_fetch(
-            destination, auth_chain,
-            outlier=True, room_version=room_version,
+            destination, auth_chain, outlier=True
         )
 
         signed_auth.sort(key=lambda e: e.depth)
@@ -551,8 +522,6 @@ class FederationClient(FederationBase):
         Does so by asking one of the already participating servers to create an
         event with proper context.
 
-        Returns a fully signed and hashed event.
-
         Note that this does not append any events to any graphs.
 
         Args:
@@ -567,10 +536,8 @@ class FederationClient(FederationBase):
             params (dict[str, str|Iterable[str]]): Query parameters to include in the
                 request.
         Return:
-            Deferred[tuple[str, FrozenEvent, int]]: resolves to a tuple of
-            `(origin, event, event_format)` where origin is the remote
-            homeserver which generated the event, and event_format is one of
-            `synapse.api.constants.EventFormatVersions`.
+            Deferred: resolves to a tuple of (origin (str), event (object))
+            where origin is the remote homeserver which generated the event.
 
             Fails with a ``SynapseError`` if the chosen remote server
             returns a 300/400 code.
@@ -590,11 +557,6 @@ class FederationClient(FederationBase):
                 destination, room_id, user_id, membership, params,
             )
 
-            # Note: If not supplied, the room version may be either v1 or v2,
-            # however either way the event format version will be v1.
-            room_version = ret.get("room_version", RoomVersions.V1)
-            event_format = room_version_to_event_format(room_version)
-
             pdu_dict = ret.get("event", None)
             if not isinstance(pdu_dict, dict):
                 raise InvalidResponseError("Bad 'event' field in response")
@@ -609,20 +571,17 @@ class FederationClient(FederationBase):
             if "prev_state" not in pdu_dict:
                 pdu_dict["prev_state"] = []
 
-            ev = builder.create_local_event_from_event_dict(
-                self._clock, self.hostname, self.signing_key,
-                format_version=event_format, event_dict=pdu_dict,
-            )
+            ev = builder.EventBuilder(pdu_dict)
 
             defer.returnValue(
-                (destination, ev, event_format)
+                (destination, ev)
             )
 
         return self._try_destination_list(
             "make_" + membership, destinations, send_request,
         )
 
-    def send_join(self, destinations, pdu, event_format_version):
+    def send_join(self, destinations, pdu):
         """Sends a join event to one of a list of homeservers.
 
         Doing so will cause the remote server to add the event to the graph,
@@ -632,7 +591,6 @@ class FederationClient(FederationBase):
             destinations (str): Candidate homeservers which are probably
                 participating in the room.
             pdu (BaseEvent): event to be sent
-            event_format_version (int): The event format version
 
         Return:
             Deferred: resolves to a dict with members ``origin`` (a string
@@ -678,12 +636,12 @@ class FederationClient(FederationBase):
             logger.debug("Got content: %s", content)
 
             state = [
-                event_from_pdu_json(p, event_format_version, outlier=True)
+                event_from_pdu_json(p, outlier=True)
                 for p in content.get("state", [])
             ]
 
             auth_chain = [
-                event_from_pdu_json(p, event_format_version, outlier=True)
+                event_from_pdu_json(p, outlier=True)
                 for p in content.get("auth_chain", [])
             ]
 
@@ -692,21 +650,9 @@ class FederationClient(FederationBase):
                 for p in itertools.chain(state, auth_chain)
             }
 
-            room_version = None
-            for e in state:
-                if (e.type, e.state_key) == (EventTypes.Create, ""):
-                    room_version = e.content.get("room_version", RoomVersions.V1)
-                    break
-
-            if room_version is None:
-                # If the state doesn't have a create event then the room is
-                # invalid, and it would fail auth checks anyway.
-                raise SynapseError(400, "No create event in state")
-
             valid_pdus = yield self._check_sigs_and_hash_and_fetch(
                 destination, list(pdus.values()),
                 outlier=True,
-                room_version=room_version,
             )
 
             valid_pdus_map = {
@@ -744,90 +690,32 @@ class FederationClient(FederationBase):
 
     @defer.inlineCallbacks
     def send_invite(self, destination, room_id, event_id, pdu):
-        room_version = yield self.store.get_room_version(room_id)
-
-        content = yield self._do_send_invite(destination, pdu, room_version)
+        time_now = self._clock.time_msec()
+        try:
+            code, content = yield self.transport_layer.send_invite(
+                destination=destination,
+                room_id=room_id,
+                event_id=event_id,
+                content=pdu.get_pdu_json(time_now),
+            )
+        except HttpResponseException as e:
+            if e.code == 403:
+                raise e.to_synapse_error()
+            raise
 
         pdu_dict = content["event"]
 
         logger.debug("Got response to send_invite: %s", pdu_dict)
 
-        room_version = yield self.store.get_room_version(room_id)
-        format_ver = room_version_to_event_format(room_version)
-
-        pdu = event_from_pdu_json(pdu_dict, format_ver)
+        pdu = event_from_pdu_json(pdu_dict)
 
         # Check signatures are correct.
-        pdu = yield self._check_sigs_and_hash(room_version, pdu)
+        pdu = yield self._check_sigs_and_hash(pdu)
 
         # FIXME: We should handle signature failures more gracefully.
 
         defer.returnValue(pdu)
 
-    @defer.inlineCallbacks
-    def _do_send_invite(self, destination, pdu, room_version):
-        """Actually sends the invite, first trying v2 API and falling back to
-        v1 API if necessary.
-
-        Args:
-            destination (str): Target server
-            pdu (FrozenEvent)
-            room_version (str)
-
-        Returns:
-            dict: The event as a dict as returned by the remote server
-        """
-        time_now = self._clock.time_msec()
-
-        try:
-            content = yield self.transport_layer.send_invite_v2(
-                destination=destination,
-                room_id=pdu.room_id,
-                event_id=pdu.event_id,
-                content={
-                    "event": pdu.get_pdu_json(time_now),
-                    "room_version": room_version,
-                    "invite_room_state": pdu.unsigned.get("invite_room_state", []),
-                },
-            )
-            defer.returnValue(content)
-        except HttpResponseException as e:
-            if e.code in [400, 404]:
-                err = e.to_synapse_error()
-
-                # If we receive an error response that isn't a generic error, we
-                # assume that the remote understands the v2 invite API and this
-                # is a legitimate error.
-                if err.errcode != Codes.UNKNOWN:
-                    raise err
-
-                # Otherwise, we assume that the remote server doesn't understand
-                # the v2 invite API.
-
-                if room_version in (RoomVersions.V1, RoomVersions.V2):
-                    pass  # We'll fall through
-                else:
-                    raise SynapseError(
-                        400,
-                        "User's homeserver does not support this room version",
-                        Codes.UNSUPPORTED_ROOM_VERSION,
-                    )
-            elif e.code == 403:
-                raise e.to_synapse_error()
-            else:
-                raise
-
-        # Didn't work, try v1 API.
-        # Note the v1 API returns a tuple of `(200, content)`
-
-        _, content = yield self.transport_layer.send_invite_v1(
-            destination=destination,
-            room_id=pdu.room_id,
-            event_id=pdu.event_id,
-            content=pdu.get_pdu_json(time_now),
-        )
-        defer.returnValue(content)
-
     def send_leave(self, destinations, pdu):
         """Sends a leave event to one of a list of homeservers.
 
@@ -897,16 +785,13 @@ class FederationClient(FederationBase):
             content=send_content,
         )
 
-        room_version = yield self.store.get_room_version(room_id)
-        format_ver = room_version_to_event_format(room_version)
-
         auth_chain = [
-            event_from_pdu_json(e, format_ver)
+            event_from_pdu_json(e)
             for e in content["auth_chain"]
         ]
 
         signed_auth = yield self._check_sigs_and_hash_and_fetch(
-            destination, auth_chain, outlier=True, room_version=room_version,
+            destination, auth_chain, outlier=True
         )
 
         signed_auth.sort(key=lambda e: e.depth)
@@ -948,16 +833,13 @@ class FederationClient(FederationBase):
                 timeout=timeout,
             )
 
-            room_version = yield self.store.get_room_version(room_id)
-            format_ver = room_version_to_event_format(room_version)
-
             events = [
-                event_from_pdu_json(e, format_ver)
+                event_from_pdu_json(e)
                 for e in content.get("events", [])
             ]
 
             signed_events = yield self._check_sigs_and_hash_and_fetch(
-                destination, events, outlier=False, room_version=room_version,
+                destination, events, outlier=False
             )
         except HttpResponseException as e:
             if not e.code == 400:

synapse/federation/federation_server.py

@@ -25,17 +25,15 @@ from twisted.internet import defer
 from twisted.internet.abstract import isIPAddress
 from twisted.python import failure
 
-from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventTypes, Membership
+from synapse.api.constants import EventTypes
 from synapse.api.errors import (
     AuthError,
-    Codes,
     FederationError,
     IncompatibleRoomVersionError,
     NotFoundError,
     SynapseError,
 )
 from synapse.crypto.event_signing import compute_event_signature
-from synapse.events import room_version_to_event_format
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.federation.persistence import TransactionActions
 from synapse.federation.units import Edu, Transaction
@@ -149,22 +147,6 @@ class FederationServer(FederationBase):
 
         logger.debug("[%s] Transaction is new", transaction.transaction_id)
 
-        # Reject if PDU count > 50 and EDU count > 100
-        if (len(transaction.pdus) > 50
-                or (hasattr(transaction, "edus") and len(transaction.edus) > 100)):
-
-            logger.info(
-                "Transaction PDU or EDU count too large. Returning 400",
-            )
-
-            response = {}
-            yield self.transaction_actions.set_response(
-                origin,
-                transaction,
-                400, response
-            )
-            defer.returnValue((400, response))
-
         received_pdus_counter.inc(len(transaction.pdus))
 
         origin_host, _ = parse_server_name(origin)
@@ -196,13 +178,14 @@ class FederationServer(FederationBase):
                 continue
 
             try:
-                room_version = yield self.store.get_room_version(room_id)
-                format_ver = room_version_to_event_format(room_version)
+                # In future we will actually use the room version to parse the
+                # PDU into an event.
+                yield self.store.get_room_version(room_id)
             except NotFoundError:
                 logger.info("Ignoring PDU for unknown room_id: %s", room_id)
                 continue
 
-            event = event_from_pdu_json(p, format_ver)
+            event = event_from_pdu_json(p)
             pdus_by_room.setdefault(room_id, []).append(event)
 
         pdu_results = {}
@@ -240,9 +223,8 @@ class FederationServer(FederationBase):
                         f = failure.Failure()
                         pdu_results[event_id] = {"error": str(e)}
                         logger.error(
-                            "Failed to handle PDU %s",
-                            event_id,
-                            exc_info=(f.type, f.value, f.getTracebackObject()),
+                            "Failed to handle PDU %s: %s",
+                            event_id, f.getTraceback().rstrip(),
                         )
 
         yield concurrently_execute(
@@ -340,7 +322,7 @@ class FederationServer(FederationBase):
             if self.hs.is_mine_id(event.event_id):
                 event.signatures.update(
                     compute_event_signature(
-                        event.get_pdu_json(),
+                        event,
                         self.hs.hostname,
                         self.hs.config.signing_key[0]
                     )
@@ -387,30 +369,18 @@ class FederationServer(FederationBase):
         })
 
     @defer.inlineCallbacks
-    def on_invite_request(self, origin, content, room_version):
-        if room_version not in KNOWN_ROOM_VERSIONS:
-            raise SynapseError(
-                400,
-                "Homeserver does not support this room version",
-                Codes.UNSUPPORTED_ROOM_VERSION,
-            )
-
-        format_ver = room_version_to_event_format(room_version)
-
-        pdu = event_from_pdu_json(content, format_ver)
+    def on_invite_request(self, origin, content):
+        pdu = event_from_pdu_json(content)
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, pdu.room_id)
         ret_pdu = yield self.handler.on_invite_request(origin, pdu)
         time_now = self._clock.time_msec()
-        defer.returnValue({"event": ret_pdu.get_pdu_json(time_now)})
+        defer.returnValue((200, {"event": ret_pdu.get_pdu_json(time_now)}))
 
     @defer.inlineCallbacks
-    def on_send_join_request(self, origin, content, room_id):
+    def on_send_join_request(self, origin, content):
         logger.debug("on_send_join_request: content: %s", content)
-
-        room_version = yield self.store.get_room_version(room_id)
-        format_ver = room_version_to_event_format(room_version)
-        pdu = event_from_pdu_json(content, format_ver)
+        pdu = event_from_pdu_json(content)
 
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, pdu.room_id)
@@ -430,22 +400,13 @@ class FederationServer(FederationBase):
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, room_id)
         pdu = yield self.handler.on_make_leave_request(room_id, user_id)
-
-        room_version = yield self.store.get_room_version(room_id)
-
         time_now = self._clock.time_msec()
-        defer.returnValue({
-            "event": pdu.get_pdu_json(time_now),
-            "room_version": room_version,
-        })
+        defer.returnValue({"event": pdu.get_pdu_json(time_now)})
 
     @defer.inlineCallbacks
-    def on_send_leave_request(self, origin, content, room_id):
+    def on_send_leave_request(self, origin, content):
         logger.debug("on_send_leave_request: content: %s", content)
-
-        room_version = yield self.store.get_room_version(room_id)
-        format_ver = room_version_to_event_format(room_version)
-        pdu = event_from_pdu_json(content, format_ver)
+        pdu = event_from_pdu_json(content)
 
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, pdu.room_id)
@@ -491,16 +452,13 @@ class FederationServer(FederationBase):
             origin_host, _ = parse_server_name(origin)
             yield self.check_server_matches_acl(origin_host, room_id)
 
-            room_version = yield self.store.get_room_version(room_id)
-            format_ver = room_version_to_event_format(room_version)
-
             auth_chain = [
-                event_from_pdu_json(e, format_ver)
+                event_from_pdu_json(e)
                 for e in content["auth_chain"]
             ]
 
             signed_auth = yield self._check_sigs_and_hash_and_fetch(
-                origin, auth_chain, outlier=True, room_version=room_version,
+                origin, auth_chain, outlier=True
             )
 
             ret = yield self.handler.on_query_auth(
@@ -645,19 +603,16 @@ class FederationServer(FederationBase):
         """
         # check that it's actually being sent from a valid destination to
         # workaround bug #1753 in 0.18.5 and 0.18.6
-        if origin != get_domain_from_id(pdu.sender):
+        if origin != get_domain_from_id(pdu.event_id):
             # We continue to accept join events from any server; this is
             # necessary for the federation join dance to work correctly.
             # (When we join over federation, the "helper" server is
             # responsible for sending out the join event, rather than the
-            # origin. See bug #1893. This is also true for some third party
-            # invites).
+            # origin. See bug #1893).
             if not (
                 pdu.type == 'm.room.member' and
                 pdu.content and
-                pdu.content.get("membership", None) in (
-                    Membership.JOIN, Membership.INVITE,
-                )
+                pdu.content.get("membership", None) == 'join'
             ):
                 logger.info(
                     "Discarding PDU %s from invalid origin %s",
@@ -670,12 +625,9 @@ class FederationServer(FederationBase):
                     pdu.event_id, origin
                 )
 
-        # We've already checked that we know the room version by this point
-        room_version = yield self.store.get_room_version(pdu.room_id)
-
         # Check signature.
         try:
-            pdu = yield self._check_sigs_and_hash(room_version, pdu)
+            pdu = yield self._check_sigs_and_hash(pdu)
         except SynapseError as e:
             raise FederationError(
                 "ERROR",

synapse/federation/transaction_queue.py

@@ -33,6 +33,7 @@ from synapse.metrics import (
     event_processing_loop_counter,
     event_processing_loop_room_count,
     events_processed_counter,
+    sent_edus_counter,
     sent_transactions_counter,
 )
 from synapse.metrics.background_process_metrics import run_as_background_process
@@ -46,24 +47,10 @@ from .units import Edu, Transaction
 logger = logging.getLogger(__name__)
 
 sent_pdus_destination_dist_count = Counter(
-    "synapse_federation_client_sent_pdu_destinations:count",
-    "Number of PDUs queued for sending to one or more destinations",
+    "synapse_federation_client_sent_pdu_destinations:count", ""
 )
-
 sent_pdus_destination_dist_total = Counter(
     "synapse_federation_client_sent_pdu_destinations:total", ""
-    "Total number of PDUs queued for sending across all destinations",
-)
-
-sent_edus_counter = Counter(
-    "synapse_federation_client_sent_edus",
-    "Total number of EDUs successfully sent",
-)
-
-sent_edus_by_type = Counter(
-    "synapse_federation_client_sent_edus_by_type",
-    "Number of sent EDUs successfully sent, by event type",
-    ["type"],
 )
 
 
@@ -188,7 +175,7 @@ class TransactionQueue(object):
                 def handle_event(event):
                     # Only send events for this server.
                     send_on_behalf_of = event.internal_metadata.get_send_on_behalf_of()
-                    is_mine = self.is_mine_id(event.sender)
+                    is_mine = self.is_mine_id(event.event_id)
                     if not is_mine and send_on_behalf_of is None:
                         return
 
@@ -373,6 +360,8 @@ class TransactionQueue(object):
             logger.info("Not sending EDU to ourselves")
             return
 
+        sent_edus_counter.inc()
+
         if key:
             self.pending_edus_keyed_by_dest.setdefault(
                 destination, {}
@@ -507,9 +496,6 @@ class TransactionQueue(object):
                 )
                 if success:
                     sent_transactions_counter.inc()
-                    sent_edus_counter.inc(len(pending_edus))
-                    for edu in pending_edus:
-                        sent_edus_by_type.labels(edu.edu_type).inc()
                     # Remove the acknowledged device messages from the database
                     # Only bother if we actually sent some device messages
                     if device_message_edus:

synapse/federation/transport/client.py

@@ -21,7 +21,7 @@ from six.moves import urllib
 from twisted.internet import defer
 
 from synapse.api.constants import Membership
-from synapse.api.urls import FEDERATION_V1_PREFIX, FEDERATION_V2_PREFIX
+from synapse.api.urls import FEDERATION_PREFIX as PREFIX
 from synapse.util.logutils import log_function
 
 logger = logging.getLogger(__name__)
@@ -51,7 +51,7 @@ class TransportLayerClient(object):
         logger.debug("get_room_state dest=%s, room=%s",
                      destination, room_id)
 
-        path = _create_v1_path("/state/%s/", room_id)
+        path = _create_path(PREFIX, "/state/%s/", room_id)
         return self.client.get_json(
             destination, path=path, args={"event_id": event_id},
         )
@@ -73,7 +73,7 @@ class TransportLayerClient(object):
         logger.debug("get_room_state_ids dest=%s, room=%s",
                      destination, room_id)
 
-        path = _create_v1_path("/state_ids/%s/", room_id)
+        path = _create_path(PREFIX, "/state_ids/%s/", room_id)
         return self.client.get_json(
             destination, path=path, args={"event_id": event_id},
         )
@@ -95,7 +95,7 @@ class TransportLayerClient(object):
         logger.debug("get_pdu dest=%s, event_id=%s",
                      destination, event_id)
 
-        path = _create_v1_path("/event/%s/", event_id)
+        path = _create_path(PREFIX, "/event/%s/", event_id)
         return self.client.get_json(destination, path=path, timeout=timeout)
 
     @log_function
@@ -121,7 +121,7 @@ class TransportLayerClient(object):
             # TODO: raise?
             return
 
-        path = _create_v1_path("/backfill/%s/", room_id)
+        path = _create_path(PREFIX, "/backfill/%s/", room_id)
 
         args = {
             "v": event_tuples,
@@ -167,7 +167,7 @@ class TransportLayerClient(object):
         # generated by the json_data_callback.
         json_data = transaction.get_dict()
 
-        path = _create_v1_path("/send/%s/", transaction.transaction_id)
+        path = _create_path(PREFIX, "/send/%s/", transaction.transaction_id)
 
         response = yield self.client.put_json(
             transaction.destination,
@@ -184,7 +184,7 @@ class TransportLayerClient(object):
     @log_function
     def make_query(self, destination, query_type, args, retry_on_dns_fail,
                    ignore_backoff=False):
-        path = _create_v1_path("/query/%s", query_type)
+        path = _create_path(PREFIX, "/query/%s", query_type)
 
         content = yield self.client.get_json(
             destination=destination,
@@ -231,7 +231,7 @@ class TransportLayerClient(object):
                 "make_membership_event called with membership='%s', must be one of %s" %
                 (membership, ",".join(valid_memberships))
             )
-        path = _create_v1_path("/make_%s/%s/%s", membership, room_id, user_id)
+        path = _create_path(PREFIX, "/make_%s/%s/%s", membership, room_id, user_id)
 
         ignore_backoff = False
         retry_on_dns_fail = False
@@ -258,7 +258,7 @@ class TransportLayerClient(object):
     @defer.inlineCallbacks
     @log_function
     def send_join(self, destination, room_id, event_id, content):
-        path = _create_v1_path("/send_join/%s/%s", room_id, event_id)
+        path = _create_path(PREFIX, "/send_join/%s/%s", room_id, event_id)
 
         response = yield self.client.put_json(
             destination=destination,
@@ -271,7 +271,7 @@ class TransportLayerClient(object):
     @defer.inlineCallbacks
     @log_function
     def send_leave(self, destination, room_id, event_id, content):
-        path = _create_v1_path("/send_leave/%s/%s", room_id, event_id)
+        path = _create_path(PREFIX, "/send_leave/%s/%s", room_id, event_id)
 
         response = yield self.client.put_json(
             destination=destination,
@@ -289,22 +289,8 @@ class TransportLayerClient(object):
 
     @defer.inlineCallbacks
     @log_function
-    def send_invite_v1(self, destination, room_id, event_id, content):
-        path = _create_v1_path("/invite/%s/%s", room_id, event_id)
-
-        response = yield self.client.put_json(
-            destination=destination,
-            path=path,
-            data=content,
-            ignore_backoff=True,
-        )
-
-        defer.returnValue(response)
-
-    @defer.inlineCallbacks
-    @log_function
-    def send_invite_v2(self, destination, room_id, event_id, content):
-        path = _create_v2_path("/invite/%s/%s", room_id, event_id)
+    def send_invite(self, destination, room_id, event_id, content):
+        path = _create_path(PREFIX, "/invite/%s/%s", room_id, event_id)
 
         response = yield self.client.put_json(
             destination=destination,
@@ -320,7 +306,7 @@ class TransportLayerClient(object):
     def get_public_rooms(self, remote_server, limit, since_token,
                          search_filter=None, include_all_networks=False,
                          third_party_instance_id=None):
-        path = _create_v1_path("/publicRooms")
+        path = PREFIX + "/publicRooms"
 
         args = {
             "include_all_networks": "true" if include_all_networks else "false",
@@ -346,7 +332,7 @@ class TransportLayerClient(object):
     @defer.inlineCallbacks
     @log_function
     def exchange_third_party_invite(self, destination, room_id, event_dict):
-        path = _create_v1_path("/exchange_third_party_invite/%s", room_id,)
+        path = _create_path(PREFIX, "/exchange_third_party_invite/%s", room_id,)
 
         response = yield self.client.put_json(
             destination=destination,
@@ -359,7 +345,7 @@ class TransportLayerClient(object):
     @defer.inlineCallbacks
     @log_function
     def get_event_auth(self, destination, room_id, event_id):
-        path = _create_v1_path("/event_auth/%s/%s", room_id, event_id)
+        path = _create_path(PREFIX, "/event_auth/%s/%s", room_id, event_id)
 
         content = yield self.client.get_json(
             destination=destination,
@@ -371,7 +357,7 @@ class TransportLayerClient(object):
     @defer.inlineCallbacks
     @log_function
     def send_query_auth(self, destination, room_id, event_id, content):
-        path = _create_v1_path("/query_auth/%s/%s", room_id, event_id)
+        path = _create_path(PREFIX, "/query_auth/%s/%s", room_id, event_id)
 
         content = yield self.client.post_json(
             destination=destination,
@@ -406,7 +392,7 @@ class TransportLayerClient(object):
         Returns:
             A dict containg the device keys.
         """
-        path = _create_v1_path("/user/keys/query")
+        path = PREFIX + "/user/keys/query"
 
         content = yield self.client.post_json(
             destination=destination,
@@ -433,7 +419,7 @@ class TransportLayerClient(object):
         Returns:
             A dict containg the device keys.
         """
-        path = _create_v1_path("/user/devices/%s", user_id)
+        path = _create_path(PREFIX, "/user/devices/%s", user_id)
 
         content = yield self.client.get_json(
             destination=destination,
@@ -469,7 +455,7 @@ class TransportLayerClient(object):
             A dict containg the one-time keys.
         """
 
-        path = _create_v1_path("/user/keys/claim")
+        path = PREFIX + "/user/keys/claim"
 
         content = yield self.client.post_json(
             destination=destination,
@@ -483,7 +469,7 @@ class TransportLayerClient(object):
     @log_function
     def get_missing_events(self, destination, room_id, earliest_events,
                            latest_events, limit, min_depth, timeout):
-        path = _create_v1_path("/get_missing_events/%s", room_id,)
+        path = _create_path(PREFIX, "/get_missing_events/%s", room_id,)
 
         content = yield self.client.post_json(
             destination=destination,
@@ -503,7 +489,7 @@ class TransportLayerClient(object):
     def get_group_profile(self, destination, group_id, requester_user_id):
         """Get a group profile
         """
-        path = _create_v1_path("/groups/%s/profile", group_id,)
+        path = _create_path(PREFIX, "/groups/%s/profile", group_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -522,7 +508,7 @@ class TransportLayerClient(object):
             requester_user_id (str)
             content (dict): The new profile of the group
         """
-        path = _create_v1_path("/groups/%s/profile", group_id,)
+        path = _create_path(PREFIX, "/groups/%s/profile", group_id,)
 
         return self.client.post_json(
             destination=destination,
@@ -536,7 +522,7 @@ class TransportLayerClient(object):
     def get_group_summary(self, destination, group_id, requester_user_id):
         """Get a group summary
         """
-        path = _create_v1_path("/groups/%s/summary", group_id,)
+        path = _create_path(PREFIX, "/groups/%s/summary", group_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -549,7 +535,7 @@ class TransportLayerClient(object):
     def get_rooms_in_group(self, destination, group_id, requester_user_id):
         """Get all rooms in a group
         """
-        path = _create_v1_path("/groups/%s/rooms", group_id,)
+        path = _create_path(PREFIX, "/groups/%s/rooms", group_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -562,7 +548,7 @@ class TransportLayerClient(object):
                           content):
         """Add a room to a group
         """
-        path = _create_v1_path("/groups/%s/room/%s", group_id, room_id,)
+        path = _create_path(PREFIX, "/groups/%s/room/%s", group_id, room_id,)
 
         return self.client.post_json(
             destination=destination,
@@ -576,8 +562,8 @@ class TransportLayerClient(object):
                              config_key, content):
         """Update room in group
         """
-        path = _create_v1_path(
-            "/groups/%s/room/%s/config/%s",
+        path = _create_path(
+            PREFIX, "/groups/%s/room/%s/config/%s",
             group_id, room_id, config_key,
         )
 
@@ -592,7 +578,7 @@ class TransportLayerClient(object):
     def remove_room_from_group(self, destination, group_id, requester_user_id, room_id):
         """Remove a room from a group
         """
-        path = _create_v1_path("/groups/%s/room/%s", group_id, room_id,)
+        path = _create_path(PREFIX, "/groups/%s/room/%s", group_id, room_id,)
 
         return self.client.delete_json(
             destination=destination,
@@ -605,7 +591,7 @@ class TransportLayerClient(object):
     def get_users_in_group(self, destination, group_id, requester_user_id):
         """Get users in a group
         """
-        path = _create_v1_path("/groups/%s/users", group_id,)
+        path = _create_path(PREFIX, "/groups/%s/users", group_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -618,7 +604,7 @@ class TransportLayerClient(object):
     def get_invited_users_in_group(self, destination, group_id, requester_user_id):
         """Get users that have been invited to a group
         """
-        path = _create_v1_path("/groups/%s/invited_users", group_id,)
+        path = _create_path(PREFIX, "/groups/%s/invited_users", group_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -631,8 +617,8 @@ class TransportLayerClient(object):
     def accept_group_invite(self, destination, group_id, user_id, content):
         """Accept a group invite
         """
-        path = _create_v1_path(
-            "/groups/%s/users/%s/accept_invite",
+        path = _create_path(
+            PREFIX, "/groups/%s/users/%s/accept_invite",
             group_id, user_id,
         )
 
@@ -647,7 +633,7 @@ class TransportLayerClient(object):
     def join_group(self, destination, group_id, user_id, content):
         """Attempts to join a group
         """
-        path = _create_v1_path("/groups/%s/users/%s/join", group_id, user_id)
+        path = _create_path(PREFIX, "/groups/%s/users/%s/join", group_id, user_id)
 
         return self.client.post_json(
             destination=destination,
@@ -660,7 +646,7 @@ class TransportLayerClient(object):
     def invite_to_group(self, destination, group_id, user_id, requester_user_id, content):
         """Invite a user to a group
         """
-        path = _create_v1_path("/groups/%s/users/%s/invite", group_id, user_id)
+        path = _create_path(PREFIX, "/groups/%s/users/%s/invite", group_id, user_id)
 
         return self.client.post_json(
             destination=destination,
@@ -676,7 +662,7 @@ class TransportLayerClient(object):
         invited.
         """
 
-        path = _create_v1_path("/groups/local/%s/users/%s/invite", group_id, user_id)
+        path = _create_path(PREFIX, "/groups/local/%s/users/%s/invite", group_id, user_id)
 
         return self.client.post_json(
             destination=destination,
@@ -690,7 +676,7 @@ class TransportLayerClient(object):
                                user_id, content):
         """Remove a user fron a group
         """
-        path = _create_v1_path("/groups/%s/users/%s/remove", group_id, user_id)
+        path = _create_path(PREFIX, "/groups/%s/users/%s/remove", group_id, user_id)
 
         return self.client.post_json(
             destination=destination,
@@ -707,7 +693,7 @@ class TransportLayerClient(object):
         kicked from the group.
         """
 
-        path = _create_v1_path("/groups/local/%s/users/%s/remove", group_id, user_id)
+        path = _create_path(PREFIX, "/groups/local/%s/users/%s/remove", group_id, user_id)
 
         return self.client.post_json(
             destination=destination,
@@ -722,7 +708,7 @@ class TransportLayerClient(object):
         the attestations
         """
 
-        path = _create_v1_path("/groups/%s/renew_attestation/%s", group_id, user_id)
+        path = _create_path(PREFIX, "/groups/%s/renew_attestation/%s", group_id, user_id)
 
         return self.client.post_json(
             destination=destination,
@@ -737,12 +723,12 @@ class TransportLayerClient(object):
         """Update a room entry in a group summary
         """
         if category_id:
-            path = _create_v1_path(
-                "/groups/%s/summary/categories/%s/rooms/%s",
+            path = _create_path(
+                PREFIX, "/groups/%s/summary/categories/%s/rooms/%s",
                 group_id, category_id, room_id,
             )
         else:
-            path = _create_v1_path("/groups/%s/summary/rooms/%s", group_id, room_id,)
+            path = _create_path(PREFIX, "/groups/%s/summary/rooms/%s", group_id, room_id,)
 
         return self.client.post_json(
             destination=destination,
@@ -758,12 +744,12 @@ class TransportLayerClient(object):
         """Delete a room entry in a group summary
         """
         if category_id:
-            path = _create_v1_path(
-                "/groups/%s/summary/categories/%s/rooms/%s",
+            path = _create_path(
+                PREFIX + "/groups/%s/summary/categories/%s/rooms/%s",
                 group_id, category_id, room_id,
             )
         else:
-            path = _create_v1_path("/groups/%s/summary/rooms/%s", group_id, room_id,)
+            path = _create_path(PREFIX, "/groups/%s/summary/rooms/%s", group_id, room_id,)
 
         return self.client.delete_json(
             destination=destination,
@@ -776,7 +762,7 @@ class TransportLayerClient(object):
     def get_group_categories(self, destination, group_id, requester_user_id):
         """Get all categories in a group
         """
-        path = _create_v1_path("/groups/%s/categories", group_id,)
+        path = _create_path(PREFIX, "/groups/%s/categories", group_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -789,7 +775,7 @@ class TransportLayerClient(object):
     def get_group_category(self, destination, group_id, requester_user_id, category_id):
         """Get category info in a group
         """
-        path = _create_v1_path("/groups/%s/categories/%s", group_id, category_id,)
+        path = _create_path(PREFIX, "/groups/%s/categories/%s", group_id, category_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -803,7 +789,7 @@ class TransportLayerClient(object):
                               content):
         """Update a category in a group
         """
-        path = _create_v1_path("/groups/%s/categories/%s", group_id, category_id,)
+        path = _create_path(PREFIX, "/groups/%s/categories/%s", group_id, category_id,)
 
         return self.client.post_json(
             destination=destination,
@@ -818,7 +804,7 @@ class TransportLayerClient(object):
                               category_id):
         """Delete a category in a group
         """
-        path = _create_v1_path("/groups/%s/categories/%s", group_id, category_id,)
+        path = _create_path(PREFIX, "/groups/%s/categories/%s", group_id, category_id,)
 
         return self.client.delete_json(
             destination=destination,
@@ -831,7 +817,7 @@ class TransportLayerClient(object):
     def get_group_roles(self, destination, group_id, requester_user_id):
         """Get all roles in a group
         """
-        path = _create_v1_path("/groups/%s/roles", group_id,)
+        path = _create_path(PREFIX, "/groups/%s/roles", group_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -844,7 +830,7 @@ class TransportLayerClient(object):
     def get_group_role(self, destination, group_id, requester_user_id, role_id):
         """Get a roles info
         """
-        path = _create_v1_path("/groups/%s/roles/%s", group_id, role_id,)
+        path = _create_path(PREFIX, "/groups/%s/roles/%s", group_id, role_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -858,7 +844,7 @@ class TransportLayerClient(object):
                           content):
         """Update a role in a group
         """
-        path = _create_v1_path("/groups/%s/roles/%s", group_id, role_id,)
+        path = _create_path(PREFIX, "/groups/%s/roles/%s", group_id, role_id,)
 
         return self.client.post_json(
             destination=destination,
@@ -872,7 +858,7 @@ class TransportLayerClient(object):
     def delete_group_role(self, destination, group_id, requester_user_id, role_id):
         """Delete a role in a group
         """
-        path = _create_v1_path("/groups/%s/roles/%s", group_id, role_id,)
+        path = _create_path(PREFIX, "/groups/%s/roles/%s", group_id, role_id,)
 
         return self.client.delete_json(
             destination=destination,
@@ -887,12 +873,12 @@ class TransportLayerClient(object):
         """Update a users entry in a group
         """
         if role_id:
-            path = _create_v1_path(
-                "/groups/%s/summary/roles/%s/users/%s",
+            path = _create_path(
+                PREFIX, "/groups/%s/summary/roles/%s/users/%s",
                 group_id, role_id, user_id,
             )
         else:
-            path = _create_v1_path("/groups/%s/summary/users/%s", group_id, user_id,)
+            path = _create_path(PREFIX, "/groups/%s/summary/users/%s", group_id, user_id,)
 
         return self.client.post_json(
             destination=destination,
@@ -907,7 +893,7 @@ class TransportLayerClient(object):
                               content):
         """Sets the join policy for a group
         """
-        path = _create_v1_path("/groups/%s/settings/m.join_policy", group_id,)
+        path = _create_path(PREFIX, "/groups/%s/settings/m.join_policy", group_id,)
 
         return self.client.put_json(
             destination=destination,
@@ -923,12 +909,12 @@ class TransportLayerClient(object):
         """Delete a users entry in a group
         """
         if role_id:
-            path = _create_v1_path(
-                "/groups/%s/summary/roles/%s/users/%s",
+            path = _create_path(
+                PREFIX, "/groups/%s/summary/roles/%s/users/%s",
                 group_id, role_id, user_id,
             )
         else:
-            path = _create_v1_path("/groups/%s/summary/users/%s", group_id, user_id,)
+            path = _create_path(PREFIX, "/groups/%s/summary/users/%s", group_id, user_id,)
 
         return self.client.delete_json(
             destination=destination,
@@ -941,7 +927,7 @@ class TransportLayerClient(object):
         """Get the groups a list of users are publicising
         """
 
-        path = _create_v1_path("/get_groups_publicised")
+        path = PREFIX + "/get_groups_publicised"
 
         content = {"user_ids": user_ids}
 
@@ -953,43 +939,20 @@ class TransportLayerClient(object):
         )
 
 
-def _create_v1_path(path, *args):
-    """Creates a path against V1 federation API from the path template and
-    args. Ensures that all args are url encoded.
+def _create_path(prefix, path, *args):
+    """Creates a path from the prefix, path template and args. Ensures that
+    all args are url encoded.
 
     Example:
 
-        _create_v1_path("/event/%s/", event_id)
+        _create_path(PREFIX, "/event/%s/", event_id)
 
     Args:
+        prefix (str)
         path (str): String template for the path
         args: ([str]): Args to insert into path. Each arg will be url encoded
 
     Returns:
         str
     """
-    return (
-        FEDERATION_V1_PREFIX
-        + path % tuple(urllib.parse.quote(arg, "") for arg in args)
-    )
-
-
-def _create_v2_path(path, *args):
-    """Creates a path against V2 federation API from the path template and
-    args. Ensures that all args are url encoded.
-
-    Example:
-
-        _create_v2_path("/event/%s/", event_id)
-
-    Args:
-        path (str): String template for the path
-        args: ([str]): Args to insert into path. Each arg will be url encoded
-
-    Returns:
-        str
-    """
-    return (
-        FEDERATION_V2_PREFIX
-        + path % tuple(urllib.parse.quote(arg, "") for arg in args)
-    )
+    return prefix + path % tuple(urllib.parse.quote(arg, "") for arg in args)

synapse/federation/transport/server.py

@@ -21,9 +21,8 @@ import re
 from twisted.internet import defer
 
 import synapse
-from synapse.api.constants import RoomVersions
 from synapse.api.errors import Codes, FederationDeniedError, SynapseError
-from synapse.api.urls import FEDERATION_V1_PREFIX, FEDERATION_V2_PREFIX
+from synapse.api.urls import FEDERATION_PREFIX as PREFIX
 from synapse.http.endpoint import parse_and_validate_server_name
 from synapse.http.server import JsonResource
 from synapse.http.servlet import (
@@ -43,20 +42,9 @@ logger = logging.getLogger(__name__)
 class TransportLayerServer(JsonResource):
     """Handles incoming federation HTTP requests"""
 
-    def __init__(self, hs, servlet_groups=None):
-        """Initialize the TransportLayerServer
-
-        Will by default register all servlets. For custom behaviour, pass in
-        a list of servlet_groups to register.
-
-        Args:
-            hs (synapse.server.HomeServer): homeserver
-            servlet_groups (list[str], optional): List of servlet groups to register.
-                Defaults to ``DEFAULT_SERVLET_GROUPS``.
-        """
+    def __init__(self, hs):
         self.hs = hs
         self.clock = hs.get_clock()
-        self.servlet_groups = servlet_groups
 
         super(TransportLayerServer, self).__init__(hs, canonical_json=False)
 
@@ -78,7 +66,6 @@ class TransportLayerServer(JsonResource):
             resource=self,
             ratelimiter=self.ratelimiter,
             authenticator=self.authenticator,
-            servlet_groups=self.servlet_groups,
         )
 
 
@@ -240,8 +227,6 @@ class BaseFederationServlet(object):
     """
     REQUIRE_AUTH = True
 
-    PREFIX = FEDERATION_V1_PREFIX  # Allows specifying the API version
-
     def __init__(self, handler, authenticator, ratelimiter, server_name):
         self.handler = handler
         self.authenticator = authenticator
@@ -301,7 +286,7 @@ class BaseFederationServlet(object):
         return new_func
 
     def register(self, server):
-        pattern = re.compile("^" + self.PREFIX + self.PATH + "$")
+        pattern = re.compile("^" + PREFIX + self.PATH + "$")
 
         for method in ("GET", "PUT", "POST"):
             code = getattr(self, "on_%s" % (method), None)
@@ -481,7 +466,7 @@ class FederationSendLeaveServlet(BaseFederationServlet):
 
     @defer.inlineCallbacks
     def on_PUT(self, origin, content, query, room_id, event_id):
-        content = yield self.handler.on_send_leave_request(origin, content, room_id)
+        content = yield self.handler.on_send_leave_request(origin, content)
         defer.returnValue((200, content))
 
 
@@ -499,50 +484,18 @@ class FederationSendJoinServlet(BaseFederationServlet):
     def on_PUT(self, origin, content, query, context, event_id):
         # TODO(paul): assert that context/event_id parsed from path actually
         #   match those given in content
-        content = yield self.handler.on_send_join_request(origin, content, context)
+        content = yield self.handler.on_send_join_request(origin, content)
         defer.returnValue((200, content))
 
 
-class FederationV1InviteServlet(BaseFederationServlet):
+class FederationInviteServlet(BaseFederationServlet):
     PATH = "/invite/(?P<context>[^/]*)/(?P<event_id>[^/]*)"
 
-    @defer.inlineCallbacks
-    def on_PUT(self, origin, content, query, context, event_id):
-        # We don't get a room version, so we have to assume its EITHER v1 or
-        # v2. This is "fine" as the only difference between V1 and V2 is the
-        # state resolution algorithm, and we don't use that for processing
-        # invites
-        content = yield self.handler.on_invite_request(
-            origin, content, room_version=RoomVersions.V1,
-        )
-
-        # V1 federation API is defined to return a content of `[200, {...}]`
-        # due to a historical bug.
-        defer.returnValue((200, (200, content)))
-
-
-class FederationV2InviteServlet(BaseFederationServlet):
-    PATH = "/invite/(?P<context>[^/]*)/(?P<event_id>[^/]*)"
-
-    PREFIX = FEDERATION_V2_PREFIX
-
     @defer.inlineCallbacks
     def on_PUT(self, origin, content, query, context, event_id):
         # TODO(paul): assert that context/event_id parsed from path actually
         #   match those given in content
-
-        room_version = content["room_version"]
-        event = content["event"]
-        invite_room_state = content["invite_room_state"]
-
-        # Synapse expects invite_room_state to be in unsigned, as it is in v1
-        # API
-
-        event.setdefault("unsigned", {})["invite_room_state"] = invite_room_state
-
-        content = yield self.handler.on_invite_request(
-            origin, event, room_version=room_version,
-        )
+        content = yield self.handler.on_invite_request(origin, content)
         defer.returnValue((200, content))
 
 
@@ -736,8 +689,7 @@ class PublicRoomList(BaseFederationServlet):
 
         data = yield self.handler.get_local_public_room_list(
             limit, since_token,
-            network_tuple=network_tuple,
-            from_federation=True,
+            network_tuple=network_tuple
         )
         defer.returnValue((200, data))
 
@@ -1311,8 +1263,7 @@ FEDERATION_SERVLET_CLASSES = (
     FederationEventServlet,
     FederationSendJoinServlet,
     FederationSendLeaveServlet,
-    FederationV1InviteServlet,
-    FederationV2InviteServlet,
+    FederationInviteServlet,
     FederationQueryAuthServlet,
     FederationGetMissingEventsServlet,
     FederationEventAuthServlet,
@@ -1321,12 +1272,10 @@ FEDERATION_SERVLET_CLASSES = (
     FederationClientKeysClaimServlet,
     FederationThirdPartyInviteExchangeServlet,
     On3pidBindServlet,
+    OpenIdUserInfo,
     FederationVersionServlet,
 )
 
-OPENID_SERVLET_CLASSES = (
-    OpenIdUserInfo,
-)
 
 ROOM_LIST_CLASSES = (
     PublicRoomList,
@@ -1365,83 +1314,44 @@ GROUP_ATTESTATION_SERVLET_CLASSES = (
     FederationGroupsRenewAttestaionServlet,
 )
 
-DEFAULT_SERVLET_GROUPS = (
-    "federation",
-    "room_list",
-    "group_server",
-    "group_local",
-    "group_attestation",
-    "openid",
-)
 
+def register_servlets(hs, resource, authenticator, ratelimiter):
+    for servletclass in FEDERATION_SERVLET_CLASSES:
+        servletclass(
+            handler=hs.get_federation_server(),
+            authenticator=authenticator,
+            ratelimiter=ratelimiter,
+            server_name=hs.hostname,
+        ).register(resource)
 
-def register_servlets(hs, resource, authenticator, ratelimiter, servlet_groups=None):
-    """Initialize and register servlet classes.
+    for servletclass in ROOM_LIST_CLASSES:
+        servletclass(
+            handler=hs.get_room_list_handler(),
+            authenticator=authenticator,
+            ratelimiter=ratelimiter,
+            server_name=hs.hostname,
+        ).register(resource)
 
-    Will by default register all servlets. For custom behaviour, pass in
-    a list of servlet_groups to register.
+    for servletclass in GROUP_SERVER_SERVLET_CLASSES:
+        servletclass(
+            handler=hs.get_groups_server_handler(),
+            authenticator=authenticator,
+            ratelimiter=ratelimiter,
+            server_name=hs.hostname,
+        ).register(resource)
 
-    Args:
-        hs (synapse.server.HomeServer): homeserver
-        resource (TransportLayerServer): resource class to register to
-        authenticator (Authenticator): authenticator to use
-        ratelimiter (util.ratelimitutils.FederationRateLimiter): ratelimiter to use
-        servlet_groups (list[str], optional): List of servlet groups to register.
-            Defaults to ``DEFAULT_SERVLET_GROUPS``.
-    """
-    if not servlet_groups:
-        servlet_groups = DEFAULT_SERVLET_GROUPS
+    for servletclass in GROUP_LOCAL_SERVLET_CLASSES:
+        servletclass(
+            handler=hs.get_groups_local_handler(),
+            authenticator=authenticator,
+            ratelimiter=ratelimiter,
+            server_name=hs.hostname,
+        ).register(resource)
 
-    if "federation" in servlet_groups:
-        for servletclass in FEDERATION_SERVLET_CLASSES:
-            servletclass(
-                handler=hs.get_federation_server(),
-                authenticator=authenticator,
-                ratelimiter=ratelimiter,
-                server_name=hs.hostname,
-            ).register(resource)
-
-    if "openid" in servlet_groups:
-        for servletclass in OPENID_SERVLET_CLASSES:
-            servletclass(
-                handler=hs.get_federation_server(),
-                authenticator=authenticator,
-                ratelimiter=ratelimiter,
-                server_name=hs.hostname,
-            ).register(resource)
-
-    if "room_list" in servlet_groups:
-        for servletclass in ROOM_LIST_CLASSES:
-            servletclass(
-                handler=hs.get_room_list_handler(),
-                authenticator=authenticator,
-                ratelimiter=ratelimiter,
-                server_name=hs.hostname,
-            ).register(resource)
-
-    if "group_server" in servlet_groups:
-        for servletclass in GROUP_SERVER_SERVLET_CLASSES:
-            servletclass(
-                handler=hs.get_groups_server_handler(),
-                authenticator=authenticator,
-                ratelimiter=ratelimiter,
-                server_name=hs.hostname,
-            ).register(resource)
-
-    if "group_local" in servlet_groups:
-        for servletclass in GROUP_LOCAL_SERVLET_CLASSES:
-            servletclass(
-                handler=hs.get_groups_local_handler(),
-                authenticator=authenticator,
-                ratelimiter=ratelimiter,
-                server_name=hs.hostname,
-            ).register(resource)
-
-    if "group_attestation" in servlet_groups:
-        for servletclass in GROUP_ATTESTATION_SERVLET_CLASSES:
-            servletclass(
-                handler=hs.get_groups_attestation_renewer(),
-                authenticator=authenticator,
-                ratelimiter=ratelimiter,
-                server_name=hs.hostname,
-            ).register(resource)
+    for servletclass in GROUP_ATTESTATION_SERVLET_CLASSES:
+        servletclass(
+            handler=hs.get_groups_attestation_renewer(),
+            authenticator=authenticator,
+            ratelimiter=ratelimiter,
+            server_name=hs.hostname,
+        ).register(resource)

synapse/groups/attestations.py

@@ -42,7 +42,7 @@ from signedjson.sign import sign_json
 
 from twisted.internet import defer
 
-from synapse.api.errors import RequestSendFailed, SynapseError
+from synapse.api.errors import SynapseError
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.types import get_domain_from_id
 from synapse.util.logcontext import run_in_background
@@ -191,11 +191,6 @@ class GroupAttestionRenewer(object):
                 yield self.store.update_attestation_renewal(
                     group_id, user_id, attestation
                 )
-            except RequestSendFailed as e:
-                logger.warning(
-                    "Failed to renew attestation of %r in %r: %s",
-                    user_id, group_id, e,
-                )
             except Exception:
                 logger.exception("Error renewing attestation of %r in %r",
                                  user_id, group_id)

synapse/groups/groups_server.py

@@ -113,7 +113,8 @@ class GroupsServerHandler(object):
             room_id = room_entry["room_id"]
             joined_users = yield self.store.get_users_in_room(room_id)
             entry = yield self.room_list_handler.generate_room_entry(
-                room_id, len(joined_users), with_alias=False, allow_private=True,
+                room_id, len(joined_users),
+                with_alias=False, allow_private=True,
             )
             entry = dict(entry)  # so we don't change whats cached
             entry.pop("room_id", None)
@@ -543,7 +544,8 @@ class GroupsServerHandler(object):
 
             joined_users = yield self.store.get_users_in_room(room_id)
             entry = yield self.room_list_handler.generate_room_entry(
-                room_id, len(joined_users), with_alias=False, allow_private=True,
+                room_id, len(joined_users),
+                with_alias=False, allow_private=True,
             )
 
             if not entry: