update sample config

Add some debug about processing read receipts.

.buildkite/.env

@@ -0,0 +1,13 @@
+CI
+BUILDKITE
+BUILDKITE_BUILD_NUMBER
+BUILDKITE_BRANCH
+BUILDKITE_BUILD_NUMBER
+BUILDKITE_JOB_ID
+BUILDKITE_BUILD_URL
+BUILDKITE_PROJECT_SLUG
+BUILDKITE_COMMIT
+BUILDKITE_PULL_REQUEST
+BUILDKITE_TAG
+CODECOV_TOKEN
+TRIAL_FLAGS

.buildkite/docker-compose.py27.pg94.yaml

@@ -0,0 +1,21 @@
+version: '3.1'
+
+services:
+
+  postgres:
+    image: postgres:9.4
+    environment:
+      POSTGRES_PASSWORD: postgres
+
+  testenv:
+    image: python:2.7
+    depends_on:
+      - postgres
+    env_file: .env
+    environment:
+      SYNAPSE_POSTGRES_HOST: postgres
+      SYNAPSE_POSTGRES_USER: postgres
+      SYNAPSE_POSTGRES_PASSWORD: postgres
+    working_dir: /app
+    volumes:
+      - ..:/app

.buildkite/docker-compose.py27.pg95.yaml

@@ -0,0 +1,21 @@
+version: '3.1'
+
+services:
+
+  postgres:
+    image: postgres:9.5
+    environment:
+      POSTGRES_PASSWORD: postgres
+
+  testenv:
+    image: python:2.7
+    depends_on:
+      - postgres
+    env_file: .env
+    environment:
+      SYNAPSE_POSTGRES_HOST: postgres
+      SYNAPSE_POSTGRES_USER: postgres
+      SYNAPSE_POSTGRES_PASSWORD: postgres
+    working_dir: /app
+    volumes:
+      - ..:/app

.buildkite/docker-compose.py35.pg94.yaml

@@ -0,0 +1,21 @@
+version: '3.1'
+
+services:
+
+  postgres:
+    image: postgres:9.4
+    environment:
+      POSTGRES_PASSWORD: postgres
+
+  testenv:
+    image: python:3.5
+    depends_on:
+      - postgres
+    env_file: .env
+    environment:
+      SYNAPSE_POSTGRES_HOST: postgres
+      SYNAPSE_POSTGRES_USER: postgres
+      SYNAPSE_POSTGRES_PASSWORD: postgres
+    working_dir: /app
+    volumes:
+      - ..:/app

.buildkite/docker-compose.py35.pg95.yaml

@@ -0,0 +1,21 @@
+version: '3.1'
+
+services:
+
+  postgres:
+    image: postgres:9.5
+    environment:
+      POSTGRES_PASSWORD: postgres
+
+  testenv:
+    image: python:3.5
+    depends_on:
+      - postgres
+    env_file: .env
+    environment:
+      SYNAPSE_POSTGRES_HOST: postgres
+      SYNAPSE_POSTGRES_USER: postgres
+      SYNAPSE_POSTGRES_PASSWORD: postgres
+    working_dir: /app
+    volumes:
+      - ..:/app

.buildkite/docker-compose.py37.pg11.yaml

@@ -0,0 +1,21 @@
+version: '3.1'
+
+services:
+
+  postgres:
+    image: postgres:11
+    environment:
+      POSTGRES_PASSWORD: postgres
+
+  testenv:
+    image: python:3.7
+    depends_on:
+      - postgres
+    env_file: .env
+    environment:
+      SYNAPSE_POSTGRES_HOST: postgres
+      SYNAPSE_POSTGRES_USER: postgres
+      SYNAPSE_POSTGRES_PASSWORD: postgres
+    working_dir: /app
+    volumes:
+      - ..:/app

.buildkite/docker-compose.py37.pg95.yaml

@@ -0,0 +1,21 @@
+version: '3.1'
+
+services:
+
+  postgres:
+    image: postgres:9.5
+    environment:
+      POSTGRES_PASSWORD: postgres
+
+  testenv:
+    image: python:3.7
+    depends_on:
+      - postgres
+    env_file: .env
+    environment:
+      SYNAPSE_POSTGRES_HOST: postgres
+      SYNAPSE_POSTGRES_USER: postgres
+      SYNAPSE_POSTGRES_PASSWORD: postgres
+    working_dir: /app
+    volumes:
+      - ..:/app

.buildkite/pipeline.yml

@@ -0,0 +1,157 @@
+env:
+  CODECOV_TOKEN: "2dd7eb9b-0eda-45fe-a47c-9b5ac040045f"
+
+steps:
+  - command:
+      - "python -m pip install tox"
+      - "tox -e pep8"
+    label: "\U0001F9F9 PEP-8"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e packaging"
+    label: "\U0001F9F9 packaging"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e check_isort"
+    label: "\U0001F9F9 isort"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+
+  - command:
+      - "python -m pip install tox"
+      - "scripts-dev/check-newsfragment"
+    label: ":newspaper: Newsfile"
+    branches: "!master !develop !release-*"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+          propagate-environment: true
+
+  - wait
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e check-sampleconfig"
+    label: "\U0001F9F9 check-sample-config"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e py27,codecov"
+    label: ":python: 2.7 / SQLite"
+    env:
+      TRIAL_FLAGS: "-j 2"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:2.7"
+          propagate-environment: true
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e py35,codecov"
+    label: ":python: 3.5 / SQLite"
+    env:
+      TRIAL_FLAGS: "-j 2"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.5"
+          propagate-environment: true
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e py36,codecov"
+    label: ":python: 3.6 / SQLite"
+    env:
+      TRIAL_FLAGS: "-j 2"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+          propagate-environment: true
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e py37,codecov"
+    label: ":python: 3.7 / SQLite"
+    env:
+      TRIAL_FLAGS: "-j 2"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.7"
+          propagate-environment: true
+
+  - label: ":python: 2.7 / :postgres: 9.4"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py27-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py27.pg94.yaml
+
+  - label: ":python: 2.7 / :postgres: 9.5"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py27-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py27.pg95.yaml
+
+  - label: ":python: 3.5 / :postgres: 9.4"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py35.pg94.yaml
+
+  - label: ":python: 3.5 / :postgres: 9.5"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py35.pg95.yaml
+
+  - label: ":python: 3.7 / :postgres: 9.5"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py37.pg95.yaml
+
+  - label: ":python: 3.7 / :postgres: 11"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py37.pg11.yaml

.circleci/config.yml

@@ -8,6 +8,7 @@ jobs:
       - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py2
       - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
   dockerhubuploadlatest:
     machine: true
@@ -17,6 +18,7 @@ jobs:
       - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest -t matrixdotorg/synapse:latest-py3 --build-arg PYTHON_VERSION=3.6 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       - run: docker push matrixdotorg/synapse:latest
+      - run: docker push matrixdotorg/synapse:latest-py2
       - run: docker push matrixdotorg/synapse:latest-py3
   sytestpy2:
     docker:

.gitignore

@@ -1,64 +1,36 @@
-*.pyc
-.*.swp
+# filename patterns
 *~
+.*.swp
+.#*
+*.deb
+*.egg
+*.egg-info
 *.lock
-
-.DS_Store
+*.pyc
+*.tac
 _trial_temp/
 _trial_temp*/
-logs/
-dbs/
-*.egg
-dist/
-docs/build/
-*.egg-info
-pip-wheel-metadata/
 
-cmdclient_config.json
-homeserver*.db
-homeserver*.log
-homeserver*.log.*
-homeserver*.pid
-/homeserver*.yaml
+# stuff that is likely to exist when you run a server locally
+/*.signing.key
+/*.tls.crt
+/*.tls.key
+/uploads
+/media_store/
 
-*.signing.key
-*.tls.crt
-*.tls.dh
-*.tls.key
+# IDEs
+/.idea/
+/.ropeproject/
+/.vscode/
 
-.coverage*
-coverage.*
-!.coveragerc
-htmlcov
+# build products
+/.coverage*
+!/.coveragerc
+/.tox
+/build/
+/coverage.*
+/dist/
+/docs/build/
+/htmlcov
+/pip-wheel-metadata/
 
-demo/*/*.db
-demo/*/*.log
-demo/*/*.log.*
-demo/*/*.pid
-demo/media_store.*
-demo/etc
-
-uploads
-cache
-
-.idea/
-media_store/
-
-*.tac
-
-build/
-venv/
-venv*/
-*venv/
-
-localhost-800*/
-static/client/register/register_config.js
-.tox
-
-env/
-*.config
-
-.vscode/
-.ropeproject/
-*.deb
-/debs

.travis.yml

@@ -1,83 +0,0 @@
-sudo: false
-language: python
-
-cache:
-  directories:
-    # we only bother to cache the wheels; parts of the http cache get
-    # invalidated every build (because they get served with a max-age of 600
-    # seconds), which means that we end up re-uploading the whole cache for
-    # every build, which is time-consuming In any case, it's not obvious that
-    # downloading the cache from S3 would be much faster than downloading the
-    # originals from pypi.
-    #
-    - $HOME/.cache/pip/wheels
-
-addons:
-  postgresql: "9.4"
-
-# don't clone the whole repo history, one commit will do
-git:
-  depth: 1
-
-# only build branches we care about (PRs are built seperately)
-branches:
-  only:
-    - master
-    - develop
-    - /^release-v/
-
-# When running the tox environments that call Twisted Trial, we can pass the -j
-# flag to run the tests concurrently. We set this to 2 for CPU bound tests
-# (SQLite) and 4 for I/O bound tests (PostgreSQL).
-matrix:
-  fast_finish: true
-  include:
-  - python: 2.7
-    env: TOX_ENV=packaging
-
-  - python: 3.6
-    env: TOX_ENV="pep8,check_isort"
-
-  - python: 2.7
-    env: TOX_ENV=py27,codecov TRIAL_FLAGS="-j 2"
-
-  - python: 2.7
-    env: TOX_ENV=py27-old TRIAL_FLAGS="-j 2"
-
-  - python: 2.7
-    env: TOX_ENV=py27-postgres,codecov TRIAL_FLAGS="-j 4"
-    services:
-      - postgresql
-
-  - python: 3.5
-    env: TOX_ENV=py35,codecov TRIAL_FLAGS="-j 2"
-
-  - python: 3.6
-    env: TOX_ENV=py36,codecov TRIAL_FLAGS="-j 2"
-
-  - python: 3.6
-    env: TOX_ENV=py36-postgres,codecov TRIAL_FLAGS="-j 4"
-    services:
-      - postgresql
-
-  - # we only need to check for the newsfragment if it's a PR build
-    if: type = pull_request
-    python: 3.6
-    env: TOX_ENV=check-newsfragment
-    script:
-      - git remote set-branches --add origin develop
-      - git fetch origin develop
-      - tox -e $TOX_ENV
-
-install:
-  - pip install tox
-  
-  # if we don't have python3.6 in this environment, travis unhelpfully gives us
-  # a `python3.6` on our path which does nothing but spit out a warning. Tox
-  # tries to run it (even if we're not running a py36 env), so the build logs
-  # then have warnings which look like errors. To reduce the noise, remove the
-  # non-functional python3.6.
-  - ( ! command -v python3.6 || python3.6 --version ) &>/dev/null || rm -f $(command -v python3.6)
-
-script:
-  - tox -e $TOX_ENV

CHANGES.md

@@ -1,3 +1,118 @@
+Synapse 0.99.2 (2019-03-01)
+===========================
+
+Features
+--------
+
+- Added an HAProxy example in the reverse proxy documentation. Contributed by Benoît S. (“Benpro”). ([\#4541](https://github.com/matrix-org/synapse/issues/4541))
+- Add basic optional sentry integration. ([\#4632](https://github.com/matrix-org/synapse/issues/4632), [\#4694](https://github.com/matrix-org/synapse/issues/4694))
+- Transfer bans on room upgrade. ([\#4642](https://github.com/matrix-org/synapse/issues/4642))
+- Add configurable room list publishing rules. ([\#4647](https://github.com/matrix-org/synapse/issues/4647))
+- Support .well-known delegation when issuing certificates through ACME. ([\#4652](https://github.com/matrix-org/synapse/issues/4652))
+- Allow registration and login to be handled by a worker instance. ([\#4666](https://github.com/matrix-org/synapse/issues/4666), [\#4670](https://github.com/matrix-org/synapse/issues/4670), [\#4682](https://github.com/matrix-org/synapse/issues/4682))
+- Reduce the overhead of creating outbound federation connections over TLS by caching the TLS client options. ([\#4674](https://github.com/matrix-org/synapse/issues/4674))
+- Add prometheus metrics for number of outgoing EDUs, by type. ([\#4695](https://github.com/matrix-org/synapse/issues/4695))
+- Return correct error code when inviting a remote user to a room whose homeserver does not support the room version. ([\#4721](https://github.com/matrix-org/synapse/issues/4721))
+- Prevent showing rooms to other servers that were set to not federate. ([\#4746](https://github.com/matrix-org/synapse/issues/4746))
+
+
+Bugfixes
+--------
+
+- Fix possible exception when paginating. ([\#4263](https://github.com/matrix-org/synapse/issues/4263))
+- The dependency checker now correctly reports a version mismatch for optional
+  dependencies, instead of reporting the dependency missing. ([\#4450](https://github.com/matrix-org/synapse/issues/4450))
+- Set CORS headers on .well-known requests. ([\#4651](https://github.com/matrix-org/synapse/issues/4651))
+- Fix kicking guest users on guest access revocation in worker mode. ([\#4667](https://github.com/matrix-org/synapse/issues/4667))
+- Fix an issue in the database migration script where the
+  `e2e_room_keys.is_verified` column wasn't considered as
+  a boolean. ([\#4680](https://github.com/matrix-org/synapse/issues/4680))
+- Fix TaskStopped exceptions in logs when outbound requests time out. ([\#4690](https://github.com/matrix-org/synapse/issues/4690))
+- Fix ACME config for python 2. ([\#4717](https://github.com/matrix-org/synapse/issues/4717))
+- Fix paginating over federation persisting incorrect state. ([\#4718](https://github.com/matrix-org/synapse/issues/4718))
+
+
+Internal Changes
+----------------
+
+- Run `black` to reformat user directory code. ([\#4635](https://github.com/matrix-org/synapse/issues/4635))
+- Reduce number of exceptions we log. ([\#4643](https://github.com/matrix-org/synapse/issues/4643), [\#4668](https://github.com/matrix-org/synapse/issues/4668))
+- Introduce upsert batching functionality in the database layer. ([\#4644](https://github.com/matrix-org/synapse/issues/4644))
+- Fix various spelling mistakes. ([\#4657](https://github.com/matrix-org/synapse/issues/4657))
+- Cleanup request exception logging. ([\#4669](https://github.com/matrix-org/synapse/issues/4669), [\#4737](https://github.com/matrix-org/synapse/issues/4737), [\#4738](https://github.com/matrix-org/synapse/issues/4738))
+- Improve replication performance by reducing cache invalidation traffic. ([\#4671](https://github.com/matrix-org/synapse/issues/4671), [\#4715](https://github.com/matrix-org/synapse/issues/4715), [\#4748](https://github.com/matrix-org/synapse/issues/4748))
+- Test against Postgres 9.5 as well as 9.4. ([\#4676](https://github.com/matrix-org/synapse/issues/4676))
+- Run unit tests against python 3.7. ([\#4677](https://github.com/matrix-org/synapse/issues/4677))
+- Attempt to clarify installation instructions/config. ([\#4681](https://github.com/matrix-org/synapse/issues/4681))
+- Clean up gitignores. ([\#4688](https://github.com/matrix-org/synapse/issues/4688))
+- Minor tweaks to acme docs. ([\#4689](https://github.com/matrix-org/synapse/issues/4689))
+- Improve the logging in the pusher process. ([\#4691](https://github.com/matrix-org/synapse/issues/4691))
+- Better checks on newsfragments. ([\#4698](https://github.com/matrix-org/synapse/issues/4698), [\#4750](https://github.com/matrix-org/synapse/issues/4750))
+- Avoid some redundant work when processing read receipts. ([\#4706](https://github.com/matrix-org/synapse/issues/4706))
+- Run `push_receipts_to_remotes` as background job. ([\#4707](https://github.com/matrix-org/synapse/issues/4707))
+- Add prometheus metrics for number of badge update pushes. ([\#4709](https://github.com/matrix-org/synapse/issues/4709))
+- Reduce pusher logging on startup ([\#4716](https://github.com/matrix-org/synapse/issues/4716))
+- Don't log exceptions when failing to fetch remote server keys. ([\#4722](https://github.com/matrix-org/synapse/issues/4722))
+- Correctly proxy exception in frontend_proxy worker. ([\#4723](https://github.com/matrix-org/synapse/issues/4723))
+- Add database version to phonehome stats. ([\#4753](https://github.com/matrix-org/synapse/issues/4753))
+
+
+Synapse 0.99.1.1 (2019-02-14)
+=============================
+
+Bugfixes
+--------
+
+- Fix "TypeError: '>' not supported" when starting without an existing certificate.
+  Fix a bug where an existing certificate would be reprovisoned every day. ([\#4648](https://github.com/matrix-org/synapse/issues/4648))
+
+
+Synapse 0.99.1 (2019-02-14)
+===========================
+
+Features
+--------
+
+- Include m.room.encryption on invites by default ([\#3902](https://github.com/matrix-org/synapse/issues/3902))
+- Federation OpenID listener resource can now be activated even if federation is disabled ([\#4420](https://github.com/matrix-org/synapse/issues/4420))
+- Synapse's ACME support will now correctly reprovision a certificate that approaches its expiry while Synapse is running. ([\#4522](https://github.com/matrix-org/synapse/issues/4522))
+- Add ability to update backup versions ([\#4580](https://github.com/matrix-org/synapse/issues/4580))
+- Allow the "unavailable" presence status for /sync.
+  This change makes Synapse compliant with r0.4.0 of the Client-Server specification. ([\#4592](https://github.com/matrix-org/synapse/issues/4592))
+- There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners ([\#4613](https://github.com/matrix-org/synapse/issues/4613), [\#4615](https://github.com/matrix-org/synapse/issues/4615), [\#4617](https://github.com/matrix-org/synapse/issues/4617), [\#4636](https://github.com/matrix-org/synapse/issues/4636))
+- The default configuration no longer requires TLS certificates. ([\#4614](https://github.com/matrix-org/synapse/issues/4614))
+
+
+Bugfixes
+--------
+
+- Copy over room federation ability on room upgrade. ([\#4530](https://github.com/matrix-org/synapse/issues/4530))
+- Fix noisy "twisted.internet.task.TaskStopped" errors in logs ([\#4546](https://github.com/matrix-org/synapse/issues/4546))
+- Synapse is now tolerant of the `tls_fingerprints` option being None or not specified. ([\#4589](https://github.com/matrix-org/synapse/issues/4589))
+- Fix 'no unique or exclusion constraint' error ([\#4591](https://github.com/matrix-org/synapse/issues/4591))
+- Transfer Server ACLs on room upgrade. ([\#4608](https://github.com/matrix-org/synapse/issues/4608))
+- Fix failure to start when not TLS certificate was given even if TLS was disabled. ([\#4618](https://github.com/matrix-org/synapse/issues/4618))
+- Fix self-signed cert notice from generate-config. ([\#4625](https://github.com/matrix-org/synapse/issues/4625))
+- Fix performance of `user_ips` table deduplication background update ([\#4626](https://github.com/matrix-org/synapse/issues/4626), [\#4627](https://github.com/matrix-org/synapse/issues/4627))
+
+
+Internal Changes
+----------------
+
+- Change the user directory state query to use a filtered call to the db instead of a generic one. ([\#4462](https://github.com/matrix-org/synapse/issues/4462))
+- Reject federation transactions if they include more than 50 PDUs or 100 EDUs. ([\#4513](https://github.com/matrix-org/synapse/issues/4513))
+- Reduce duplication of ``synapse.app`` code. ([\#4567](https://github.com/matrix-org/synapse/issues/4567))
+- Fix docker upload job to push -py2 images. ([\#4576](https://github.com/matrix-org/synapse/issues/4576))
+- Add port configuration information to ACME instructions. ([\#4578](https://github.com/matrix-org/synapse/issues/4578))
+- Update MSC1711 FAQ to calrify .well-known usage ([\#4584](https://github.com/matrix-org/synapse/issues/4584))
+- Clean up default listener configuration ([\#4586](https://github.com/matrix-org/synapse/issues/4586))
+- Clarifications for reverse proxy docs ([\#4607](https://github.com/matrix-org/synapse/issues/4607))
+- Move ClientTLSOptionsFactory init out of `refresh_certificates` ([\#4611](https://github.com/matrix-org/synapse/issues/4611))
+- Fail cleanly if listener config lacks a 'port' ([\#4616](https://github.com/matrix-org/synapse/issues/4616))
+- Remove redundant entries from docker config ([\#4619](https://github.com/matrix-org/synapse/issues/4619))
+- README updates ([\#4621](https://github.com/matrix-org/synapse/issues/4621))
+
+
 Synapse 0.99.0 (2019-02-05)
 ===========================
 

CONTRIBUTING.rst

@@ -30,7 +30,7 @@ use github's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Travis CI 
+We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Travis CI
 <https://travis-ci.org/matrix-org/synapse>`_ for continuous integration. All
 pull requests to synapse get automatically tested by Travis and CircleCI.
 If your change breaks the build, this will be shown in GitHub, so please
@@ -74,16 +74,39 @@ entry. These are managed by Towncrier
 To create a changelog entry, make a new file in the ``changelog.d``
 file named in the format of ``PRnumber.type``. The type can be
 one of ``feature``, ``bugfix``, ``removal`` (also used for
-deprecations), or ``misc`` (for internal-only changes). The content of
-the file is your changelog entry, which can contain Markdown
-formatting. Adding credits to the changelog is encouraged, we value
-your contributions and would like to have you shouted out in the
-release notes!
+deprecations), or ``misc`` (for internal-only changes).
+
+The content of the file is your changelog entry, which can contain Markdown
+formatting. The entry should end with a full stop ('.') for consistency.
+
+Adding credits to the changelog is encouraged, we value your
+contributions and would like to have you shouted out in the release notes!
 
 For example, a fix in PR #1234 would have its changelog entry in
 ``changelog.d/1234.bugfix``, and contain content like "The security levels of
 Florbs are now validated when recieved over federation. Contributed by Jane
-Matrix".
+Matrix.".
+
+Debian changelog
+----------------
+
+Changes which affect the debian packaging files (in ``debian``) are an
+exception.
+
+In this case, you will need to add an entry to the debian changelog for the
+next release. For this, run the following command::
+
+  dch
+
+This will make up a new version number (if there isn't already an unreleased
+version in flight), and open an editor where you can add a new changelog entry.
+(Our release process will ensure that the version number and maintainer name is
+corrected for the release.)
+
+If your change affects both the debian packaging *and* files outside the debian
+directory, you will need both a regular newsfragment *and* an entry in the
+debian changelog. (Though typically such changes should be submitted as two
+separate pull requests.)
 
 Attribution
 ~~~~~~~~~~~

INSTALL.md

@@ -350,17 +350,33 @@ Once you have installed synapse as above, you will need to configure it.
 
 ## TLS certificates
 
-The default configuration exposes two HTTP ports: 8008 and 8448. Port 8008 is
-configured without TLS; it should be behind a reverse proxy for TLS/SSL
-termination on port 443 which in turn should be used for clients. Port 8448
-is configured to use TLS for Federation with a self-signed or verified
-certificate, but please be aware that a valid certificate will be required in
-Synapse v1.0. Instructions for having Synapse automatically provision and renew federation certificates through ACME can be found at [ACME.md](docs/ACME.md).
+The default configuration exposes a single HTTP port: http://localhost:8008. It
+is suitable for local testing, but for any practical use, you will either need
+to enable a reverse proxy, or configure Synapse to expose an HTTPS port.
 
-If you would like to use your own certificates, you can do so by changing
-`tls_certificate_path` and `tls_private_key_path` in `homeserver.yaml`;
-alternatively, you can use a reverse-proxy. Apart from port 8448 using TLS,
-both ports are the same in the default configuration.
+For information on using a reverse proxy, see
+[docs/reverse_proxy.rst](docs/reverse_proxy.rst).
+
+To configure Synapse to expose an HTTPS port, you will need to edit
+`homeserver.yaml`, as follows:
+
+* First, under the `listeners` section, uncomment the configuration for the
+  TLS-enabled listener. (Remove the hash sign (`#`) at the start of
+  each line). The relevant lines are like this:
+
+  ```
+    - port: 8448
+      type: http
+      tls: true
+      resources:
+        - names: [client, federation]
+  ```
+* You will also need to uncomment the `tls_certificate_path` and
+  `tls_private_key_path` lines under the `TLS` section. You can either
+  point these settings at an existing certificate and key, or you can
+  enable Synapse's built-in ACME (Let's Encrypt) support.  Instructions
+  for having Synapse automatically provision and renew federation 
+  certificates through ACME can be found at [ACME.md](docs/ACME.md).
 
 ## Registering a user
 
@@ -374,7 +390,7 @@ users. This can be done as follows:
 ```
 $ source ~/synapse/env/bin/activate
 $ synctl start # if not already running
-$ register_new_matrix_user -c homeserver.yaml https://localhost:8448
+$ register_new_matrix_user -c homeserver.yaml http://localhost:8008
 New user localpart: erikj
 Password:
 Confirm password:

MANIFEST.in

@@ -39,6 +39,7 @@ prune .circleci
 prune .coveragerc
 prune debian
 prune .codecov.yml
+prune .buildkite
 
 exclude jenkins*
 recursive-exclude jenkins *.sh

README.rst

@@ -26,7 +26,6 @@ via IRC bridge at irc://irc.freenode.net/matrix.
 Synapse is currently in rapid development, but as of version 0.5 we believe it
 is sufficiently stable to be run as an internet-facing service for real usage!
 
-
 About Matrix
 ============
 
@@ -88,18 +87,20 @@ Connecting to Synapse from a client
 ===================================
 
 The easiest way to try out your new Synapse installation is by connecting to it
-from a web client. The easiest option is probably the one at
-https://riot.im/app. You will need to specify a "Custom server" when you log on
-or register: set this to ``https://domain.tld`` if you setup a reverse proxy
-following the recommended setup, or ``https://localhost:8448`` - remember to specify the
-port (``:8448``) if not ``:443`` unless you changed the configuration. (Leave the identity
-server as the default - see `Identity servers`_.)
+from a web client.
 
-If using port 8448 you will run into errors if you are using a self-signed
-certificate. To overcome this, simply go to ``https://localhost:8448``
-directly with your browser and accept the presented certificate. You can then
-go back in your web client and proceed further. Valid federation certificates
-should not have this problem.
+Unless you are running a test instance of Synapse on your local machine, in
+general, you will need to enable TLS support before you can successfully
+connect from a client: see `<INSTALL.md#tls-certificates>`_.
+
+An easy way to get started is to login or register via Riot at 
+https://riot.im/app/#/login or https://riot.im/app/#/register respectively. 
+You will need to change the server you are logging into from ``matrix.org``
+and instead specify a Homeserver URL of ``https://<server_name>:8448`` 
+(or just ``https://<server_name>`` if you are using a reverse proxy). 
+(Leave the identity server as the default - see `Identity servers`_.) 
+If you prefer to use another client, refer to our 
+`client breakdown <https://matrix.org/docs/projects/clients-matrix>`_.
 
 If all goes well you should at least be able to log in, create a room, and
 start sending messages.
@@ -174,9 +175,32 @@ Separately, Synapse may leak file handles if inbound HTTP requests get stuck
 during processing - e.g. blocked behind a lock or talking to a remote server etc.
 This is best diagnosed by matching up the 'Received request' and 'Processed request'
 log lines and looking for any 'Processed request' lines which take more than
-a few seconds to execute.  Please let us know at #matrix-dev:matrix.org if
+a few seconds to execute.  Please let us know at #synapse:matrix.org if
 you see this failure mode so we can help debug it, however.
 
+Help!! Synapse eats all my RAM!
+-------------------------------
+
+Synapse's architecture is quite RAM hungry currently - we deliberately
+cache a lot of recent room data and metadata in RAM in order to speed up
+common requests.  We'll improve this in future, but for now the easiest
+way to either reduce the RAM usage (at the risk of slowing things down)
+is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment
+variable.  The default is 0.5, which can be decreased to reduce RAM usage
+in memory constrained enviroments, or increased if performance starts to
+degrade.
+
+Using `libjemalloc <http://jemalloc.net/>`_ can also yield a significant
+improvement in overall amount, and especially in terms of giving back RAM
+to the OS. To use it, the library must simply be put in the LD_PRELOAD
+environment variable when launching Synapse. On Debian, this can be done
+by installing the ``libjemalloc1`` package and adding this line to
+``/etc/default/matrix-synapse``::
+
+    LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
+
+This can make a significant difference on Python 2.7 - it's unclear how
+much of an improvement it provides on Python 3.x.
 
 Upgrading an existing Synapse
 =============================
@@ -196,12 +220,12 @@ Federation is the process by which users on different servers can participate
 in the same room. For this to work, those other servers must be able to contact
 yours to send messages.
 
-The ``server_name`` in your
-``homeserver.yaml`` file determines the way that other servers will reach
-yours. By default, they will treat it as a hostname and try to connect to
-port 8448. This is easy to set up and will work with the default configuration,
-provided you set the ``server_name`` to match your machine's public DNS
-hostname.
+The ``server_name`` in your ``homeserver.yaml`` file determines the way that
+other servers will reach yours. By default, they will treat it as a hostname
+and try to connect to port 8448. This is easy to set up and will work with the
+default configuration, provided you set the ``server_name`` to match your
+machine's public DNS hostname, and give Synapse a TLS certificate which is
+valid for your ``server_name``.
 
 For a more flexible configuration, you can set up a DNS SRV record. This allows
 you to run your server on a machine that might not have the same name as your
@@ -243,11 +267,8 @@ largest boxes pause for thought.)
 Troubleshooting
 ---------------
 
-You can use the federation tester to check if your homeserver is all set:
-``https://matrix.org/federationtester/api/report?server_name=<your_server_name>``
-If any of the attributes under "checks" is false, federation won't work.
-There is also a nicer interface available from a community member at
-`<https://neo.lain.haus/fed-tester>`_.
+You can use the `federation tester <https://matrix.org/federationtester>`_ to
+check if your homeserver is all set.
 
 The typical failure mode with federation is that when you try to join a room,
 it is rejected with "401: Unauthorized". Generally this means that other
@@ -263,6 +284,11 @@ So, things to check are:
   (it should be ``_matrix._tcp.<server_name>``), and that the port and hostname
   it specifies are reachable from outside your network.
 
+Another common problem is that people on other servers can't join rooms that
+you invite them to. This can be caused by an incorrectly-configured reverse
+proxy: see `<docs/reverse_proxy.rst>`_ for instructions on how to correctly
+configure a reverse proxy.
+
 Running a Demo Federation of Synapses
 -------------------------------------
 
@@ -290,7 +316,6 @@ The advantages of Postgres include:
 For information on how to install and use PostgreSQL, please see
 `docs/postgres.rst <docs/postgres.rst>`_.
 
-
 .. _reverse-proxy:
 
 Using a reverse proxy with Synapse
@@ -304,54 +329,7 @@ It is recommended to put a reverse proxy such as
 doing so is that it means that you can expose the default https port (443) to
 Matrix clients without needing to run Synapse with root privileges.
 
-The most important thing to know here is that Matrix clients and other Matrix
-servers do not necessarily need to connect to your server via the same
-port. Indeed, clients will use port 443 by default, whereas servers default to
-port 8448. Where these are different, we refer to the 'client port' and the
-'federation port'.
-
-All Matrix endpoints begin with ``/_matrix``, so an example nginx
-configuration for forwarding client connections to Synapse might look like::
-
-  server {
-      listen 443 ssl;
-      listen [::]:443 ssl;
-      server_name matrix.example.com;
-
-      location /_matrix {
-          proxy_pass http://localhost:8008;
-          proxy_set_header X-Forwarded-For $remote_addr;
-      }
-  }
-
-an example Caddy configuration might look like::
-
-    matrix.example.com {
-      proxy /_matrix http://localhost:8008 {
-        transparent
-      }
-    }
-
-and an example Apache configuration might look like::
-
-    <VirtualHost *:443>
-        SSLEngine on
-        ServerName matrix.example.com;
-
-        <Location /_matrix>
-            ProxyPass http://127.0.0.1:8008/_matrix nocanon
-            ProxyPassReverse http://127.0.0.1:8008/_matrix
-        </Location>
-    </VirtualHost>
-
-You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true``
-for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are
-recorded correctly.
-
-Having done so, you can then use ``https://matrix.example.com`` (instead of
-``https://matrix.example.com:8448``) as the "Custom server" when `Connecting to
-Synapse from a client`_.
-
+For information on configuring one, see `<docs/reverse_proxy.rst>`_.
 
 Identity Servers
 ================
@@ -409,7 +387,7 @@ Synapse Development
 
 Before setting up a development environment for synapse, make sure you have the
 system dependencies (such as the python header files) installed - see
-`Installing from source`_.
+`Installing from source <INSTALL.md#installing-from-source>`_.
 
 To check out a synapse for development, clone the git repo into a working
 directory of your choice::
@@ -420,7 +398,7 @@ directory of your choice::
 Synapse has a number of external dependencies, that are easiest
 to install using pip and a virtualenv::
 
-    virtualenv -p python2.7 env
+    virtualenv -p python3 env
     source env/bin/activate
     python -m pip install -e .[all]
 
@@ -462,25 +440,3 @@ sphinxcontrib-napoleon::
 Building internal API documentation::
 
     python setup.py build_sphinx
-
-
-Help!! Synapse eats all my RAM!
-===============================
-
-Synapse's architecture is quite RAM hungry currently - we deliberately
-cache a lot of recent room data and metadata in RAM in order to speed up
-common requests.  We'll improve this in future, but for now the easiest
-way to either reduce the RAM usage (at the risk of slowing things down)
-is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment
-variable.  The default is 0.5, which can be decreased to reduce RAM usage
-in memory constrained enviroments, or increased if performance starts to
-degrade.
-
-Using `libjemalloc <http://jemalloc.net/>`_ can also yield a significant
-improvement in overall amount, and especially in terms of giving back RAM
-to the OS. To use it, the library must simply be put in the LD_PRELOAD
-environment variable when launching Synapse. On Debian, this can be done
-by installing the ``libjemalloc1`` package and adding this line to
-``/etc/default/matrix-synapse``::
-
-    LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1

UPGRADE.rst

@@ -39,7 +39,7 @@ instructions that may be required are listed later in this document.
        ./synctl restart
 
 
-To check whether your update was sucessful, you can check the Server header
+To check whether your update was successful, you can check the Server header
 returned by the Client-Server API:
 
 .. code:: bash
@@ -57,7 +57,7 @@ will need to replace any self-signed certificates with those verified by a
 root CA. Information on how to do so can be found at `the ACME docs
 <docs/ACME.md>`_.
 
-For more information on configuring TLS certificates see the `FAQ <https://github.com/matrix-org/synapse/blob/master/docs/MSC1711_certificates_FAQ.md>`_
+For more information on configuring TLS certificates see the `FAQ <docs/MSC1711_certificates_FAQ.md>`_.
 
 Upgrading to v0.34.0
 ====================

changelog.d/4740.bugfix

@@ -0,0 +1 @@
+'event_id' is now a required parameter in federated state requests, as per the matrix spec.

changelog.d/4749.bugfix

@@ -0,0 +1 @@
+Fix tightloop over connecting to replication server.

changelog.d/4752.misc

@@ -0,0 +1 @@
+Change from TravisCI to Buildkite for CI.

changelog.d/4757.feature

@@ -0,0 +1 @@
+Move server key queries to federation reader.

changelog.d/4757.misc

@@ -0,0 +1 @@
+When presence is disabled don't send over replication.

changelog.d/4759.feature

@@ -0,0 +1 @@
+Add support for /account/3pid REST endpoint to client_reader worker.

changelog.d/4763.bugfix

@@ -0,0 +1 @@
+Fix parsing of Content-Disposition headers on remote media requests and URL previews.

changelog.d/4765.misc

@@ -0,0 +1 @@
+Minor docstring fixes for MatrixFederationAgent.

changelog.d/4770.misc

@@ -0,0 +1 @@
+Optimise EDU transmission for the federation_sender worker.

changelog.d/4771.misc

@@ -0,0 +1 @@
+Update test_typing to use HomeserverTestCase.

changelog.d/4776.bugfix

@@ -0,0 +1 @@
+Fix incorrect log about not persisting duplicate state event.

changelog.d/4790.bugfix

@@ -0,0 +1 @@
+Fix v4v6 option in HAProxy example config. Contributed by Flakebi.

changelog.d/4791.feature

@@ -0,0 +1 @@
+Include a default configuration file in the 'docs' directory.

changelog.d/4797.misc

@@ -0,0 +1 @@
+Clean up read-receipt handling.

changelog.d/4798.misc

@@ -0,0 +1 @@
+Add some debug about processing read receipts.

changelog.d/4801.feature

@@ -0,0 +1 @@
+Include a default configuration file in the 'docs' directory.

debian/changelog

@@ -1,3 +1,26 @@
+matrix-synapse-py3 (0.99.2) stable; urgency=medium
+
+  * Fix overwriting of config settings on upgrade.
+  * New synapse release 0.99.2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Fri, 01 Mar 2019 10:55:08 +0000
+
+matrix-synapse-py3 (0.99.1.1) stable; urgency=medium
+
+  * New synapse release 0.99.1.1
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 14 Feb 2019 17:19:44 +0000
+
+matrix-synapse-py3 (0.99.1) stable; urgency=medium
+
+  [ Damjan Georgievski ]
+  * Added ExecReload= in service unit file to send a HUP signal
+
+  [ Synapse Packaging team ]
+  * New synapse release 0.99.1
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 14 Feb 2019 14:12:26 +0000
+
 matrix-synapse-py3 (0.99.0) stable; urgency=medium
 
   * New synapse release 0.99.0

debian/install

@@ -1 +1,2 @@
 debian/log.yaml etc/matrix-synapse
+debian/manage_debconf.pl /opt/venvs/matrix-synapse/lib/

debian/manage_debconf.pl

@@ -0,0 +1,130 @@
+#!/usr/bin/perl
+#
+# Interface between our config files and the debconf database.
+#
+# Usage:
+#
+#   manage_debconf.pl <action>
+#
+# where <action> can be:
+#
+#   read:    read the configuration from the yaml into debconf
+#   update:  update the yaml config according to the debconf database
+use strict;
+use warnings;
+
+use Debconf::Client::ConfModule (qw/get set/);
+
+# map from the name of a setting in our .yaml file to the relevant debconf
+# setting.
+my %MAPPINGS=(
+    server_name => 'matrix-synapse/server-name',
+    report_stats => 'matrix-synapse/report-stats',
+);
+
+# enable debug if dpkg --debug
+my $DEBUG = $ENV{DPKG_MAINTSCRIPT_DEBUG};
+
+sub read_config {
+    my @files = @_;
+
+    foreach my $file (@files)  {
+        print STDERR "reading $file\n" if $DEBUG;
+
+        open my $FH, "<", $file or next;
+
+        # rudimentary parsing which (a) avoids having to depend on a yaml library,
+        # and (b) is tolerant of yaml errors
+        while($_ = <$FH>) {
+            while (my ($setting, $debconf) = each %MAPPINGS) {
+                $setting = quotemeta $setting;
+                if(/^${setting}\s*:(.*)$/) {
+                    my $val = $1;
+
+                    # remove leading/trailing whitespace
+                    $val =~ s/^\s*//;
+                    $val =~ s/\s*$//;
+
+                    # remove surrounding quotes
+                    if ($val =~ /^"(.*)"$/ || $val =~ /^'(.*)'$/) {
+                        $val = $1;
+                    }
+
+                    print STDERR ">> $debconf = $val\n" if $DEBUG;
+                    set($debconf, $val);
+                }
+            }
+        }
+        close $FH;
+    }
+}
+
+sub update_config {
+    my @files = @_;
+
+    my %substs = ();
+    while (my ($setting, $debconf) = each %MAPPINGS) {
+        my @res = get($debconf);
+        $substs{$setting} = $res[1] if $res[0] == 0;
+    }
+
+    foreach my $file (@files) {
+        print STDERR "checking $file\n" if $DEBUG;
+
+        open my $FH, "<", $file or next;
+
+        my $updated = 0;
+
+        # read the whole file into memory
+        my @lines = <$FH>;
+
+        while (my ($setting, $val) = each %substs) {
+            $setting = quotemeta $setting;
+
+            map {
+                if (/^${setting}\s*:\s*(.*)\s*$/) {
+                    my $current = $1;
+                    if ($val ne $current) {
+                        $_ = "${setting}: $val\n";
+                        $updated = 1;
+                    }
+                }
+            } @lines;
+        }
+        close $FH;
+
+        next unless $updated;
+
+        print STDERR "updating $file\n" if $DEBUG;
+        open $FH, ">", $file or die "unable to update $file";
+        print $FH @lines;
+        close $FH;
+    }
+}
+
+
+my $cmd = $ARGV[0];
+
+my $read = 0;
+my $update = 0;
+
+if (not $cmd) {
+    die "must specify a command to perform\n";
+} elsif ($cmd eq 'read') {
+    $read = 1;
+} elsif ($cmd eq 'update') {
+    $update = 1;
+} else {
+    die "unknown command '$cmd'\n";
+}
+
+my @files = (
+    "/etc/matrix-synapse/homeserver.yaml",
+    glob("/etc/matrix-synapse/conf.d/*.yaml"),
+);
+
+if ($read) {
+    read_config(@files);
+} elsif ($update) {
+    update_config(@files);
+}

debian/matrix-synapse-py3.config

@@ -4,6 +4,9 @@ set -e
 
 . /usr/share/debconf/confmodule
 
+# try to update the debconf db according to whatever is in the config files
+/opt/venvs/matrix-synapse/lib/manage_debconf.pl read || true
+
 db_input high matrix-synapse/server-name || true
 db_input high matrix-synapse/report-stats || true
 db_go

debian/matrix-synapse-py3.postinst

@@ -8,19 +8,36 @@ USER="matrix-synapse"
 
 case "$1" in
   configure|reconfigure)
-    # Set server name in config file
+
+    # generate template config files if they don't exist
     mkdir -p "/etc/matrix-synapse/conf.d/"
-    db_get matrix-synapse/server-name
+    if [ ! -e "$CONFIGFILE_SERVERNAME" ]; then
+        cat > "$CONFIGFILE_SERVERNAME" <<EOF
+# This file is autogenerated, and will be recreated on upgrade if it is deleted.
+# Any changes you make will be preserved.
 
-    if [ "$RET" ]; then
-        echo "server_name: $RET" > $CONFIGFILE_SERVERNAME
+# The domain name of the server, with optional explicit port.
+# This is used by remote servers to connect to this server,
+# e.g. matrix.org, localhost:8080, etc.
+# This is also the last part of your UserID.
+#
+server_name: ''
+EOF
     fi
 
-    db_get matrix-synapse/report-stats
-    if [ "$RET" ]; then
-        echo "report_stats: $RET" > $CONFIGFILE_REPORTSTATS
+    if [ ! -e "$CONFIGFILE_REPORTSTATS" ]; then
+        cat > "$CONFIGFILE_REPORTSTATS" <<EOF
+# This file is autogenerated, and will be recreated on upgrade if it is deleted.
+# Any changes you make will be preserved.
+
+# Whether to report anonymized homeserver usage statistics.
+report_stats: false
+EOF
     fi
 
+    # update the config files according to whatever is in the debconf database
+    /opt/venvs/matrix-synapse/lib/manage_debconf.pl update
+
     if ! getent passwd $USER >/dev/null; then
       adduser --quiet --system --no-create-home --home /var/lib/matrix-synapse $USER
     fi

debian/matrix-synapse.service

@@ -8,6 +8,7 @@ WorkingDirectory=/var/lib/matrix-synapse
 EnvironmentFile=/etc/default/matrix-synapse
 ExecStartPre=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys
 ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/
+ExecReload=/bin/kill -HUP $MAINPID
 Restart=always
 RestartSec=3
 

demo/.gitignore

@@ -0,0 +1,7 @@
+*.db
+*.log
+*.log.*
+*.pid
+
+/media_store.*
+/etc

docker/Dockerfile-dhvirtualenv

@@ -58,7 +58,11 @@ RUN apt-get update -qq -o Acquire::Languages=none \
         sqlite3
 
 COPY --from=builder /dh-virtualenv_1.1-1_all.deb /
-RUN apt-get install -yq /dh-virtualenv_1.1-1_all.deb
+
+# install dhvirtualenv. Update the apt cache again first, in case we got a
+# cached cache from docker the first time.
+RUN apt-get update -qq -o Acquire::Languages=none \
+    && apt-get install -yq /dh-virtualenv_1.1-1_all.deb
 
 WORKDIR /synapse/source
 ENTRYPOINT ["bash","/synapse/source/docker/build_debian.sh"]

docker/conf/dummy.tls.crt

@@ -1,17 +0,0 @@
------BEGIN CERTIFICATE-----
-MIICnTCCAYUCAgPoMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNVBAMMCWxvY2FsaG9z
-dDAeFw0xOTAxMTUwMDQxNTBaFw0yOTAxMTIwMDQxNTBaMBQxEjAQBgNVBAMMCWxv
-Y2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMKqm81/8j5d
-R1s7VZ8ueg12gJrPVCCAOkp0UnuC/ZlXhN0HTvnhQ+B0IlSgB4CcQZyf4jnA6o4M
-rwSc7VX0MPE9x/idoA0g/0WoC6tsxugOrvbzCw8Tv+fnXglm6uVc7aFPfx69wU3q
-lUHGD/8jtEoHxmCG177Pt2lHAfiVLBAyMQGtETzxt/yAfkloaybe316qoljgK5WK
-cokdAt9G84EEqxNeEnx5FG3Vc100bAqJS4GvQlFgtF9KFEqZKEyB1yKBpPMDfPIS
-V9hIV0gswSmYI8dpyBlGf5lPElY68ZGABmOQgr0RI5qHK/h28OpFPE0q3v4AMHgZ
-I36wii4NrAUCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAfD8kcpZ+dn08xh1qtKtp
-X+/YNZaOBIeVdlCzfoZKNblSFAFD/jCfObNJYvZMUQ8NX2UtEJp1lTA6m7ltSsdY
-gpC2k1VD8iN+ooXklJmL0kxc7UUqho8I0l9vn35h+lhLF0ihT6XfZVi/lDHWl+4G
-rG+v9oxvCSCWrNWLearSlFPtQQ8xPtOE0nLwfXtOI/H/2kOuC38ihaIWM4jjbWXK
-E/ksgUfuDv0mFiwf1YdBF5/M3/qOowqzU8HgMJ3WoT/9Po5Ya1pWc+3BcxxytUDf
-XdMu0tWHKX84tZxLcR1nZHzluyvFFM8xNtLi9xV0Z7WbfT76V0C/ulEOybGInYsv
-nQ==
------END CERTIFICATE-----

docker/conf/homeserver.yaml

@@ -2,13 +2,7 @@
 
 ## TLS ##
 
-{% if SYNAPSE_NO_TLS %}
-no_tls: True
-
-# workaround for https://github.com/matrix-org/synapse/issues/4554
-tls_certificate_path: "/conf/dummy.tls.crt"
-
-{% else %}
+{% if not SYNAPSE_NO_TLS %}
 
 tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt"
 tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key"

docs/.sample_config_header.yaml

@@ -0,0 +1,12 @@
+# The config is maintained as an up-to-date snapshot of the default
+# homeserver.yaml configuration generated by Synapse.
+#
+# It is intended to act as a reference for the default configuration,
+# helping admins keep track of new options and other changes, and compare
+# their configs with the current default.  As such, many of the actual
+# config values shown are placeholders.
+#
+# It is *not* intended to be copied and used as the basis for a real
+# homeserver.yaml. Instead, if you are starting from scratch, please generate
+# a fresh config using Synapse by following the instructions in INSTALL.md.
+

docs/ACME.md

@@ -10,13 +10,14 @@ through [Let's Encrypt](https://letsencrypt.org/) if you tell it to.
 
 In the case that your `server_name` config variable is the same as
 the hostname that the client connects to, then the same certificate can be
-used between client and federation ports without issue. 
+used between client and federation ports without issue.
 
-For a sample configuration, please inspect the new ACME section in the example
-generated config by running the `generate-config` executable. For example:
+If your configuration file does not already have an `acme` section, you can
+generate an example config by running the `generate_config` executable. For
+example:
 
 ```
-~/synapse/env3/bin/generate-config
+~/synapse/env3/bin/generate_config
 ```
 
 You will need to provide Let's Encrypt (or another ACME provider) access to
@@ -27,10 +28,9 @@ like `authbind` to allow Synapse to listen on port 80 without root access.
 (Do not run Synapse with root permissions!) Detailed instructions are
 available under "ACME setup" below.
 
-If you are already using self-signed certificates, you will need to back up
-or delete them (files `example.com.tls.crt` and `example.com.tls.key` in
-Synapse's root directory), Synapse's ACME implementation will not overwrite
-them.
+If you already have certificates, you will need to back up or delete them
+(files `example.com.tls.crt` and `example.com.tls.key` in Synapse's root
+directory), Synapse's ACME implementation will not overwrite them.
 
 You may wish to use alternate methods such as Certbot to obtain a certificate
 from Let's Encrypt, depending on your server configuration. Of course, if you
@@ -41,10 +41,10 @@ placed in Synapse's config directory without the need for any ACME setup.
 
 The main steps for enabling ACME support in short summary are:
 
-1. Allow Synapse to listen on port 80 with authbind, or forward it from a reverse-proxy.
-1. Set `acme:enabled` to `true` in homeserver.yaml.
+1. Allow Synapse to listen for incoming ACME challenges.
+1. Enable ACME support in `homeserver.yaml`.
 1. Move your old certificates (files `example.com.tls.crt` and `example.com.tls.key` out of the way if they currently exist at the paths specified in `homeserver.yaml`.
-1. Restart Synapse
+1. Restart Synapse.
 
 Detailed instructions for each step are provided below.
 
@@ -71,7 +71,7 @@ location /.well-known/acme-challenge {
 }
 ```
 
-For Apache, add the following to your existing webserver config::
+For Apache, add the following to your existing webserver config:
 
 ```
 ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge
@@ -79,6 +79,13 @@ ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-cha
 
 Make sure to restart/reload your webserver after making changes.
 
+Now make the relevant changes in `homeserver.yaml` to enable ACME support:
+
+```
+acme:
+    enabled: true
+    port: 8009
+```
 
 #### Authbind
 
@@ -102,25 +109,21 @@ sudo touch /etc/authbind/byport/80
 sudo chmod 777 /etc/authbind/byport/80
 ```
 
-When Synapse is started, use the following syntax::
+When Synapse is started, use the following syntax:
 
 ```
 authbind --deep <synapse start command>
 ```
 
-### Config file editing
-
-Once Synapse is able to listen on port 80 for ACME challenge
-requests, it must be told to perform ACME provisioning by setting `enabled`
-to true under the `acme` section in `homeserver.yaml`:
+Make the relevant changes in `homeserver.yaml` to enable ACME support:
 
 ```
 acme:
     enabled: true
 ```
 
-### Starting synapse
+### (Re)starting synapse
 
 Ensure that the certificate paths specified in `homeserver.yaml` (`tls_certificate_path` and `tls_private_key_path`) do not currently point to any files. Synapse will not provision certificates if files exist, as it does not want to overwrite existing certificates.
 
-Finally, start/restart Synapse.
+Finally, start/restart Synapse.

docs/MSC1711_certificates_FAQ.md

@@ -1,4 +1,4 @@
-# MSC 1711 Certificates FAQ
+# MSC1711 Certificates FAQ
 
 The goal of Synapse 0.99.0 is to act as a stepping stone to Synapse 1.0.0. It
 supports the r0.1 release of the server to server specification, but is
@@ -42,9 +42,9 @@ imminent Matrix 1.0 release, you can also see our
   * It used to work just fine, why are you breaking everything?
   * Can I manage my own certificates rather than having Synapse renew
     certificates itself?
-  * Do you still recommend against using a reverse-proxy on the federation port?
+  * Do you still recommend against using a reverse proxy on the federation port?
   * Do I still need to give my TLS certificates to Synapse if I am using a
-    reverse-proxy?
+    reverse proxy?
   * Do I need the same certificate for the client and federation port?
   * How do I tell Synapse to reload my keys/certificates after I replace them?
 
@@ -107,10 +107,10 @@ hosted at a target domain of `customer.example.net`. Currently you should have
 an SRV record which looks like:
 
 ```
-_matrix._tcp.example.com. IN SRV 10 5 443 customer.example.net.
+_matrix._tcp.example.com. IN SRV 10 5 8000 customer.example.net.
 ```
 
-In this situation, you have two choices for how to proceed:
+In this situation, you have three choices for how to proceed:
 
 #### Option 1: give Synapse a certificate for your matrix domain
 
@@ -123,12 +123,19 @@ doing one of the following:
    and `tls_private_key_path`, or:
 
  * Use Synapse's [ACME support](./ACME.md), and forward port 80 on the
-   `server_name` domain to your Synapse instance, or:
+   `server_name` domain to your Synapse instance.
 
- * Set up a reverse-proxy on port 8448 on the `server_name` domain, which
-   forwards to Synapse. Once it is set up, you can remove the SRV record.
+#### Option 2: run Synapse behind a reverse proxy
 
-#### Option 2: add a .well-known file to delegate your matrix traffic
+If you have an existing reverse proxy set up with correct TLS certificates for
+your domain, you can simply route all traffic through the reverse proxy by
+updating the SRV record appropriately (or removing it, if the proxy listens on
+8448).
+
+See [reverse_proxy.rst](reverse_proxy.rst) for information on setting up a
+reverse proxy.
+
+#### Option 3: add a .well-known file to delegate your matrix traffic
 
 This will allow you to keep Synapse on a separate domain, without having to
 give it a certificate for the matrix domain.
@@ -151,25 +158,32 @@ You can do this with a `.well-known` file as follows:
     `https://<server_name>/.well-known/matrix/server` with contents:
 
     ```json
-    {"m.server": "<target domain>:<port>"}
+    {"m.server": "<target server name>"}
     ```
 
-    In the above example, `https://example.com/.well-known/matrix/server`
-    should have the contents:
+    where the target server name is resolved as usual (i.e. SRV lookup, falling
+    back to talking to port 8448).
+
+    In the above example, where synapse is listening on port 8000,
+    `https://example.com/.well-known/matrix/server` should have `m.server` set to one of:
+
+    1. `customer.example.net` ─ with a SRV record on
+       `_matrix._tcp.customer.example.com` pointing to port 8000, or:
+
+    2. `customer.example.net` ─ updating synapse to listen on the default port
+       8448, or:
+
+    3. `customer.example.net:8000` ─ ensuring that if there is a reverse proxy
+       on `customer.example.net:8000` it correctly handles HTTP requests with
+       Host header set to `customer.example.net:8000`.
 
-    ```json
-	{"m.server": "customer.example.net:443"}
-    ```
 
 ## FAQ
 
 ### Synapse 0.99.0 has just been released, what do I need to do right now?
 
-Upgrade as soon as you can in preparation for Synapse 1.0.0.
-
-### How do I upgrade?
-
-Follow the upgrade notes here [UPGRADE.rst](https://github.com/matrix-org/synapse/blob/master/UPGRADE.rst)
+Upgrade as soon as you can in preparation for Synapse 1.0.0, and update your
+TLS certificates as [above](#configuring-certificates-for-compatibility-with-synapse-100).
 
 ### What will happen if I do not set up a valid federation certificate immediately?
 
@@ -186,39 +200,24 @@ homeserver will not be able to federate with any Synapse >= 1.0.0
 ### When do I need a SRV record or .well-known URI?
 
 If your homeserver listens on the default federation port (8448), and your
-server_name points to the host that your homeserver runs on, you do not need an
-SRV record or .well-known/matrix/server URI.\
-For instance, if you registered example.com and pointed its DNS A record at a
+`server_name` points to the host that your homeserver runs on, you do not need an
+SRV record or `.well-known/matrix/server` URI.
+
+For instance, if you registered `example.com` and pointed its DNS A record at a
 fresh Upcloud VPS or similar, you could install Synapse 0.99 on that host,
-giving it a server_name of example.com, and it would automatically generate a
+giving it a server_name of `example.com`, and it would automatically generate a
 valid TLS certificate for you via Let's Encrypt and no SRV record or
-.well-known URI would be needed.
+`.well-known` URI would be needed.
 
 This is the common case, although you can add an SRV record or
-.well-known/matrix/server URI for completeness if you wish.
+`.well-known/matrix/server` URI for completeness if you wish.
 
-**However**, if your server does not listen on port 8448, or if your server_name
+**However**, if your server does not listen on port 8448, or if your `server_name`
 does not point to the host that your homeserver runs on, you will need to let
 other servers know how to find it.
 
-The easiest way to do this is with a .well-known/matrix/server URI on the
-webroot of the domain to advertise your server. For instance, if you ran
-"matrixhosting.com" and you were hosting a Matrix server for `example.com`, you
-would ask `example.com` to create a file at
-`https://example.com/.well-known/matrix/server` with contents:
-
-```json
-{"m.server": "example.matrixhosting.com:8448"}
-```
-
-...which would tell servers trying to connect to example.com to instead connect
-to example.matrixhosting.com on port 8448. You would then configure Synapse
-with a server_name of "example.com", but generate a TLS certificate for
-example.matrixhosting.com.
-
-As an alternative, you can still use an SRV DNS record for the delegation, but
-this will require you to have a certificate for the matrix domain (example.com
-in this example). See "Can I still use an SRV record?".
+In this case, you should see ["If you do have an SRV record
+currently"](#if-you-do-have-an-srv-record-currently) above.
 
 ### Can I still use an SRV record?
 
@@ -244,13 +243,13 @@ also need to use a .well-known URI instead. However, see also "I have created a
 
 ### I have created a .well-known URI. Do I still need an SRV record?
 
-As of Synapse 0.99, Synapse will first check for the existence of a .well-known
-URL and follow any delegation it suggests. It will only then check for the
+As of Synapse 0.99, Synapse will first check for the existence of a `.well-known`
+URI and follow any delegation it suggests. It will only then check for the
 existence of an SRV record.
 
 That means that the SRV record will often be redundant. However, you should
 remember that there may still be older versions of Synapse in the federation
-which do not understand .well-known URIs, so if you removed your SRV record you
+which do not understand `.well-known` URIs, so if you removed your SRV record you
 would no longer be able to federate with them.
 
 It is therefore best to leave the SRV record in place for now. Synapse 0.34 and
@@ -301,17 +300,20 @@ attempt to obtain certificates from Let's Encrypt if you configure it to do
 so.The only requirement is that there is a valid TLS cert present for
 federation end points.
 
-### Do you still recommend against using a reverse-proxy on the federation port?
+### Do you still recommend against using a reverse proxy on the federation port?
 
 We no longer actively recommend against using a reverse proxy. Many admins will
-find it easier to direct federation traffic to a reverse-proxy and manage their
+find it easier to direct federation traffic to a reverse proxy and manage their
 own TLS certificates, and this is a supported configuration.
 
+See [reverse_proxy.rst](reverse_proxy.rst) for information on setting up a
+reverse proxy.
+
 ### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy?
 
 Practically speaking, this is no longer necessary.
 
-If you are using a reverse-proxy for all of your TLS traffic, then you can set
+If you are using a reverse proxy for all of your TLS traffic, then you can set
 `no_tls: True`. In that case, the only reason Synapse needs the certificate is
 to populate a legacy 'tls_fingerprints' field in the federation API. This is
 ignored by Synapse 0.99.0 and later, and the only time pre-0.99 Synapses will
@@ -325,12 +327,12 @@ this, you can give it any TLS certificate at all. This will be fixed soon.
 
 ### Do I need the same certificate for the client and federation port?
 
-No. There is nothing stopping you doing so, particularly if you are using a
-reverse-proxy. However, Synapse will use the same certificate on any ports
-where TLS is configured.
+No. There is nothing stopping you from using different certificates,
+particularly if you are using a reverse proxy. However, Synapse will use the
+same certificate on any ports where TLS is configured.
 
 ### How do I tell Synapse to reload my keys/certificates after I replace them?
 
 Synapse will reload the keys and certificates when it receives a SIGHUP - for
-example kill -HUP $(cat homeserver.pid). Alternatively, simply restart Synapse,
-though this will result in downtime while it restarts.
+example `kill -HUP $(cat homeserver.pid)`. Alternatively, simply restart
+Synapse, though this will result in downtime while it restarts.

docs/reverse_proxy.rst

@@ -0,0 +1,110 @@
+Using a reverse proxy with Synapse
+==================================
+
+It is recommended to put a reverse proxy such as
+`nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
+`Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_,
+`Caddy <https://caddyserver.com/docs/proxy>`_ or
+`HAProxy <https://www.haproxy.org/>`_ in front of Synapse. One advantage of
+doing so is that it means that you can expose the default https port (443) to
+Matrix clients without needing to run Synapse with root privileges.
+
+**NOTE**: Your reverse proxy must not 'canonicalise' or 'normalise' the
+requested URI in any way (for example, by decoding ``%xx`` escapes). Beware
+that Apache *will* canonicalise URIs unless you specifify ``nocanon``.
+
+When setting up a reverse proxy, remember that Matrix clients and other Matrix
+servers do not necessarily need to connect to your server via the same server
+name or port. Indeed, clients will use port 443 by default, whereas servers
+default to port 8448. Where these are different, we refer to the 'client port'
+and the 'federation port'. See `Setting up federation
+<../README.rst#setting-up-federation>`_ for more details of the algorithm used for
+federation connections.
+
+Let's assume that we expect clients to connect to our server at
+``https://matrix.example.com``, and other servers to connect at
+``https://example.com:8448``. Here are some example configurations:
+
+* nginx::
+
+      server {
+          listen 443 ssl;
+          listen [::]:443 ssl;
+          server_name matrix.example.com;
+
+          location /_matrix {
+              proxy_pass http://localhost:8008;
+              proxy_set_header X-Forwarded-For $remote_addr;
+          }
+      }
+
+      server {
+          listen 8448 ssl default_server;
+          listen [::]:8448 ssl default_server;
+          server_name example.com;
+
+          location / {
+              proxy_pass http://localhost:8008;
+              proxy_set_header X-Forwarded-For $remote_addr;
+          }
+      }
+
+* Caddy::
+
+      matrix.example.com {
+        proxy /_matrix http://localhost:8008 {
+          transparent
+        }
+      }
+
+      example.com:8448 {
+        proxy / http://localhost:8008 {
+          transparent
+        }
+      }
+
+* Apache (note the ``nocanon`` options here!)::
+
+      <VirtualHost *:443>
+          SSLEngine on
+          ServerName matrix.example.com;
+
+          <Location /_matrix>
+              ProxyPass http://127.0.0.1:8008/_matrix nocanon
+              ProxyPassReverse http://127.0.0.1:8008/_matrix
+          </Location>
+      </VirtualHost>
+
+      <VirtualHost *:8448>
+          SSLEngine on
+          ServerName example.com;
+
+          <Location /_matrix>
+              ProxyPass http://127.0.0.1:8008/_matrix nocanon
+              ProxyPassReverse http://127.0.0.1:8008/_matrix
+          </Location>
+      </VirtualHost>
+
+* HAProxy::
+
+      frontend https
+        bind :::443 v4v6 ssl crt /etc/ssl/haproxy/ strict-sni alpn h2,http/1.1
+
+        # Matrix client traffic
+        acl matrix hdr(host) -i matrix.example.com
+        use_backend matrix if matrix
+
+      frontend matrix-federation
+        bind :::8448 v4v6 ssl crt /etc/ssl/haproxy/synapse.pem alpn h2,http/1.1
+        default_backend matrix
+
+      backend matrix
+        server matrix 127.0.0.1:8008
+
+You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true``
+for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are
+recorded correctly.
+
+Having done so, you can then use ``https://matrix.example.com`` (instead of
+``https://matrix.example.com:8448``) as the "Custom server" when connecting to
+Synapse from a client.

docs/tcp_replication.rst

@@ -137,7 +137,6 @@ for each stream so that on reconneciton it can start streaming from the correct
 place. Note: not all RDATA have valid tokens due to batching. See
 ``RdataCommand`` for more details.
 
-
 Example
 ~~~~~~~
 
@@ -189,7 +188,9 @@ RDATA (S)
     A single update in a stream
 
 POSITION (S)
-    The position of the stream has been updated
+    The position of the stream has been updated. Sent to the client after all
+    missing updates for a stream have been sent to the client and they're now
+    up to date.
 
 ERROR (S, C)
     There was an error
@@ -221,3 +222,28 @@ SYNC (S, C)
 
 See ``synapse/replication/tcp/commands.py`` for a detailed description and the
 format of each command.
+
+
+Cache Invalidation Stream
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The cache invalidation stream is used to inform workers when they need to
+invalidate any of their caches in the data store. This is done by streaming all
+cache invalidations done on master down to the workers, assuming that any caches
+on the workers also exist on the master.
+
+Each individual cache invalidation results in a row being sent down replication,
+which includes the cache name (the name of the function) and they key to
+invalidate. For example::
+
+    > RDATA caches 550953771 ["get_user_by_id", ["@bob:example.com"], 1550574873251]
+
+However, there are times when a number of caches need to be invalidated at the
+same time with the same key. To reduce traffic we batch those invalidations into
+a single poke by defining a special cache name that workers understand to mean
+to expand to invalidate the correct caches.
+
+Currently the special cache names are declared in ``synapse/storage/_base.py``
+and are:
+
+1. ``cs_cache_fake`` ─ invalidates caches that depend on the current state

docs/workers.rst

@@ -26,9 +26,8 @@ Configuration
 To make effective use of the workers, you will need to configure an HTTP
 reverse-proxy such as nginx or haproxy, which will direct incoming requests to
 the correct worker, or to the main synapse instance. Note that this includes
-requests made to the federation port. The caveats regarding running a
-reverse-proxy on the federation port still apply (see
-https://github.com/matrix-org/synapse/blob/master/README.rst#reverse-proxying-the-federation-port).
+requests made to the federation port. See `<reverse_proxy.rst>`_ for
+information on setting up a reverse proxy.
 
 To enable workers, you need to add two replication listeners to the master
 synapse, e.g.::
@@ -183,6 +182,7 @@ endpoints matching the following regular expressions::
     ^/_matrix/federation/v1/event_auth/
     ^/_matrix/federation/v1/exchange_third_party_invite/
     ^/_matrix/federation/v1/send/
+    ^/_matrix/key/v2/query
 
 The above endpoints should all be routed to the federation_reader worker by the
 reverse-proxy configuration.
@@ -223,6 +223,14 @@ following regular expressions::
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/context/.*$
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$
+    ^/_matrix/client/(api/v1|r0|unstable)/login$
+    ^/_matrix/client/(api/v1|r0|unstable)/account/3pid$
+
+Additionally, the following REST endpoints can be handled, but all requests must
+be routed to the same instance::
+
+    ^/_matrix/client/(r0|unstable)/register$
+
 
 ``synapse.app.user_dir``
 ~~~~~~~~~~~~~~~~~~~~~~~~

scripts-dev/check-newsfragment

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# A script which checks that an appropriate news file has been added on this
+# branch.
+
+set -e
+
+# make sure that origin/develop is up to date
+git remote set-branches --add origin develop
+git fetch --depth=1 origin develop
+
+UPSTREAM=origin/develop
+
+# if there are changes in the debian directory, check that the debian changelog
+# has been updated
+if ! git diff --quiet $UPSTREAM... -- debian; then
+    if git diff --quiet $UPSTREAM... -- debian/changelog; then
+        echo "Updates to debian directory, but no update to the changelog." >&2
+        exit 1
+    fi
+fi
+
+# if there are changes *outside* the debian directory, check that the
+# newsfragments have been updated.
+if git diff --name-only $UPSTREAM... | grep -qv '^develop/'; then
+    tox -e check-newsfragment
+fi
+
+echo
+echo "--------------------------"
+echo
+
+# check that any new newsfiles on this branch end with a full stop.
+for f in `git diff --name-only $UPSTREAM... -- changelog.d`; do
+    lastchar=`tr -d '\n' < $f | tail -c 1`
+    if [ $lastchar != '.' ]; then
+        echo -e "\e[31mERROR: newsfragment $f does not end with a '.'\e[39m" >&2
+        exit 1
+    fi
+done
+

scripts-dev/generate_sample_config

@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Update/check the docs/sample_config.yaml
+
+set -e
+
+cd `dirname $0`/..
+
+SAMPLE_CONFIG="docs/sample_config.yaml"
+
+if [ "$1" == "--check" ]; then
+    diff -u "$SAMPLE_CONFIG" <(./scripts/generate_config --header-file docs/.sample_config_header.yaml) >/dev/null || {
+        echo -e "\e[1m\e[31m$SAMPLE_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config\`.\e[0m" >&2
+        exit 1
+    }
+else
+    ./scripts/generate_config --header-file docs/.sample_config_header.yaml -o "$SAMPLE_CONFIG"
+fi

scripts/generate_config

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 import argparse
+import shutil
 import sys
 
 from synapse.config.homeserver import HomeServerConfig
@@ -50,6 +51,13 @@ if __name__ == "__main__":
         help="File to write the configuration to. Default: stdout",
     )
 
+    parser.add_argument(
+        "--header-file",
+        type=argparse.FileType('r'),
+        help="File from which to read a header, which will be printed before the "
+             "generated config.",
+    )
+
     args = parser.parse_args()
 
     report_stats = args.report_stats
@@ -64,4 +72,7 @@ if __name__ == "__main__":
         report_stats=report_stats,
     )
 
+    if args.header_file:
+        shutil.copyfileobj(args.header_file, args.output_file)
+
     args.output_file.write(conf)

scripts/synapse_port_db

@@ -53,6 +53,7 @@ BOOLEAN_COLUMNS = {
     "group_summary_users": ["is_public"],
     "group_roles": ["is_public"],
     "local_group_membership": ["is_publicised", "is_admin"],
+    "e2e_room_keys": ["is_verified"],
 }
 
 

synapse/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd
+# Copyright 2018-9 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,4 +27,4 @@ try:
 except ImportError:
     pass
 
-__version__ = "0.99.0"
+__version__ = "0.99.2"

synapse/api/constants.py

@@ -73,6 +73,7 @@ class EventTypes(object):
     RoomHistoryVisibility = "m.room.history_visibility"
     CanonicalAlias = "m.room.canonical_alias"
     RoomAvatar = "m.room.avatar"
+    RoomEncryption = "m.room.encryption"
     GuestAccess = "m.room.guest_access"
 
     # These are used for validation

synapse/app/_base.py

@@ -15,19 +15,38 @@
 
 import gc
 import logging
+import signal
 import sys
+import traceback
 
 import psutil
 from daemonize import Daemonize
 
 from twisted.internet import error, reactor
+from twisted.protocols.tls import TLSMemoryBIOFactory
 
+import synapse
 from synapse.app import check_bind_error
+from synapse.crypto import context_factory
 from synapse.util import PreserveLoggingContext
 from synapse.util.rlimit import change_resource_limit
+from synapse.util.versionstring import get_version_string
 
 logger = logging.getLogger(__name__)
 
+_sighup_callbacks = []
+
+
+def register_sighup(func):
+    """
+    Register a function to be called when a SIGHUP occurs.
+
+    Args:
+        func (function): Function to be called when sent a SIGHUP signal.
+            Will be called with a single argument, the homeserver.
+    """
+    _sighup_callbacks.append(func)
+
 
 def start_worker_reactor(appname, config):
     """ Run the reactor in the main process
@@ -136,9 +155,8 @@ def listen_metrics(bind_addresses, port):
     from prometheus_client import start_http_server
 
     for host in bind_addresses:
-        reactor.callInThread(start_http_server, int(port),
-                             addr=host, registry=RegistryProxy)
-        logger.info("Metrics now reporting on %s:%d", host, port)
+        logger.info("Starting metrics listener on %s:%d", host, port)
+        start_http_server(port, addr=host, registry=RegistryProxy)
 
 
 def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50):
@@ -146,21 +164,23 @@ def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50):
     Create a TCP socket for a port and several addresses
 
     Returns:
-        list (empty)
+        list[twisted.internet.tcp.Port]: listening for TCP connections
     """
+    r = []
     for address in bind_addresses:
         try:
-            reactor.listenTCP(
-                port,
-                factory,
-                backlog,
-                address
+            r.append(
+                reactor.listenTCP(
+                    port,
+                    factory,
+                    backlog,
+                    address
+                )
             )
         except error.CannotListenError as e:
             check_bind_error(e, address, bind_addresses)
 
-    logger.info("Synapse now listening on TCP port %d", port)
-    return []
+    return r
 
 
 def listen_ssl(
@@ -187,5 +207,102 @@ def listen_ssl(
         except error.CannotListenError as e:
             check_bind_error(e, address, bind_addresses)
 
-    logger.info("Synapse now listening on port %d (TLS)", port)
     return r
+
+
+def refresh_certificate(hs):
+    """
+    Refresh the TLS certificates that Synapse is using by re-reading them from
+    disk and updating the TLS context factories to use them.
+    """
+
+    if not hs.config.has_tls_listener():
+        # attempt to reload the certs for the good of the tls_fingerprints
+        hs.config.read_certificate_from_disk(require_cert_and_key=False)
+        return
+
+    hs.config.read_certificate_from_disk(require_cert_and_key=True)
+    hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config)
+
+    if hs._listening_services:
+        logger.info("Updating context factories...")
+        for i in hs._listening_services:
+            # When you listenSSL, it doesn't make an SSL port but a TCP one with
+            # a TLS wrapping factory around the factory you actually want to get
+            # requests. This factory attribute is public but missing from
+            # Twisted's documentation.
+            if isinstance(i.factory, TLSMemoryBIOFactory):
+                addr = i.getHost()
+                logger.info(
+                    "Replacing TLS context factory on [%s]:%i", addr.host, addr.port,
+                )
+                # We want to replace TLS factories with a new one, with the new
+                # TLS configuration. We do this by reaching in and pulling out
+                # the wrappedFactory, and then re-wrapping it.
+                i.factory = TLSMemoryBIOFactory(
+                    hs.tls_server_context_factory,
+                    False,
+                    i.factory.wrappedFactory
+                )
+        logger.info("Context factories updated.")
+
+
+def start(hs, listeners=None):
+    """
+    Start a Synapse server or worker.
+
+    Args:
+        hs (synapse.server.HomeServer)
+        listeners (list[dict]): Listener configuration ('listeners' in homeserver.yaml)
+    """
+    try:
+        # Set up the SIGHUP machinery.
+        if hasattr(signal, "SIGHUP"):
+            def handle_sighup(*args, **kwargs):
+                for i in _sighup_callbacks:
+                    i(hs)
+
+            signal.signal(signal.SIGHUP, handle_sighup)
+
+            register_sighup(refresh_certificate)
+
+        # Load the certificate from disk.
+        refresh_certificate(hs)
+
+        # It is now safe to start your Synapse.
+        hs.start_listening(listeners)
+        hs.get_datastore().start_profiling()
+
+        setup_sentry(hs)
+    except Exception:
+        traceback.print_exc(file=sys.stderr)
+        reactor = hs.get_reactor()
+        if reactor.running:
+            reactor.stop()
+        sys.exit(1)
+
+
+def setup_sentry(hs):
+    """Enable sentry integration, if enabled in configuration
+
+    Args:
+        hs (synapse.server.HomeServer)
+    """
+
+    if not hs.config.sentry_enabled:
+        return
+
+    import sentry_sdk
+    sentry_sdk.init(
+        dsn=hs.config.sentry_dsn,
+        release=get_version_string(synapse),
+    )
+
+    # We set some default tags that give some context to this instance
+    with sentry_sdk.configure_scope() as scope:
+        scope.set_tag("matrix_server_name", hs.config.server_name)
+
+        app = hs.config.worker_app if hs.config.worker_app else "synapse.app.homeserver"
+        name = hs.config.worker_name if hs.config.worker_name else "master"
+        scope.set_tag("worker_app", app)
+        scope.set_tag("worker_name", name)

synapse/app/appservice.py

@@ -168,12 +168,7 @@ def start(config_options):
     )
 
     ps.setup()
-    ps.start_listening(config.worker_listeners)
-
-    def start():
-        ps.get_datastore().start_profiling()
-
-    reactor.callWhenRunning(start)
+    reactor.callWhenRunning(_base.start, ps, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-appservice", config)
 

synapse/app/client_reader.py

@@ -25,7 +25,6 @@ from synapse.app import _base
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
-from synapse.crypto import context_factory
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
 from synapse.metrics import RegistryProxy
@@ -41,6 +40,7 @@ from synapse.replication.slave.storage.registration import SlavedRegistrationSto
 from synapse.replication.slave.storage.room import RoomStore
 from synapse.replication.slave.storage.transactions import SlavedTransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
+from synapse.rest.client.v1.login import LoginRestServlet
 from synapse.rest.client.v1.room import (
     JoinedRoomMemberListRestServlet,
     PublicRoomListRestServlet,
@@ -48,6 +48,8 @@ from synapse.rest.client.v1.room import (
     RoomMemberListRestServlet,
     RoomStateRestServlet,
 )
+from synapse.rest.client.v2_alpha.account import ThreepidRestServlet
+from synapse.rest.client.v2_alpha.register import RegisterRestServlet
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
 from synapse.util.httpresourcetree import create_resource_tree
@@ -93,6 +95,9 @@ class ClientReaderServer(HomeServer):
                     JoinedRoomMemberListRestServlet(self).register(resource)
                     RoomStateRestServlet(self).register(resource)
                     RoomEventContextServlet(self).register(resource)
+                    RegisterRestServlet(self).register(resource)
+                    LoginRestServlet(self).register(resource)
+                    ThreepidRestServlet(self).register(resource)
 
                     resources.update({
                         "/_matrix/client/r0": resource,
@@ -173,17 +178,7 @@ def start(config_options):
     )
 
     ss.setup()
-
-    def start():
-        ss.config.read_certificate_from_disk()
-        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
-        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
-            config
-        )
-        ss.start_listening(config.worker_listeners)
-        ss.get_datastore().start_profiling()
-
-    reactor.callWhenRunning(start)
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-client-reader", config)
 

synapse/app/event_creator.py

@@ -25,7 +25,6 @@ from synapse.app import _base
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
-from synapse.crypto import context_factory
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
 from synapse.metrics import RegistryProxy
@@ -194,17 +193,7 @@ def start(config_options):
     )
 
     ss.setup()
-
-    def start():
-        ss.config.read_certificate_from_disk()
-        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
-        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
-            config
-        )
-        ss.start_listening(config.worker_listeners)
-        ss.get_datastore().start_profiling()
-
-    reactor.callWhenRunning(start)
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-event-creator", config)
 

synapse/app/federation_reader.py

@@ -21,12 +21,11 @@ from twisted.web.resource import NoResource
 
 import synapse
 from synapse import events
-from synapse.api.urls import FEDERATION_PREFIX
+from synapse.api.urls import FEDERATION_PREFIX, SERVER_KEY_V2_PREFIX
 from synapse.app import _base
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
-from synapse.crypto import context_factory
 from synapse.federation.transport.server import TransportLayerServer
 from synapse.http.site import SynapseSite
 from synapse.metrics import RegistryProxy
@@ -41,9 +40,11 @@ from synapse.replication.slave.storage.profile import SlavedProfileStore
 from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
 from synapse.replication.slave.storage.pushers import SlavedPusherStore
 from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
+from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.replication.slave.storage.room import RoomStore
 from synapse.replication.slave.storage.transactions import SlavedTransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
+from synapse.rest.key.v2 import KeyApiV2Resource
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
 from synapse.util.httpresourcetree import create_resource_tree
@@ -63,6 +64,7 @@ class FederationReaderSlavedStore(
     SlavedReceiptsStore,
     SlavedEventStore,
     SlavedKeyStore,
+    SlavedRegistrationStore,
     RoomStore,
     DirectoryStore,
     SlavedTransactionStore,
@@ -87,6 +89,19 @@ class FederationReaderServer(HomeServer):
                     resources.update({
                         FEDERATION_PREFIX: TransportLayerServer(self),
                     })
+                if name == "openid" and "federation" not in res["names"]:
+                    # Only load the openid resource separately if federation resource
+                    # is not specified since federation resource includes openid
+                    # resource.
+                    resources.update({
+                        FEDERATION_PREFIX: TransportLayerServer(
+                            self,
+                            servlet_groups=["openid"],
+                        ),
+                    })
+
+                if name in ["keys", "federation"]:
+                    resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self)
 
         root_resource = create_resource_tree(resources, NoResource())
 
@@ -99,7 +114,8 @@ class FederationReaderServer(HomeServer):
                 listener_config,
                 root_resource,
                 self.version_string,
-            )
+            ),
+            reactor=self.get_reactor()
         )
 
         logger.info("Synapse federation reader now listening on port %d", port)
@@ -160,17 +176,7 @@ def start(config_options):
     )
 
     ss.setup()
-
-    def start():
-        ss.config.read_certificate_from_disk()
-        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
-        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
-            config
-        )
-        ss.start_listening(config.worker_listeners)
-        ss.get_datastore().start_profiling()
-
-    reactor.callWhenRunning(start)
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-federation-reader", config)
 

synapse/app/federation_sender.py

@@ -25,7 +25,6 @@ from synapse.app import _base
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
-from synapse.crypto import context_factory
 from synapse.federation import send_queue
 from synapse.http.site import SynapseSite
 from synapse.metrics import RegistryProxy
@@ -192,17 +191,8 @@ def start(config_options):
     )
 
     ss.setup()
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
-    def start():
-        ss.config.read_certificate_from_disk()
-        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
-        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
-            config
-        )
-        ss.start_listening(config.worker_listeners)
-        ss.get_datastore().start_profiling()
-
-    reactor.callWhenRunning(start)
     _base.start_worker_reactor("synapse-federation-sender", config)
 
 

synapse/app/frontend_proxy.py

@@ -21,12 +21,11 @@ from twisted.web.resource import NoResource
 
 import synapse
 from synapse import events
-from synapse.api.errors import SynapseError
+from synapse.api.errors import HttpResponseException, SynapseError
 from synapse.app import _base
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
-from synapse.crypto import context_factory
 from synapse.http.server import JsonResource
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
 from synapse.http.site import SynapseSite
@@ -67,10 +66,15 @@ class PresenceStatusStubServlet(ClientV1RestServlet):
         headers = {
             "Authorization": auth_headers,
         }
-        result = yield self.http_client.get_json(
-            self.main_uri + request.uri.decode('ascii'),
-            headers=headers,
-        )
+
+        try:
+            result = yield self.http_client.get_json(
+                self.main_uri + request.uri.decode('ascii'),
+                headers=headers,
+            )
+        except HttpResponseException as e:
+            raise e.to_synapse_error()
+
         defer.returnValue((200, result))
 
     @defer.inlineCallbacks
@@ -250,17 +254,7 @@ def start(config_options):
     )
 
     ss.setup()
-
-    def start():
-        ss.config.read_certificate_from_disk()
-        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
-        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
-            config
-        )
-        ss.start_listening(config.worker_listeners)
-        ss.get_datastore().start_profiling()
-
-    reactor.callWhenRunning(start)
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-frontend-proxy", config)
 

synapse/app/homeserver.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,12 +15,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+
 import gc
 import logging
 import os
-import signal
 import sys
-import traceback
 
 from six import iteritems
 
@@ -28,7 +29,7 @@ from prometheus_client import Gauge
 
 from twisted.application import service
 from twisted.internet import defer, reactor
-from twisted.protocols.tls import TLSMemoryBIOFactory
+from twisted.python.failure import Failure
 from twisted.web.resource import EncodingResourceWrapper, NoResource
 from twisted.web.server import GzipEncoderFactory
 from twisted.web.static import File
@@ -49,7 +50,6 @@ from synapse.app import _base
 from synapse.app._base import listen_ssl, listen_tcp, quit_with_error
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
-from synapse.crypto import context_factory
 from synapse.federation.transport.server import TransportLayerServer
 from synapse.http.additional_resource import AdditionalResource
 from synapse.http.server import RootRedirect
@@ -86,7 +86,6 @@ def gz_wrap(r):
 
 class SynapseHomeServer(HomeServer):
     DATASTORE_CLASS = DataStore
-    _listening_services = []
 
     def _listener_http(self, config, listener_config):
         port = listener_config["port"]
@@ -94,14 +93,13 @@ class SynapseHomeServer(HomeServer):
         tls = listener_config.get("tls", False)
         site_tag = listener_config.get("tag", port)
 
-        if tls and config.no_tls:
-            raise ConfigError(
-                "Listener on port %i has TLS enabled, but no_tls is set" % (port,),
-            )
-
         resources = {}
         for res in listener_config["resources"]:
             for name in res["names"]:
+                if name == "openid" and "federation" in res["names"]:
+                    # Skip loading openid resource if federation is defined
+                    # since federation resource will include openid
+                    continue
                 resources.update(self._configure_named_resource(
                     name, res.get("compress", False),
                 ))
@@ -126,7 +124,7 @@ class SynapseHomeServer(HomeServer):
         root_resource = create_resource_tree(resources, root_resource)
 
         if tls:
-            return listen_ssl(
+            ports = listen_ssl(
                 bind_addresses,
                 port,
                 SynapseSite(
@@ -137,10 +135,12 @@ class SynapseHomeServer(HomeServer):
                     self.version_string,
                 ),
                 self.tls_server_context_factory,
+                reactor=self.get_reactor(),
             )
+            logger.info("Synapse now listening on TCP port %d (TLS)", port)
 
         else:
-            return listen_tcp(
+            ports = listen_tcp(
                 bind_addresses,
                 port,
                 SynapseSite(
@@ -149,8 +149,12 @@ class SynapseHomeServer(HomeServer):
                     listener_config,
                     root_resource,
                     self.version_string,
-                )
+                ),
+                reactor=self.get_reactor(),
             )
+            logger.info("Synapse now listening on TCP port %d", port)
+
+        return ports
 
     def _configure_named_resource(self, name, compress=False):
         """Build a resource map for a named resource
@@ -196,6 +200,11 @@ class SynapseHomeServer(HomeServer):
                 FEDERATION_PREFIX: TransportLayerServer(self),
             })
 
+        if name == "openid":
+            resources.update({
+                FEDERATION_PREFIX: TransportLayerServer(self, servlet_groups=["openid"]),
+            })
+
         if name in ["static", "client"]:
             resources.update({
                 STATIC_PREFIX: File(
@@ -241,10 +250,10 @@ class SynapseHomeServer(HomeServer):
 
         return resources
 
-    def start_listening(self):
+    def start_listening(self, listeners):
         config = self.get_config()
 
-        for listener in config.listeners:
+        for listener in listeners:
             if listener["type"] == "http":
                 self._listening_services.extend(
                     self._listener_http(config, listener)
@@ -260,14 +269,14 @@ class SynapseHomeServer(HomeServer):
                     )
                 )
             elif listener["type"] == "replication":
-                bind_addresses = listener["bind_addresses"]
-                for address in bind_addresses:
-                    factory = ReplicationStreamProtocolFactory(self)
-                    server_listener = reactor.listenTCP(
-                        listener["port"], factory, interface=address
-                    )
+                services = listen_tcp(
+                    listener["bind_addresses"],
+                    listener["port"],
+                    ReplicationStreamProtocolFactory(self),
+                )
+                for s in services:
                     reactor.addSystemEventTrigger(
-                        "before", "shutdown", server_listener.stopListening,
+                        "before", "shutdown", s.stopListening,
                     )
             elif listener["type"] == "metrics":
                 if not self.get_config().enable_metrics:
@@ -328,20 +337,11 @@ def setup(config_options):
         # generating config files and shouldn't try to continue.
         sys.exit(0)
 
-    sighup_callbacks = []
     synapse.config.logger.setup_logging(
         config,
-        use_worker_options=False,
-        register_sighup=sighup_callbacks.append
+        use_worker_options=False
     )
 
-    def handle_sighup(*args, **kwargs):
-        for i in sighup_callbacks:
-            i(*args, **kwargs)
-
-    if hasattr(signal, "SIGHUP"):
-        signal.signal(signal.SIGHUP, handle_sighup)
-
     events.USE_FROZEN_DICTS = config.use_frozen_dicts
 
     database_engine = create_engine(config.database_config)
@@ -377,76 +377,77 @@ def setup(config_options):
 
     hs.setup()
 
-    def refresh_certificate(*args):
+    @defer.inlineCallbacks
+    def do_acme():
         """
-        Refresh the TLS certificates that Synapse is using by re-reading them
-        from disk and updating the TLS context factories to use them.
+        Reprovision an ACME certificate, if it's required.
+
+        Returns:
+            Deferred[bool]: Whether the cert has been updated.
         """
-        logging.info("Reloading certificate from disk...")
-        hs.config.read_certificate_from_disk()
-        hs.tls_server_context_factory = context_factory.ServerContextFactory(config)
-        hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
-            config
+        acme = hs.get_acme_handler()
+
+        # Check how long the certificate is active for.
+        cert_days_remaining = hs.config.is_disk_cert_valid(
+            allow_self_signed=False
         )
-        logging.info("Certificate reloaded.")
 
-        logging.info("Updating context factories...")
-        for i in hs._listening_services:
-            if isinstance(i.factory, TLSMemoryBIOFactory):
-                i.factory = TLSMemoryBIOFactory(
-                    hs.tls_server_context_factory,
-                    False,
-                    i.factory.wrappedFactory
-                )
-        logging.info("Context factories updated.")
+        # We want to reprovision if cert_days_remaining is None (meaning no
+        # certificate exists), or the days remaining number it returns
+        # is less than our re-registration threshold.
+        provision = False
 
-    sighup_callbacks.append(refresh_certificate)
+        if (
+            cert_days_remaining is None or
+            cert_days_remaining < hs.config.acme_reprovision_threshold
+        ):
+            provision = True
+
+        if provision:
+            yield acme.provision_certificate()
+
+        defer.returnValue(provision)
+
+    @defer.inlineCallbacks
+    def reprovision_acme():
+        """
+        Provision a certificate from ACME, if required, and reload the TLS
+        certificate if it's renewed.
+        """
+        reprovisioned = yield do_acme()
+        if reprovisioned:
+            _base.refresh_certificate(hs)
 
     @defer.inlineCallbacks
     def start():
         try:
-            # Check if the certificate is still valid.
-            cert_days_remaining = hs.config.is_disk_cert_valid()
-
+            # Run the ACME provisioning code, if it's enabled.
             if hs.config.acme_enabled:
-                # If ACME is enabled, we might need to provision a certificate
-                # before starting.
                 acme = hs.get_acme_handler()
-
                 # Start up the webservices which we will respond to ACME
-                # challenges with.
+                # challenges with, and then provision.
                 yield acme.start_listening()
+                yield do_acme()
 
-                # We want to reprovision if cert_days_remaining is None (meaning no
-                # certificate exists), or the days remaining number it returns
-                # is less than our re-registration threshold.
-                if (cert_days_remaining is None) or (
-                    not cert_days_remaining > hs.config.acme_reprovision_threshold
-                ):
-                    yield acme.provision_certificate()
+                # Check if it needs to be reprovisioned every day.
+                hs.get_clock().looping_call(
+                    reprovision_acme,
+                    24 * 60 * 60 * 1000
+                )
 
-            # Read the certificate from disk and build the context factories for
-            # TLS.
-            hs.config.read_certificate_from_disk()
-            hs.tls_server_context_factory = context_factory.ServerContextFactory(config)
-            hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
-                config
-            )
+            _base.start(hs, config.listeners)
 
-            # It is now safe to start your Synapse.
-            hs.start_listening()
             hs.get_pusherpool().start()
-            hs.get_datastore().start_profiling()
             hs.get_datastore().start_doing_background_updates()
-        except Exception as e:
-            # If a DeferredList failed (like in listening on the ACME listener),
-            # we need to print the subfailure explicitly.
-            if isinstance(e, defer.FirstError):
-                e.subFailure.printTraceback(sys.stderr)
-                sys.exit(1)
+        except Exception:
+            # Print the exception and bail out.
+            print("Error during startup:", file=sys.stderr)
 
-            # Something else went wrong when starting. Print it and bail out.
-            traceback.print_exc(file=sys.stderr)
+            # this gives better tracebacks than traceback.print_exc()
+            Failure().printTraceback(file=sys.stderr)
+
+            if reactor.running:
+                reactor.stop()
             sys.exit(1)
 
     reactor.callWhenRunning(start)
@@ -455,7 +456,8 @@ def setup(config_options):
 
 
 class SynapseService(service.Service):
-    """A twisted Service class that will start synapse. Used to run synapse
+    """
+    A twisted Service class that will start synapse. Used to run synapse
     via twistd and a .tac.
     """
     def __init__(self, config):
@@ -553,6 +555,9 @@ def run(hs):
                 stats["memory_rss"] += process.memory_info().rss
                 stats["cpu_average"] += int(process.cpu_percent(interval=None))
 
+        stats["database_engine"] = hs.get_datastore().database_engine_name
+        stats["database_server_version"] = hs.get_datastore().get_server_version()
+
         logger.info("Reporting stats to matrix.org: %s" % (stats,))
         try:
             yield hs.get_simple_http_client().put_json(

synapse/app/media_repository.py

@@ -26,7 +26,6 @@ from synapse.app import _base
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
-from synapse.crypto import context_factory
 from synapse.http.site import SynapseSite
 from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
@@ -160,17 +159,7 @@ def start(config_options):
     )
 
     ss.setup()
-
-    def start():
-        ss.config.read_certificate_from_disk()
-        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
-        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
-            config
-        )
-        ss.start_listening(config.worker_listeners)
-        ss.get_datastore().start_profiling()
-
-    reactor.callWhenRunning(start)
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-media-repository", config)
 

synapse/app/pusher.py

@@ -224,11 +224,10 @@ def start(config_options):
     )
 
     ps.setup()
-    ps.start_listening(config.worker_listeners)
 
     def start():
+        _base.start(ps, config.worker_listeners)
         ps.get_pusherpool().start()
-        ps.get_datastore().start_profiling()
 
     reactor.callWhenRunning(start)
 

synapse/app/synchrotron.py

@@ -445,12 +445,7 @@ def start(config_options):
     )
 
     ss.setup()
-    ss.start_listening(config.worker_listeners)
-
-    def start():
-        ss.get_datastore().start_profiling()
-
-    reactor.callWhenRunning(start)
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-synchrotron", config)
 

synapse/app/user_dir.py

@@ -26,7 +26,6 @@ from synapse.app import _base
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
-from synapse.crypto import context_factory
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
 from synapse.metrics import RegistryProxy
@@ -220,17 +219,7 @@ def start(config_options):
     )
 
     ss.setup()
-
-    def start():
-        ss.config.read_certificate_from_disk()
-        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
-        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
-            config
-        )
-        ss.start_listening(config.worker_listeners)
-        ss.get_datastore().start_profiling()
-
-    reactor.callWhenRunning(start)
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-user-dir", config)
 

synapse/config/_base.py

@@ -180,9 +180,7 @@ class Config(object):
         Returns:
             str: the yaml config file
         """
-        default_config = "# vim:ft=yaml\n"
-
-        default_config += "\n\n".join(
+        default_config = "\n\n".join(
             dedent(conf)
             for conf in self.invoke_all(
                 "default_config",
@@ -257,7 +255,7 @@ class Config(object):
             "--keys-directory",
             metavar="DIRECTORY",
             help="Used with 'generate-*' options to specify where files such as"
-            " certs and signing keys should be stored in, unless explicitly"
+            " signing keys should be stored, unless explicitly"
             " specified in the config.",
         )
         config_parser.add_argument(
@@ -297,32 +295,34 @@ class Config(object):
                         "Must specify a server_name to a generate config for."
                         " Pass -H server.name."
                     )
+
+                config_str = obj.generate_config(
+                    config_dir_path=config_dir_path,
+                    data_dir_path=os.getcwd(),
+                    server_name=server_name,
+                    report_stats=(config_args.report_stats == "yes"),
+                    generate_secrets=True,
+                )
+
                 if not cls.path_exists(config_dir_path):
                     os.makedirs(config_dir_path)
                 with open(config_path, "w") as config_file:
-                    config_str = obj.generate_config(
-                        config_dir_path=config_dir_path,
-                        data_dir_path=os.getcwd(),
-                        server_name=server_name,
-                        report_stats=(config_args.report_stats == "yes"),
-                        generate_secrets=True,
+                    config_file.write(
+                        "# vim:ft=yaml\n\n"
                     )
-                    config = yaml.load(config_str)
-                    obj.invoke_all("generate_files", config)
                     config_file.write(config_str)
+
+                config = yaml.load(config_str)
+                obj.invoke_all("generate_files", config)
+
                 print(
                     (
                         "A config file has been generated in %r for server name"
-                        " %r with corresponding SSL keys and self-signed"
-                        " certificates. Please review this file and customise it"
+                        " %r. Please review this file and customise it"
                         " to your needs."
                     )
                     % (config_path, server_name)
                 )
-                print(
-                    "If this server name is incorrect, you will need to"
-                    " regenerate the SSL certificates"
-                )
                 return
             else:
                 print(

synapse/config/api.py

@@ -24,6 +24,7 @@ class ApiConfig(Config):
             EventTypes.JoinRules,
             EventTypes.CanonicalAlias,
             EventTypes.RoomAvatar,
+            EventTypes.RoomEncryption,
             EventTypes.Name,
         ])
 
@@ -32,9 +33,11 @@ class ApiConfig(Config):
         ## API Configuration ##
 
         # A list of event types that will be included in the room_invite_state
+        #
         room_invite_state_types:
             - "{JoinRules}"
             - "{CanonicalAlias}"
             - "{RoomAvatar}"
+            - "{RoomEncryption}"
             - "{Name}"
         """.format(**vars(EventTypes))

synapse/config/appservice.py

@@ -38,10 +38,12 @@ class AppServiceConfig(Config):
     def default_config(cls, **kwargs):
         return """\
         # A list of application service config file to use
+        #
         app_service_config_files: []
 
         # Whether or not to track application service IP addresses. Implicitly
         # enables MAU tracking for application service users.
+        #
         track_appservice_user_ips: False
         """
 

synapse/config/captcha.py

@@ -30,19 +30,22 @@ class CaptchaConfig(Config):
         # See docs/CAPTCHA_SETUP for full details of configuring this.
 
         # This Home Server's ReCAPTCHA public key.
+        #
         recaptcha_public_key: "YOUR_PUBLIC_KEY"
 
         # This Home Server's ReCAPTCHA private key.
+        #
         recaptcha_private_key: "YOUR_PRIVATE_KEY"
 
         # Enables ReCaptcha checks when registering, preventing signup
         # unless a captcha is answered. Requires a valid ReCaptcha
         # public/private key.
+        #
         enable_registration_captcha: False
 
         # A secret key used to bypass the captcha test entirely.
         #captcha_bypass_secret: "YOUR_SECRET_HERE"
 
         # The API endpoint to use for verifying m.login.recaptcha responses.
-        recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify"
+        recaptcha_siteverify_api: "https://www.recaptcha.net/recaptcha/api/siteverify"
         """

synapse/config/cas.py

@@ -38,6 +38,7 @@ class CasConfig(Config):
     def default_config(self, config_dir_path, server_name, **kwargs):
         return """
         # Enable CAS for registration and login.
+        #
         #cas_config:
         #   enabled: true
         #   server_url: "https://cas-server.com"

synapse/config/consent_config.py

@@ -54,20 +54,20 @@ DEFAULT_CONFIG = """\
 # for an account. Has no effect unless `require_at_registration` is enabled.
 # Defaults to "Privacy Policy".
 #
-# user_consent:
-#   template_dir: res/templates/privacy
-#   version: 1.0
-#   server_notice_content:
-#     msgtype: m.text
-#     body: >-
-#       To continue using this homeserver you must review and agree to the
-#       terms and conditions at %(consent_uri)s
-#   send_server_notice_to_guests: True
-#   block_events_error: >-
-#     To continue using this homeserver you must review and agree to the
-#     terms and conditions at %(consent_uri)s
-#   require_at_registration: False
-#   policy_name: Privacy Policy
+#user_consent:
+#  template_dir: res/templates/privacy
+#  version: 1.0
+#  server_notice_content:
+#    msgtype: m.text
+#    body: >-
+#      To continue using this homeserver you must review and agree to the
+#      terms and conditions at %(consent_uri)s
+#  send_server_notice_to_guests: True
+#  block_events_error: >-
+#    To continue using this homeserver you must review and agree to the
+#    terms and conditions at %(consent_uri)s
+#  require_at_registration: False
+#  policy_name: Privacy Policy
 #
 """
 

synapse/config/database.py

@@ -49,7 +49,8 @@ class DatabaseConfig(Config):
     def default_config(self, data_dir_path, **kwargs):
         database_path = os.path.join(data_dir_path, "homeserver.db")
         return """\
-        # Database configuration
+        ## Database ##
+
         database:
           # The database engine name
           name: "sqlite3"

synapse/config/groups.py

@@ -24,9 +24,11 @@ class GroupsConfig(Config):
     def default_config(self, **kwargs):
         return """\
         # Whether to allow non server admins to create groups on this server
+        #
         enable_group_creation: false
 
         # If enabled, non server admins can only create groups with local parts
         # starting with this prefix
-        # group_creation_prefix: "unofficial/"
+        #
+        #group_creation_prefix: "unofficial/"
         """

synapse/config/homeserver.py

@@ -42,7 +42,7 @@ from .voip import VoipConfig
 from .workers import WorkerConfig
 
 
-class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig,
+class HomeServerConfig(ServerConfig, TlsConfig, DatabaseConfig, LoggingConfig,
                        RatelimitConfig, ContentRepositoryConfig, CaptchaConfig,
                        VoipConfig, RegistrationConfig, MetricsConfig, ApiConfig,
                        AppServiceConfig, KeyConfig, SAML2Config, CasConfig,

synapse/config/jwt_config.py

@@ -46,8 +46,8 @@ class JWTConfig(Config):
         return """\
         # The JWT needs to contain a globally unique "sub" (subject) claim.
         #
-        # jwt_config:
-        #    enabled: true
-        #    secret: "a secret"
-        #    algorithm: "HS256"
+        #jwt_config:
+        #   enabled: true
+        #   secret: "a secret"
+        #   algorithm: "HS256"
         """

synapse/config/key.py

@@ -40,7 +40,7 @@ class KeyConfig(Config):
     def read_config(self, config):
         self.signing_key = self.read_signing_key(config["signing_key_path"])
         self.old_signing_keys = self.read_old_signing_keys(
-            config["old_signing_keys"]
+            config.get("old_signing_keys", {})
         )
         self.key_refresh_interval = self.parse_duration(
             config["key_refresh_interval"]
@@ -56,7 +56,7 @@ class KeyConfig(Config):
         if not self.macaroon_secret_key:
             # Unfortunately, there are people out there that don't have this
             # set. Lets just be "nice" and derive one from their secret key.
-            logger.warn("Config is missing missing macaroon_secret_key")
+            logger.warn("Config is missing macaroon_secret_key")
             seed = bytes(self.signing_key[0])
             self.macaroon_secret_key = hashlib.sha256(seed).digest()
 
@@ -83,24 +83,29 @@ class KeyConfig(Config):
         # a secret which is used to sign access tokens. If none is specified,
         # the registration_shared_secret is used, if one is given; otherwise,
         # a secret key is derived from the signing key.
+        #
         %(macaroon_secret_key)s
 
         # Used to enable access token expiration.
+        #
         expire_access_token: False
 
         # a secret which is used to calculate HMACs for form values, to stop
         # falsification of values. Must be specified for the User Consent
         # forms to work.
+        #
         %(form_secret)s
 
         ## Signing Keys ##
 
         # Path to the signing key to sign messages with
+        #
         signing_key_path: "%(base_key_name)s.signing.key"
 
         # The keys that the server used to sign messages with but won't use
         # to sign new messages. E.g. it has lost its private key
-        old_signing_keys: {}
+        #
+        #old_signing_keys:
         #  "ed25519:auto":
         #    # Base64 encoded public key
         #    key: "The public part of your old signing key."
@@ -111,9 +116,11 @@ class KeyConfig(Config):
         # Used to set the valid_until_ts in /key/v2 APIs.
         # Determines how quickly servers will query to check which keys
         # are still valid.
+        #
         key_refresh_interval: "1d" # 1 Day.
 
         # The trusted servers to download signing keys from.
+        #
         perspectives:
           servers:
             "matrix.org":

synapse/config/logger.py

@@ -15,7 +15,6 @@
 import logging
 import logging.config
 import os
-import signal
 import sys
 from string import Template
 
@@ -24,6 +23,7 @@ import yaml
 from twisted.logger import STDLibLogObserver, globalLogBeginner
 
 import synapse
+from synapse.app import _base as appbase
 from synapse.util.logcontext import LoggingContextFilter
 from synapse.util.versionstring import get_version_string
 
@@ -81,8 +81,11 @@ class LoggingConfig(Config):
 
     def default_config(self, config_dir_path, server_name, **kwargs):
         log_config = os.path.join(config_dir_path, server_name + ".log.config")
-        return """
+        return """\
+        ## Logging ##
+
         # A yaml python logging config file
+        #
         log_config: "%(log_config)s"
         """ % locals()
 
@@ -127,7 +130,7 @@ class LoggingConfig(Config):
                 )
 
 
-def setup_logging(config, use_worker_options=False, register_sighup=None):
+def setup_logging(config, use_worker_options=False):
     """ Set up python logging
 
     Args:
@@ -140,12 +143,6 @@ def setup_logging(config, use_worker_options=False, register_sighup=None):
         register_sighup (func | None): Function to call to register a
             sighup handler.
     """
-    if not register_sighup:
-        if getattr(signal, "SIGHUP"):
-            register_sighup = lambda x: signal.signal(signal.SIGHUP, x)
-        else:
-            register_sighup = lambda x: None
-
     log_config = (config.worker_log_config if use_worker_options
                   else config.log_config)
     log_file = (config.worker_log_file if use_worker_options
@@ -187,7 +184,7 @@ def setup_logging(config, use_worker_options=False, register_sighup=None):
         else:
             handler = logging.StreamHandler()
 
-            def sighup(signum, stack):
+            def sighup(*args):
                 pass
 
         handler.setFormatter(formatter)
@@ -200,14 +197,14 @@ def setup_logging(config, use_worker_options=False, register_sighup=None):
             with open(log_config, 'r') as f:
                 logging.config.dictConfig(yaml.load(f))
 
-        def sighup(signum, stack):
+        def sighup(*args):
             # it might be better to use a file watcher or something for this.
             load_log_config()
             logging.info("Reloaded log config from %s due to SIGHUP", log_config)
 
         load_log_config()
 
-    register_sighup(sighup)
+    appbase.register_sighup(sighup)
 
     # make sure that the first thing we log is a thing we can grep backwards
     # for
@@ -248,3 +245,5 @@ def setup_logging(config, use_worker_options=False, register_sighup=None):
         [_log],
         redirectStandardIO=not config.no_redirect_stdio,
     )
+    if not config.no_redirect_stdio:
+        print("Redirected stdout/stderr to logs")

synapse/config/metrics.py

@@ -13,7 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._base import Config
+from ._base import Config, ConfigError
+
+MISSING_SENTRY = (
+    """Missing sentry-sdk library. This is required to enable sentry
+    integration.
+    """
+)
 
 
 class MetricsConfig(Config):
@@ -23,12 +29,38 @@ class MetricsConfig(Config):
         self.metrics_port = config.get("metrics_port")
         self.metrics_bind_host = config.get("metrics_bind_host", "127.0.0.1")
 
+        self.sentry_enabled = "sentry" in config
+        if self.sentry_enabled:
+            try:
+                import sentry_sdk  # noqa F401
+            except ImportError:
+                raise ConfigError(MISSING_SENTRY)
+
+            self.sentry_dsn = config["sentry"].get("dsn")
+            if not self.sentry_dsn:
+                raise ConfigError(
+                    "sentry.dsn field is required when sentry integration is enabled",
+                )
+
     def default_config(self, report_stats=None, **kwargs):
         res = """\
         ## Metrics ###
 
         # Enable collection and rendering of performance metrics
+        #
         enable_metrics: False
+
+        # Enable sentry integration
+        # NOTE: While attempts are made to ensure that the logs don't contain
+        # any sensitive information, this cannot be guaranteed. By enabling
+        # this option the sentry server may therefore receive sensitive
+        # information, and it in turn may then diseminate sensitive information
+        # through insecure notification channels if so configured.
+        #
+        #sentry:
+        #    dsn: "..."
+
+        # Whether or not to report anonymized homeserver usage statistics.
         """
 
         if report_stats is None:

synapse/config/password.py

@@ -28,6 +28,7 @@ class PasswordConfig(Config):
     def default_config(self, config_dir_path, server_name, **kwargs):
         return """
         # Enable password for login.
+        #
         password_config:
            enabled: true
            # Uncomment and change to a secret random string for extra security.

synapse/config/password_auth_providers.py

@@ -52,18 +52,18 @@ class PasswordAuthProviderConfig(Config):
 
     def default_config(self, **kwargs):
         return """\
-        # password_providers:
-        #     - module: "ldap_auth_provider.LdapAuthProvider"
-        #       config:
-        #         enabled: true
-        #         uri: "ldap://ldap.example.com:389"
-        #         start_tls: true
-        #         base: "ou=users,dc=example,dc=com"
-        #         attributes:
-        #            uid: "cn"
-        #            mail: "email"
-        #            name: "givenName"
-        #         #bind_dn:
-        #         #bind_password:
-        #         #filter: "(objectClass=posixAccount)"
+        #password_providers:
+        #    - module: "ldap_auth_provider.LdapAuthProvider"
+        #      config:
+        #        enabled: true
+        #        uri: "ldap://ldap.example.com:389"
+        #        start_tls: true
+        #        base: "ou=users,dc=example,dc=com"
+        #        attributes:
+        #           uid: "cn"
+        #           mail: "email"
+        #           name: "givenName"
+        #        #bind_dn:
+        #        #bind_password:
+        #        #filter: "(objectClass=posixAccount)"
         """

synapse/config/push.py

@@ -51,11 +51,11 @@ class PushConfig(Config):
         # notification request includes the content of the event (other details
         # like the sender are still included). For `event_id_only` push, it
         # has no effect.
-
+        #
         # For modern android devices the notification content will still appear
         # because it is loaded by the app. iPhone, however will send a
         # notification saying only that a message arrived and who it came from.
         #
         #push:
-        #   include_content: true
+        #  include_content: true
         """

synapse/config/ratelimiting.py

@@ -32,27 +32,34 @@ class RatelimitConfig(Config):
         ## Ratelimiting ##
 
         # Number of messages a client can send per second
+        #
         rc_messages_per_second: 0.2
 
         # Number of message a client can send before being throttled
+        #
         rc_message_burst_count: 10.0
 
         # The federation window size in milliseconds
+        #
         federation_rc_window_size: 1000
 
         # The number of federation requests from a single server in a window
         # before the server will delay processing the request.
+        #
         federation_rc_sleep_limit: 10
 
         # The duration in milliseconds to delay processing events from
         # remote servers by if they go over the sleep limit.
+        #
         federation_rc_sleep_delay: 500
 
         # The maximum number of concurrent federation requests allowed
         # from a single server
+        #
         federation_rc_reject_limit: 50
 
         # The number of federation requests to concurrently process from a
         # single server
+        #
         federation_rc_concurrent: 3
         """

synapse/config/registration.py

@@ -70,28 +70,29 @@ class RegistrationConfig(Config):
 
         # The user must provide all of the below types of 3PID when registering.
         #
-        # registrations_require_3pid:
-        #     - email
-        #     - msisdn
+        #registrations_require_3pid:
+        #  - email
+        #  - msisdn
 
         # Explicitly disable asking for MSISDNs from the registration
         # flow (overrides registrations_require_3pid if MSISDNs are set as required)
         #
-        # disable_msisdn_registration = True
+        #disable_msisdn_registration: True
 
         # Mandate that users are only allowed to associate certain formats of
         # 3PIDs with accounts on this server.
         #
-        # allowed_local_3pids:
-        #     - medium: email
-        #       pattern: '.*@matrix\\.org'
-        #     - medium: email
-        #       pattern: '.*@vector\\.im'
-        #     - medium: msisdn
-        #       pattern: '\\+44'
+        #allowed_local_3pids:
+        #  - medium: email
+        #    pattern: '.*@matrix\\.org'
+        #  - medium: email
+        #    pattern: '.*@vector\\.im'
+        #  - medium: msisdn
+        #    pattern: '\\+44'
 
         # If set, allows registration by anyone who also has the shared
         # secret, even if registration is otherwise disabled.
+        #
         %(registration_shared_secret)s
 
         # Set the number of bcrypt rounds used to generate password hash.
@@ -99,11 +100,13 @@ class RegistrationConfig(Config):
         # The default number is 12 (which equates to 2^12 rounds).
         # N.B. that increasing this will exponentially increase the time required
         # to register or login - e.g. 24 => 2^24 rounds which will take >20 mins.
+        #
         bcrypt_rounds: 12
 
         # Allows users to register as guests without a password/email/etc, and
         # participate in rooms hosted on this server which have been made
         # accessible to anonymous users.
+        #
         allow_guest_access: False
 
         # The identity server which we suggest that clients should use when users log
@@ -112,27 +115,30 @@ class RegistrationConfig(Config):
         # (By default, no suggestion is made, so it is left up to the client.
         # This setting is ignored unless public_baseurl is also set.)
         #
-        # default_identity_server: https://matrix.org
+        #default_identity_server: https://matrix.org
 
         # The list of identity servers trusted to verify third party
         # identifiers by this server.
         #
         # Also defines the ID server which will be called when an account is
         # deactivated (one will be picked arbitrarily).
+        #
         trusted_third_party_id_servers:
-            - matrix.org
-            - vector.im
+          - matrix.org
+          - vector.im
 
         # Users who register on this homeserver will automatically be joined
         # to these rooms
+        #
         #auto_join_rooms:
-        #    - "#example:example.com"
+        #  - "#example:example.com"
 
         # Where auto_join_rooms are specified, setting this flag ensures that the
         # the rooms exist by creating them when the first user on the
         # homeserver registers.
         # Setting to false means that if the rooms are not manually created,
         # users cannot be auto-joined since they do not exist.
+        #
         autocreate_auto_join_rooms: true
         """ % locals()
 

synapse/config/repository.py

@@ -180,29 +180,34 @@ class ContentRepositoryConfig(Config):
         uploads_path = os.path.join(data_dir_path, "uploads")
         return r"""
         # Directory where uploaded images and attachments are stored.
+        #
         media_store_path: "%(media_store)s"
 
         # Media storage providers allow media to be stored in different
         # locations.
-        # media_storage_providers:
-        # - module: file_system
-        #   # Whether to write new local files.
-        #   store_local: false
-        #   # Whether to write new remote media
-        #   store_remote: false
-        #   # Whether to block upload requests waiting for write to this
-        #   # provider to complete
-        #   store_synchronous: false
-        #   config:
-        #     directory: /mnt/some/other/directory
+        #
+        #media_storage_providers:
+        #  - module: file_system
+        #    # Whether to write new local files.
+        #    store_local: false
+        #    # Whether to write new remote media
+        #    store_remote: false
+        #    # Whether to block upload requests waiting for write to this
+        #    # provider to complete
+        #    store_synchronous: false
+        #    config:
+        #       directory: /mnt/some/other/directory
 
         # Directory where in-progress uploads are stored.
+        #
         uploads_path: "%(uploads_path)s"
 
         # The largest allowed upload size in bytes
+        #
         max_upload_size: "10M"
 
         # Maximum number of pixels that will be thumbnailed
+        #
         max_image_pixels: "32M"
 
         # Whether to generate new thumbnails on the fly to precisely match
@@ -210,9 +215,11 @@ class ContentRepositoryConfig(Config):
         # a new resolution is requested by the client the server will
         # generate a new thumbnail. If false the server will pick a thumbnail
         # from a precalculated list.
+        #
         dynamic_thumbnails: false
 
-        # List of thumbnail to precalculate when an image is uploaded.
+        # List of thumbnails to precalculate when an image is uploaded.
+        #
         thumbnail_sizes:
         - width: 32
           height: 32
@@ -233,6 +240,7 @@ class ContentRepositoryConfig(Config):
         # Is the preview URL API enabled?  If enabled, you *must* specify
         # an explicit url_preview_ip_range_blacklist of IPs that the spider is
         # denied from accessing.
+        #
         url_preview_enabled: False
 
         # List of IP address CIDR ranges that the URL preview spider is denied
@@ -243,16 +251,16 @@ class ContentRepositoryConfig(Config):
         # synapse to issue arbitrary GET requests to your internal services,
         # causing serious security issues.
         #
-        # url_preview_ip_range_blacklist:
-        # - '127.0.0.0/8'
-        # - '10.0.0.0/8'
-        # - '172.16.0.0/12'
-        # - '192.168.0.0/16'
-        # - '100.64.0.0/10'
-        # - '169.254.0.0/16'
-        # - '::1/128'
-        # - 'fe80::/64'
-        # - 'fc00::/7'
+        #url_preview_ip_range_blacklist:
+        #  - '127.0.0.0/8'
+        #  - '10.0.0.0/8'
+        #  - '172.16.0.0/12'
+        #  - '192.168.0.0/16'
+        #  - '100.64.0.0/10'
+        #  - '169.254.0.0/16'
+        #  - '::1/128'
+        #  - 'fe80::/64'
+        #  - 'fc00::/7'
         #
         # List of IP address CIDR ranges that the URL preview spider is allowed
         # to access even if they are specified in url_preview_ip_range_blacklist.
@@ -260,8 +268,8 @@ class ContentRepositoryConfig(Config):
         # target IP ranges - e.g. for enabling URL previews for a specific private
         # website only visible in your network.
         #
-        # url_preview_ip_range_whitelist:
-        # - '192.168.1.1'
+        #url_preview_ip_range_whitelist:
+        #   - '192.168.1.1'
 
         # Optional list of URL matches that the URL preview spider is
         # denied from accessing.  You should use url_preview_ip_range_blacklist
@@ -279,26 +287,25 @@ class ContentRepositoryConfig(Config):
         # specified component matches for a given list item succeed, the URL is
         # blacklisted.
         #
-        # url_preview_url_blacklist:
-        # # blacklist any URL with a username in its URI
-        # - username: '*'
+        #url_preview_url_blacklist:
+        #  # blacklist any URL with a username in its URI
+        #  - username: '*'
         #
-        # # blacklist all *.google.com URLs
-        # - netloc: 'google.com'
-        # - netloc: '*.google.com'
+        #  # blacklist all *.google.com URLs
+        #  - netloc: 'google.com'
+        #  - netloc: '*.google.com'
         #
-        # # blacklist all plain HTTP URLs
-        # - scheme: 'http'
+        #  # blacklist all plain HTTP URLs
+        #  - scheme: 'http'
         #
-        # # blacklist http(s)://www.acme.com/foo
-        # - netloc: 'www.acme.com'
-        #   path: '/foo'
+        #  # blacklist http(s)://www.acme.com/foo
+        #  - netloc: 'www.acme.com'
+        #    path: '/foo'
         #
-        # # blacklist any URL with a literal IPv4 address
-        # - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'
+        #  # blacklist any URL with a literal IPv4 address
+        #  - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'
 
         # The largest allowed URL preview spidering size in bytes
         max_spider_size: "10M"
 
-
         """ % locals()

synapse/config/room_directory.py

@@ -20,12 +20,37 @@ from ._base import Config, ConfigError
 
 class RoomDirectoryConfig(Config):
     def read_config(self, config):
-        alias_creation_rules = config["alias_creation_rules"]
+        alias_creation_rules = config.get("alias_creation_rules")
 
-        self._alias_creation_rules = [
-            _AliasRule(rule)
-            for rule in alias_creation_rules
-        ]
+        if alias_creation_rules is not None:
+            self._alias_creation_rules = [
+                _RoomDirectoryRule("alias_creation_rules", rule)
+                for rule in alias_creation_rules
+            ]
+        else:
+            self._alias_creation_rules = [
+                _RoomDirectoryRule(
+                    "alias_creation_rules", {
+                        "action": "allow",
+                    }
+                )
+            ]
+
+        room_list_publication_rules = config.get("room_list_publication_rules")
+
+        if room_list_publication_rules is not None:
+            self._room_list_publication_rules = [
+                _RoomDirectoryRule("room_list_publication_rules", rule)
+                for rule in room_list_publication_rules
+            ]
+        else:
+            self._room_list_publication_rules = [
+                _RoomDirectoryRule(
+                    "room_list_publication_rules", {
+                        "action": "allow",
+                    }
+                )
+            ]
 
     def default_config(self, config_dir_path, server_name, **kwargs):
         return """
@@ -33,60 +58,138 @@ class RoomDirectoryConfig(Config):
         # on this server.
         #
         # The format of this option is a list of rules that contain globs that
-        # match against user_id and the new alias (fully qualified with server
-        # name). The action in the first rule that matches is taken, which can
-        # currently either be "allow" or "deny".
+        # match against user_id, room_id and the new alias (fully qualified with
+        # server name). The action in the first rule that matches is taken,
+        # which can currently either be "allow" or "deny".
         #
-        # If no rules match the request is denied.
-        alias_creation_rules:
-            - user_id: "*"
-              alias: "*"
-              action: allow
+        # Missing user_id/room_id/alias fields default to "*".
+        #
+        # If no rules match the request is denied. An empty list means no one
+        # can create aliases.
+        #
+        # Options for the rules include:
+        #
+        #   user_id: Matches against the creator of the alias
+        #   alias: Matches against the alias being created
+        #   room_id: Matches against the room ID the alias is being pointed at
+        #   action: Whether to "allow" or "deny" the request if the rule matches
+        #
+        # The default is:
+        #
+        #alias_creation_rules:
+        #  - user_id: "*"
+        #    alias: "*"
+        #    room_id: "*"
+        #    action: allow
+
+        # The `room_list_publication_rules` option controls who can publish and
+        # which rooms can be published in the public room list.
+        #
+        # The format of this option is the same as that for
+        # `alias_creation_rules`.
+        #
+        # If the room has one or more aliases associated with it, only one of
+        # the aliases needs to match the alias rule. If there are no aliases
+        # then only rules with `alias: *` match.
+        #
+        # If no rules match the request is denied. An empty list means no one
+        # can publish rooms.
+        #
+        # Options for the rules include:
+        #
+        #   user_id: Matches agaisnt the creator of the alias
+        #   room_id: Matches against the room ID being published
+        #   alias: Matches against any current local or canonical aliases
+        #            associated with the room
+        #   action: Whether to "allow" or "deny" the request if the rule matches
+        #
+        # The default is:
+        #
+        #room_list_publication_rules:
+        #  - user_id: "*"
+        #    alias: "*"
+        #    room_id: "*"
+        #    action: allow
         """
 
-    def is_alias_creation_allowed(self, user_id, alias):
+    def is_alias_creation_allowed(self, user_id, room_id, alias):
         """Checks if the given user is allowed to create the given alias
 
         Args:
             user_id (str)
+            room_id (str)
             alias (str)
 
         Returns:
             boolean: True if user is allowed to crate the alias
         """
         for rule in self._alias_creation_rules:
-            if rule.matches(user_id, alias):
+            if rule.matches(user_id, room_id, [alias]):
+                return rule.action == "allow"
+
+        return False
+
+    def is_publishing_room_allowed(self, user_id, room_id, aliases):
+        """Checks if the given user is allowed to publish the room
+
+        Args:
+            user_id (str)
+            room_id (str)
+            aliases (list[str]): any local aliases associated with the room
+
+        Returns:
+            boolean: True if user can publish room
+        """
+        for rule in self._room_list_publication_rules:
+            if rule.matches(user_id, room_id, aliases):
                 return rule.action == "allow"
 
         return False
 
 
-class _AliasRule(object):
-    def __init__(self, rule):
+class _RoomDirectoryRule(object):
+    """Helper class to test whether a room directory action is allowed, like
+    creating an alias or publishing a room.
+    """
+
+    def __init__(self, option_name, rule):
+        """
+        Args:
+            option_name (str): Name of the config option this rule belongs to
+            rule (dict): The rule as specified in the config
+        """
+
         action = rule["action"]
-        user_id = rule["user_id"]
-        alias = rule["alias"]
+        user_id = rule.get("user_id", "*")
+        room_id = rule.get("room_id", "*")
+        alias = rule.get("alias", "*")
 
         if action in ("allow", "deny"):
             self.action = action
         else:
             raise ConfigError(
-                "alias_creation_rules rules can only have action of 'allow'"
-                " or 'deny'"
+                "%s rules can only have action of 'allow'"
+                " or 'deny'" % (option_name,)
             )
 
+        self._alias_matches_all = alias == "*"
+
         try:
             self._user_id_regex = glob_to_regex(user_id)
             self._alias_regex = glob_to_regex(alias)
+            self._room_id_regex = glob_to_regex(room_id)
         except Exception as e:
             raise ConfigError("Failed to parse glob into regex: %s", e)
 
-    def matches(self, user_id, alias):
-        """Tests if this rule matches the given user_id and alias.
+    def matches(self, user_id, room_id, aliases):
+        """Tests if this rule matches the given user_id, room_id and aliases.
 
         Args:
             user_id (str)
-            alias (str)
+            room_id (str)
+            aliases (list[str]): The associated aliases to the room. Will be a
+                single element for testing alias creation, and can be empty for
+                testing room publishing.
 
         Returns:
             boolean
@@ -96,7 +199,22 @@ class _AliasRule(object):
         if not self._user_id_regex.match(user_id):
             return False
 
-        if not self._alias_regex.match(alias):
+        if not self._room_id_regex.match(room_id):
             return False
 
-        return True
+        # We only have alias checks left, so we can short circuit if the alias
+        # rule matches everything.
+        if self._alias_matches_all:
+            return True
+
+        # If we are not given any aliases then this rule only matches if the
+        # alias glob matches all aliases, which we checked above.
+        if not aliases:
+            return False
+
+        # Otherwise, we just need one alias to match
+        for alias in aliases:
+            if self._alias_regex.match(alias):
+                return True
+
+        return False

synapse/config/saml2_config.py

@@ -67,44 +67,43 @@ class SAML2Config(Config):
         return """
         # Enable SAML2 for registration and login. Uses pysaml2.
         #
-        # saml2_config:
+        # `sp_config` is the configuration for the pysaml2 Service Provider.
+        # See pysaml2 docs for format of config.
         #
-        #   # The following is the configuration for the pysaml2 Service Provider.
-        #   # See pysaml2 docs for format of config.
-        #   #
-        #   # Default values will be used for the 'entityid' and 'service' settings,
-        #   # so it is not normally necessary to specify them unless you need to
-        #   # override them.
+        # Default values will be used for the 'entityid' and 'service' settings,
+        # so it is not normally necessary to specify them unless you need to
+        # override them.
         #
-        #   sp_config:
-        #     # point this to the IdP's metadata. You can use either a local file or
-        #     # (preferably) a URL.
-        #     metadata:
-        #       # local: ["saml2/idp.xml"]
-        #       remote:
-        #         - url: https://our_idp/metadata.xml
+        #saml2_config:
+        #  sp_config:
+        #    # point this to the IdP's metadata. You can use either a local file or
+        #    # (preferably) a URL.
+        #    metadata:
+        #      #local: ["saml2/idp.xml"]
+        #      remote:
+        #        - url: https://our_idp/metadata.xml
         #
-        #     # The following is just used to generate our metadata xml, and you
-        #     # may well not need it, depending on your setup. Alternatively you
-        #     # may need a whole lot more detail - see the pysaml2 docs!
+        #    # The rest of sp_config is just used to generate our metadata xml, and you
+        #    # may well not need it, depending on your setup. Alternatively you
+        #    # may need a whole lot more detail - see the pysaml2 docs!
         #
-        #     description: ["My awesome SP", "en"]
-        #     name: ["Test SP", "en"]
+        #    description: ["My awesome SP", "en"]
+        #    name: ["Test SP", "en"]
         #
-        #     organization:
-        #       name: Example com
-        #       display_name:
-        #         - ["Example co", "en"]
-        #       url: "http://example.com"
+        #    organization:
+        #      name: Example com
+        #      display_name:
+        #        - ["Example co", "en"]
+        #      url: "http://example.com"
         #
-        #     contact_person:
-        #       - given_name: Bob
-        #         sur_name: "the Sysadmin"
-        #         email_address": ["admin@example.com"]
-        #         contact_type": technical
+        #    contact_person:
+        #      - given_name: Bob
+        #        sur_name: "the Sysadmin"
+        #        email_address": ["admin@example.com"]
+        #        contact_type": technical
         #
-        #   # Instead of putting the config inline as above, you can specify a
-        #   # separate pysaml2 configuration file:
-        #   #
-        #   # config_path: "%(config_dir_path)s/sp_conf.py"
+        #  # Instead of putting the config inline as above, you can specify a
+        #  # separate pysaml2 configuration file:
+        #  #
+        #  config_path: "%(config_dir_path)s/sp_conf.py"
         """ % {"config_dir_path": config_dir_path}

synapse/config/server.py

@@ -24,6 +24,14 @@ from ._base import Config, ConfigError
 
 logger = logging.Logger(__name__)
 
+# by default, we attempt to listen on both '::' *and* '0.0.0.0' because some OSes
+# (Windows, macOS, other BSD/Linux where net.ipv6.bindv6only is set) will only listen
+# on IPv6 when '::' is set.
+#
+# We later check for errors when binding to 0.0.0.0 and ignore them if :: is also in
+# in the list.
+DEFAULT_BIND_ADDRESSES = ['::', '0.0.0.0']
+
 
 class ServerConfig(Config):
 
@@ -118,16 +126,38 @@ class ServerConfig(Config):
                 self.public_baseurl += '/'
         self.start_pushers = config.get("start_pushers", True)
 
-        self.listeners = config.get("listeners", [])
+        self.listeners = []
+        for listener in config.get("listeners", []):
+            if not isinstance(listener.get("port", None), int):
+                raise ConfigError(
+                    "Listener configuration is lacking a valid 'port' option"
+                )
+
+            if listener.setdefault("tls", False):
+                # no_tls is not really supported any more, but let's grandfather it in
+                # here.
+                if config.get("no_tls", False):
+                    logger.info(
+                        "Ignoring TLS-enabled listener on port %i due to no_tls"
+                    )
+                    continue
 
-        for listener in self.listeners:
             bind_address = listener.pop("bind_address", None)
             bind_addresses = listener.setdefault("bind_addresses", [])
 
+            # if bind_address was specified, add it to the list of addresses
             if bind_address:
                 bind_addresses.append(bind_address)
-            elif not bind_addresses:
-                bind_addresses.append('')
+
+            # if we still have an empty list of addresses, use the default list
+            if not bind_addresses:
+                if listener['type'] == 'metrics':
+                    # the metrics listener doesn't support IPv6
+                    bind_addresses.append('0.0.0.0')
+                else:
+                    bind_addresses.extend(DEFAULT_BIND_ADDRESSES)
+
+            self.listeners.append(listener)
 
         if not self.web_client_location:
             _warn_if_webclient_configured(self.listeners)
@@ -136,6 +166,9 @@ class ServerConfig(Config):
 
         bind_port = config.get("bind_port")
         if bind_port:
+            if config.get("no_tls", False):
+                raise ConfigError("no_tls is incompatible with bind_port")
+
             self.listeners = []
             bind_host = config.get("bind_host", "")
             gzip_responses = config.get("gzip_responses", True)
@@ -182,6 +215,7 @@ class ServerConfig(Config):
                 "port": manhole,
                 "bind_addresses": ["127.0.0.1"],
                 "type": "manhole",
+                "tls": False,
             })
 
         metrics_port = config.get("metrics_port")
@@ -207,6 +241,9 @@ class ServerConfig(Config):
 
         _check_resource_config(self.listeners)
 
+    def has_tls_listener(self):
+        return any(l["tls"] for l in self.listeners)
+
     def default_config(self, server_name, data_dir_path, **kwargs):
         _, bind_port = parse_and_validate_server_name(server_name)
         if bind_port is not None:
@@ -223,9 +260,11 @@ class ServerConfig(Config):
         # This is used by remote servers to connect to this server,
         # e.g. matrix.org, localhost:8080, etc.
         # This is also the last part of your UserID.
+        #
         server_name: "%(server_name)s"
 
         # When running as a daemon, the file to store the pid in
+        #
         pid_file: %(pid_file)s
 
         # CPU affinity mask. Setting this restricts the CPUs on which the
@@ -249,38 +288,51 @@ class ServerConfig(Config):
         #
         # This setting requires the affinity package to be installed!
         #
-        # cpu_affinity: 0xFFFFFFFF
+        #cpu_affinity: 0xFFFFFFFF
 
         # The path to the web client which will be served at /_matrix/client/
         # if 'webclient' is configured under the 'listeners' configuration.
         #
-        # web_client_location: "/path/to/web/root"
+        #web_client_location: "/path/to/web/root"
 
         # The public-facing base URL that clients use to access this HS
         # (not including _matrix/...). This is the same URL a user would
         # enter into the 'custom HS URL' field on their client. If you
         # use synapse with a reverse proxy, this should be the URL to reach
         # synapse via the proxy.
-        # public_baseurl: https://example.com/
+        #
+        #public_baseurl: https://example.com/
 
         # Set the soft limit on the number of file descriptors synapse can use
         # Zero is used to indicate synapse should set the soft limit to the
         # hard limit.
+        #
         soft_file_limit: 0
 
         # Set to false to disable presence tracking on this homeserver.
+        #
         use_presence: true
 
         # The GC threshold parameters to pass to `gc.set_threshold`, if defined
-        # gc_thresholds: [700, 10, 10]
+        #
+        #gc_thresholds: [700, 10, 10]
 
         # Set the limit on the returned events in the timeline in the get
         # and sync operations. The default value is -1, means no upper limit.
-        # filter_timeline_limit: 5000
+        #
+        #filter_timeline_limit: 5000
 
         # Whether room invites to users on this server should be blocked
         # (except those sent by local server admins). The default is False.
-        # block_non_admin_invites: True
+        #
+        #block_non_admin_invites: True
+
+        # Room searching
+        #
+        # If disabled, new messages will not be indexed for searching and users
+        # will receive errors when searching for messages. Defaults to enabled.
+        #
+        #enable_search: false
 
         # Restrict federation to the following whitelist of domains.
         # N.B. we recommend also firewalling your federation listener to limit
@@ -288,117 +340,145 @@ class ServerConfig(Config):
         # purely on this application-layer restriction.  If not specified, the
         # default is to whitelist everything.
         #
-        # federation_domain_whitelist:
+        #federation_domain_whitelist:
         #  - lon.example.com
         #  - nyc.example.com
         #  - syd.example.com
 
         # List of ports that Synapse should listen on, their purpose and their
         # configuration.
+        #
+        # Options for each listener include:
+        #
+        #   port: the TCP port to bind to
+        #
+        #   bind_addresses: a list of local addresses to listen on. The default is
+        #       'all local interfaces'.
+        #
+        #   type: the type of listener. Normally 'http', but other valid options are:
+        #       'manhole' (see docs/manhole.md),
+        #       'metrics' (see docs/metrics-howto.rst),
+        #       'replication' (see docs/workers.rst).
+        #
+        #   tls: set to true to enable TLS for this listener. Will use the TLS
+        #       key/cert specified in tls_private_key_path / tls_certificate_path.
+        #
+        #   x_forwarded: Only valid for an 'http' listener. Set to true to use the
+        #       X-Forwarded-For header as the client IP. Useful when Synapse is
+        #       behind a reverse-proxy.
+        #
+        #   resources: Only valid for an 'http' listener. A list of resources to host
+        #       on this port. Options for each resource are:
+        #
+        #       names: a list of names of HTTP resources. See below for a list of
+        #           valid resource names.
+        #
+        #       compress: set to true to enable HTTP comression for this resource.
+        #
+        #   additional_resources: Only valid for an 'http' listener. A map of
+        #        additional endpoints which should be loaded via dynamic modules.
+        #
+        # Valid resource names are:
+        #
+        #   client: the client-server API (/_matrix/client). Also implies 'media' and
+        #       'static'.
+        #
+        #   consent: user consent forms (/_matrix/consent). See
+        #       docs/consent_tracking.md.
+        #
+        #   federation: the server-server API (/_matrix/federation). Also implies
+        #       'media', 'keys', 'openid'
+        #
+        #   keys: the key discovery API (/_matrix/keys).
+        #
+        #   media: the media API (/_matrix/media).
+        #
+        #   metrics: the metrics interface. See docs/metrics-howto.rst.
+        #
+        #   openid: OpenID authentication.
+        #
+        #   replication: the HTTP replication API (/_synapse/replication). See
+        #       docs/workers.rst.
+        #
+        #   static: static resources under synapse/static (/_matrix/static). (Mostly
+        #       useful for 'fallback authentication'.)
+        #
+        #   webclient: A web client. Requires web_client_location to be set.
+        #
         listeners:
-          # Main HTTPS listener
-          # For when matrix traffic is sent directly to synapse.
-          -
-            # The port to listen for HTTPS requests on.
-            port: %(bind_port)s
+          # TLS-enabled listener: for when matrix traffic is sent directly to synapse.
+          #
+          # Disabled by default. To enable it, uncomment the following. (Note that you
+          # will also need to give Synapse a TLS key and certificate: see the TLS section
+          # below.)
+          #
+          #- port: %(bind_port)s
+          #  type: http
+          #  tls: true
+          #  resources:
+          #    - names: [client, federation]
 
-            # Local addresses to listen on.
-            # On Linux and Mac OS, `::` will listen on all IPv4 and IPv6
-            # addresses by default. For most other OSes, this will only listen
-            # on IPv6.
-            bind_addresses:
-              - '::'
-              - '0.0.0.0'
-
-            # This is a 'http' listener, allows us to specify 'resources'.
-            type: http
-
-            tls: true
-
-            # Use the X-Forwarded-For (XFF) header as the client IP and not the
-            # actual client IP.
-            x_forwarded: false
-
-            # List of HTTP resources to serve on this listener.
-            resources:
-              -
-                # List of resources to host on this listener.
-                names:
-                  - client       # The client-server APIs, both v1 and v2
-                  # - webclient  # A web client. Requires web_client_location to be set.
-
-                # Should synapse compress HTTP responses to clients that support it?
-                # This should be disabled if running synapse behind a load balancer
-                # that can do automatic compression.
-                compress: true
-
-              - names: [federation]  # Federation APIs
-                compress: false
-
-            # optional list of additional endpoints which can be loaded via
-            # dynamic modules
-            # additional_resources:
-            #   "/_matrix/my/custom/endpoint":
-            #     module: my_module.CustomRequestHandler
-            #     config: {}
-
-          # Unsecure HTTP listener,
-          # For when matrix traffic passes through loadbalancer that unwraps TLS.
+          # Unsecure HTTP listener: for when matrix traffic passes through a reverse proxy
+          # that unwraps TLS.
+          #
+          # If you plan to use a reverse proxy, please see
+          # https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.rst.
+          #
           - port: %(unsecure_port)s
             tls: false
-            bind_addresses: ['::', '0.0.0.0']
+            bind_addresses: ['::1', '127.0.0.1']
             type: http
-
-            x_forwarded: false
+            x_forwarded: true
 
             resources:
-              - names: [client]
-                compress: true
-              - names: [federation]
+              - names: [client, federation]
                 compress: false
 
+            # example additonal_resources:
+            #
+            #additional_resources:
+            #  "/_matrix/my/custom/endpoint":
+            #    module: my_module.CustomRequestHandler
+            #    config: {}
+
           # Turn on the twisted ssh manhole service on localhost on the given
           # port.
-          # - port: 9000
-          #   bind_addresses: ['::1', '127.0.0.1']
-          #   type: manhole
+          #
+          #- port: 9000
+          #  bind_addresses: ['::1', '127.0.0.1']
+          #  type: manhole
 
 
-        # Homeserver blocking
-        #
+        ## Homeserver blocking ##
+
         # How to reach the server admin, used in ResourceLimitError
-        # admin_contact: 'mailto:admin@server.com'
         #
-        # Global block config
-        #
-        # hs_disabled: False
-        # hs_disabled_message: 'Human readable reason for why the HS is blocked'
-        # hs_disabled_limit_type: 'error code(str), to help clients decode reason'
+        #admin_contact: 'mailto:admin@server.com'
+
+        # Global blocking
         #
+        #hs_disabled: False
+        #hs_disabled_message: 'Human readable reason for why the HS is blocked'
+        #hs_disabled_limit_type: 'error code(str), to help clients decode reason'
+
         # Monthly Active User Blocking
         #
-        # Enables monthly active user checking
-        # limit_usage_by_mau: False
-        # max_mau_value: 50
-        # mau_trial_days: 2
-        #
+        #limit_usage_by_mau: False
+        #max_mau_value: 50
+        #mau_trial_days: 2
+
         # If enabled, the metrics for the number of monthly active users will
         # be populated, however no one will be limited. If limit_usage_by_mau
         # is true, this is implied to be true.
-        # mau_stats_only: False
         #
+        #mau_stats_only: False
+
         # Sometimes the server admin will want to ensure certain accounts are
         # never blocked by mau checking. These accounts are specified here.
         #
-        # mau_limit_reserved_threepids:
-        # - medium: 'email'
-        #   address: 'reserved_user@example.com'
-        #
-        # Room searching
-        #
-        # If disabled, new messages will not be indexed for searching and users
-        # will receive errors when searching for messages. Defaults to enabled.
-        # enable_search: true
+        #mau_limit_reserved_threepids:
+        #  - medium: 'email'
+        #    address: 'reserved_user@example.com'
         """ % locals()
 
     def read_arguments(self, args):
@@ -480,6 +560,7 @@ KNOWN_RESOURCES = (
     'keys',
     'media',
     'metrics',
+    'openid',
     'replication',
     'static',
     'webclient',

synapse/config/server_notices_config.py

@@ -30,11 +30,11 @@ DEFAULT_CONFIG = """\
 # It's also possible to override the room name, the display name of the
 # "notices" user, and the avatar for the user.
 #
-# server_notices:
-#   system_mxid_localpart: notices
-#   system_mxid_display_name: "Server Notices"
-#   system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
-#   room_name: "Server Notices"
+#server_notices:
+#  system_mxid_localpart: notices
+#  system_mxid_display_name: "Server Notices"
+#  system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
+#  room_name: "Server Notices"
 """
 
 

synapse/config/spam_checker.py

@@ -28,8 +28,8 @@ class SpamCheckerConfig(Config):
 
     def default_config(self, **kwargs):
         return """\
-        # spam_checker:
-        #     module: "my_custom_project.SuperSpamChecker"
-        #     config:
-        #         example_option: 'things'
+        #spam_checker:
+        #  module: "my_custom_project.SuperSpamChecker"
+        #  config:
+        #    example_option: 'things'
         """

synapse/config/tls.py

@@ -19,13 +19,15 @@ import warnings
 from datetime import datetime
 from hashlib import sha256
 
+import six
+
 from unpaddedbase64 import encode_base64
 
 from OpenSSL import crypto
 
-from synapse.config._base import Config
+from synapse.config._base import Config, ConfigError
 
-logger = logging.getLogger()
+logger = logging.getLogger(__name__)
 
 
 class TlsConfig(Config):
@@ -36,18 +38,37 @@ class TlsConfig(Config):
             acme_config = {}
 
         self.acme_enabled = acme_config.get("enabled", False)
-        self.acme_url = acme_config.get(
+
+        # hyperlink complains on py2 if this is not a Unicode
+        self.acme_url = six.text_type(acme_config.get(
             "url", u"https://acme-v01.api.letsencrypt.org/directory"
-        )
+        ))
         self.acme_port = acme_config.get("port", 80)
         self.acme_bind_addresses = acme_config.get("bind_addresses", ['::', '0.0.0.0'])
         self.acme_reprovision_threshold = acme_config.get("reprovision_threshold", 30)
+        self.acme_domain = acme_config.get("domain", config.get("server_name"))
 
         self.tls_certificate_file = self.abspath(config.get("tls_certificate_path"))
         self.tls_private_key_file = self.abspath(config.get("tls_private_key_path"))
-        self._original_tls_fingerprints = config["tls_fingerprints"]
+
+        if self.has_tls_listener():
+            if not self.tls_certificate_file:
+                raise ConfigError(
+                    "tls_certificate_path must be specified if TLS-enabled listeners are "
+                    "configured."
+                )
+            if not self.tls_private_key_file:
+                raise ConfigError(
+                    "tls_private_key_path must be specified if TLS-enabled listeners are "
+                    "configured."
+                )
+
+        self._original_tls_fingerprints = config.get("tls_fingerprints", [])
+
+        if self._original_tls_fingerprints is None:
+            self._original_tls_fingerprints = []
+
         self.tls_fingerprints = list(self._original_tls_fingerprints)
-        self.no_tls = config.get("no_tls", False)
 
         # This config option applies to non-federation HTTP clients
         # (e.g. for talking to recaptcha, identity servers, and such)
@@ -60,10 +81,14 @@ class TlsConfig(Config):
         self.tls_certificate = None
         self.tls_private_key = None
 
-    def is_disk_cert_valid(self):
+    def is_disk_cert_valid(self, allow_self_signed=True):
         """
         Is the certificate we have on disk valid, and if so, for how long?
 
+        Args:
+            allow_self_signed (bool): Should we allow the certificate we
+                read to be self signed?
+
         Returns:
             int: Days remaining of certificate validity.
             None: No certificate exists.
@@ -84,6 +109,12 @@ class TlsConfig(Config):
             logger.exception("Failed to parse existing certificate off disk!")
             raise
 
+        if not allow_self_signed:
+            if tls_certificate.get_subject() == tls_certificate.get_issuer():
+                raise ValueError(
+                    "TLS Certificate is self signed, and this is not permitted"
+                )
+
         # YYYYMMDDhhmmssZ -- in UTC
         expires_on = datetime.strptime(
             tls_certificate.get_notAfter().decode('ascii'), "%Y%m%d%H%M%SZ"
@@ -92,36 +123,40 @@ class TlsConfig(Config):
         days_remaining = (expires_on - now).days
         return days_remaining
 
-    def read_certificate_from_disk(self):
+    def read_certificate_from_disk(self, require_cert_and_key):
         """
-        Read the certificates from disk.
-        """
-        self.tls_certificate = self.read_tls_certificate(self.tls_certificate_file)
+        Read the certificates and private key from disk.
 
-        # Check if it is self-signed, and issue a warning if so.
-        if self.tls_certificate.get_issuer() == self.tls_certificate.get_subject():
-            warnings.warn(
-                (
-                    "Self-signed TLS certificates will not be accepted by Synapse 1.0. "
-                    "Please either provide a valid certificate, or use Synapse's ACME "
-                    "support to provision one."
+        Args:
+            require_cert_and_key (bool): set to True to throw an error if the certificate
+                and key file are not given
+        """
+        if require_cert_and_key:
+            self.tls_private_key = self.read_tls_private_key()
+            self.tls_certificate = self.read_tls_certificate()
+        elif self.tls_certificate_file:
+            # we only need the certificate for the tls_fingerprints. Reload it if we
+            # can, but it's not a fatal error if we can't.
+            try:
+                self.tls_certificate = self.read_tls_certificate()
+            except Exception as e:
+                logger.info(
+                    "Unable to read TLS certificate (%s). Ignoring as no "
+                    "tls listeners enabled.", e,
                 )
-            )
-
-        if not self.no_tls:
-            self.tls_private_key = self.read_tls_private_key(self.tls_private_key_file)
 
         self.tls_fingerprints = list(self._original_tls_fingerprints)
 
-        # Check that our own certificate is included in the list of fingerprints
-        # and include it if it is not.
-        x509_certificate_bytes = crypto.dump_certificate(
-            crypto.FILETYPE_ASN1, self.tls_certificate
-        )
-        sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest())
-        sha256_fingerprints = set(f["sha256"] for f in self.tls_fingerprints)
-        if sha256_fingerprint not in sha256_fingerprints:
-            self.tls_fingerprints.append({u"sha256": sha256_fingerprint})
+        if self.tls_certificate:
+            # Check that our own certificate is included in the list of fingerprints
+            # and include it if it is not.
+            x509_certificate_bytes = crypto.dump_certificate(
+                crypto.FILETYPE_ASN1, self.tls_certificate
+            )
+            sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest())
+            sha256_fingerprints = set(f["sha256"] for f in self.tls_fingerprints)
+            if sha256_fingerprint not in sha256_fingerprints:
+                self.tls_fingerprints.append({u"sha256": sha256_fingerprint})
 
     def default_config(self, config_dir_path, server_name, **kwargs):
         base_key_name = os.path.join(config_dir_path, server_name)
@@ -137,6 +172,8 @@ class TlsConfig(Config):
 
         return (
             """\
+        ## TLS ##
+
         # PEM-encoded X509 certificate for TLS.
         # This certificate, as of Synapse 1.0, will need to be a valid and verifiable
         # certificate, signed by a recognised Certificate Authority.
@@ -144,10 +181,11 @@ class TlsConfig(Config):
         # See 'ACME support' below to enable auto-provisioning this certificate via
         # Let's Encrypt.
         #
-        tls_certificate_path: "%(tls_certificate_path)s"
+        #tls_certificate_path: "%(tls_certificate_path)s"
 
         # PEM-encoded private key for TLS
-        tls_private_key_path: "%(tls_private_key_path)s"
+        #
+        #tls_private_key_path: "%(tls_private_key_path)s"
 
         # ACME support: This will configure Synapse to request a valid TLS certificate
         # for your configured `server_name` via Let's Encrypt.
@@ -172,37 +210,44 @@ class TlsConfig(Config):
         #
         acme:
             # ACME support is disabled by default. Uncomment the following line
-            # to enable it.
+            # (and tls_certificate_path and tls_private_key_path above) to enable it.
             #
-            # enabled: true
+            #enabled: true
 
             # Endpoint to use to request certificates. If you only want to test,
             # use Let's Encrypt's staging url:
             #     https://acme-staging.api.letsencrypt.org/directory
             #
-            # url: https://acme-v01.api.letsencrypt.org/directory
+            #url: https://acme-v01.api.letsencrypt.org/directory
 
             # Port number to listen on for the HTTP-01 challenge. Change this if
             # you are forwarding connections through Apache/Nginx/etc.
             #
-            # port: 80
+            #port: 80
 
             # Local addresses to listen on for incoming connections.
             # Again, you may want to change this if you are forwarding connections
             # through Apache/Nginx/etc.
             #
-            # bind_addresses: ['::', '0.0.0.0']
+            #bind_addresses: ['::', '0.0.0.0']
 
             # How many days remaining on a certificate before it is renewed.
             #
-            # reprovision_threshold: 30
+            #reprovision_threshold: 30
 
-        # If your server runs behind a reverse-proxy which terminates TLS connections
-        # (for both client and federation connections), it may be useful to disable
-        # All TLS support for incoming connections. Setting no_tls to True will
-        # do so (and avoid the need to give synapse a TLS private key).
-        #
-        # no_tls: True
+            # The domain that the certificate should be for. Normally this
+            # should be the same as your Matrix domain (i.e., 'server_name'), but,
+            # by putting a file at 'https://<server_name>/.well-known/matrix/server',
+            # you can delegate incoming traffic to another server. If you do that,
+            # you should give the target of the delegation here.
+            #
+            # For example: if your 'server_name' is 'example.com', but
+            # 'https://example.com/.well-known/matrix/server' delegates to
+            # 'matrix.example.com', you should put 'matrix.example.com' here.
+            #
+            # If not set, defaults to your 'server_name'.
+            #
+            #domain: matrix.example.com
 
         # List of allowed TLS fingerprints for this server to publish along
         # with the signing keys for this server. Other matrix servers that
@@ -229,17 +274,44 @@ class TlsConfig(Config):
         #   openssl x509 -outform DER | openssl sha256 -binary | base64 | tr -d '='
         # or by checking matrix.org/federationtester/api/report?server_name=$host
         #
-        tls_fingerprints: []
-        # tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
+        #tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
 
         """
             % locals()
         )
 
-    def read_tls_certificate(self, cert_path):
-        cert_pem = self.read_file(cert_path, "tls_certificate")
-        return crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)
+    def read_tls_certificate(self):
+        """Reads the TLS certificate from the configured file, and returns it
 
-    def read_tls_private_key(self, private_key_path):
-        private_key_pem = self.read_file(private_key_path, "tls_private_key")
+        Also checks if it is self-signed, and warns if so
+
+        Returns:
+            OpenSSL.crypto.X509: the certificate
+        """
+        cert_path = self.tls_certificate_file
+        logger.info("Loading TLS certificate from %s", cert_path)
+        cert_pem = self.read_file(cert_path, "tls_certificate_path")
+        cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)
+
+        # Check if it is self-signed, and issue a warning if so.
+        if cert.get_issuer() == cert.get_subject():
+            warnings.warn(
+                (
+                    "Self-signed TLS certificates will not be accepted by Synapse 1.0. "
+                    "Please either provide a valid certificate, or use Synapse's ACME "
+                    "support to provision one."
+                )
+            )
+
+        return cert
+
+    def read_tls_private_key(self):
+        """Reads the TLS private key from the configured file, and returns it
+
+        Returns:
+            OpenSSL.crypto.PKey: the private key
+        """
+        private_key_path = self.tls_private_key_file
+        logger.info("Loading TLS key from %s", private_key_path)
+        private_key_pem = self.read_file(private_key_path, "tls_private_key_path")
         return crypto.load_privatekey(crypto.FILETYPE_PEM, private_key_pem)

synapse/config/user_directory.py

@@ -40,5 +40,5 @@ class UserDirectoryConfig(Config):
         # on your database to tell it to rebuild the user_directory search indexes.
         #
         #user_directory:
-        #   search_all_users: false
+        #  search_all_users: false
         """

synapse/config/voip.py

@@ -27,20 +27,24 @@ class VoipConfig(Config):
 
     def default_config(self, **kwargs):
         return """\
-        ## Turn ##
+        ## TURN ##
 
         # The public URIs of the TURN server to give to clients
+        #
         #turn_uris: []
 
         # The shared secret used to compute passwords for the TURN server
+        #
         #turn_shared_secret: "YOUR_SHARED_SECRET"
 
         # The Username and password if the TURN server needs them and
         # does not use a token
+        #
         #turn_username: "TURNSERVER_USERNAME"
         #turn_password: "TURNSERVER_PASSWORD"
 
         # How long generated TURN credentials last
+        #
         turn_user_lifetime: "1h"
 
         # Whether guests should be allowed to use the TURN server.
@@ -48,5 +52,6 @@ class VoipConfig(Config):
         # However, it does introduce a slight security risk as it allows users to
         # connect to arbitrary endpoints without having first signed up for a
         # valid account (e.g. by passing a CAPTCHA).
+        #
         turn_allow_guests: True
         """

synapse/crypto/context_factory.py

@@ -1,4 +1,5 @@
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import logging
 
 from zope.interface import implementer
@@ -43,9 +45,7 @@ class ServerContextFactory(ContextFactory):
             logger.exception("Failed to enable elliptic curve for TLS")
         context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)
         context.use_certificate_chain_file(config.tls_certificate_file)
-
-        if not config.no_tls:
-            context.use_privatekey(config.tls_private_key)
+        context.use_privatekey(config.tls_private_key)
 
         # https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
         context.set_cipher_list(
@@ -107,9 +107,7 @@ class ClientTLSOptions(object):
             self._hostnameBytes = _idnaBytes(hostname)
             self._sendSNI = True
 
-        ctx.set_info_callback(
-            _tolerateErrors(self._identityVerifyingInfoCallback)
-        )
+        ctx.set_info_callback(_tolerateErrors(self._identityVerifyingInfoCallback))
 
     def clientConnectionForTLS(self, tlsProtocol):
         context = self._ctx
@@ -130,10 +128,8 @@ class ClientTLSOptionsFactory(object):
 
     def __init__(self, config):
         # We don't use config options yet
-        pass
+        self._options = CertificateOptions(verify=False)
 
     def get_options(self, host):
-        return ClientTLSOptions(
-            host,
-            CertificateOptions(verify=False).getContext()
-        )
+        # Use _makeContext so that we get a fresh OpenSSL CTX each time.
+        return ClientTLSOptions(host, self._options._makeContext())

synapse/crypto/keyring.py

@@ -17,6 +17,7 @@
 import logging
 from collections import namedtuple
 
+from six import raise_from
 from six.moves import urllib
 
 from signedjson.key import (
@@ -35,7 +36,12 @@ from unpaddedbase64 import decode_base64
 
 from twisted.internet import defer
 
-from synapse.api.errors import Codes, SynapseError
+from synapse.api.errors import (
+    Codes,
+    HttpResponseException,
+    RequestSendFailed,
+    SynapseError,
+)
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.logcontext import (
     LoggingContext,
@@ -44,6 +50,7 @@ from synapse.util.logcontext import (
     run_in_background,
 )
 from synapse.util.metrics import Measure
+from synapse.util.retryutils import NotRetryingDestination
 
 logger = logging.getLogger(__name__)
 
@@ -367,13 +374,18 @@ class Keyring(object):
                     server_name_and_key_ids, perspective_name, perspective_keys
                 )
                 defer.returnValue(result)
+            except KeyLookupError as e:
+                logger.warning(
+                    "Key lookup failed from %r: %s", perspective_name, e,
+                )
             except Exception as e:
                 logger.exception(
                     "Unable to get key from %r: %s %s",
                     perspective_name,
                     type(e).__name__, str(e),
                 )
-                defer.returnValue({})
+
+            defer.returnValue({})
 
         results = yield logcontext.make_deferred_yieldable(defer.gatherResults(
             [
@@ -421,21 +433,30 @@ class Keyring(object):
         # TODO(mark): Set the minimum_valid_until_ts to that needed by
         # the events being validated or the current time if validating
         # an incoming request.
-        query_response = yield self.client.post_json(
-            destination=perspective_name,
-            path="/_matrix/key/v2/query",
-            data={
-                u"server_keys": {
-                    server_name: {
-                        key_id: {
-                            u"minimum_valid_until_ts": 0
-                        } for key_id in key_ids
+        try:
+            query_response = yield self.client.post_json(
+                destination=perspective_name,
+                path="/_matrix/key/v2/query",
+                data={
+                    u"server_keys": {
+                        server_name: {
+                            key_id: {
+                                u"minimum_valid_until_ts": 0
+                            } for key_id in key_ids
+                        }
+                        for server_name, key_ids in server_names_and_key_ids
                     }
-                    for server_name, key_ids in server_names_and_key_ids
-                }
-            },
-            long_retries=True,
-        )
+                },
+                long_retries=True,
+            )
+        except (NotRetryingDestination, RequestSendFailed) as e:
+            raise_from(
+                KeyLookupError("Failed to connect to remote server"), e,
+            )
+        except HttpResponseException as e:
+            raise_from(
+                KeyLookupError("Remote server returned an error"), e,
+            )
 
         keys = {}
 
@@ -502,11 +523,20 @@ class Keyring(object):
             if requested_key_id in keys:
                 continue
 
-            response = yield self.client.get_json(
-                destination=server_name,
-                path="/_matrix/key/v2/server/" + urllib.parse.quote(requested_key_id),
-                ignore_backoff=True,
-            )
+            try:
+                response = yield self.client.get_json(
+                    destination=server_name,
+                    path="/_matrix/key/v2/server/" + urllib.parse.quote(requested_key_id),
+                    ignore_backoff=True,
+                )
+            except (NotRetryingDestination, RequestSendFailed) as e:
+                raise_from(
+                    KeyLookupError("Failed to connect to remote server"), e,
+                )
+            except HttpResponseException as e:
+                raise_from(
+                    KeyLookupError("Remote server returned an error"), e,
+                )
 
             if (u"signatures" not in response
                     or server_name not in response[u"signatures"]):
@@ -656,7 +686,7 @@ def _handle_key_deferred(verify_request):
     try:
         with PreserveLoggingContext():
             _, key_id, verify_key = yield verify_request.deferred
-    except IOError as e:
+    except (IOError, RequestSendFailed) as e:
         logger.warn(
             "Got IOError when downloading keys for %s: %s %s",
             server_name, type(e).__name__, str(e),

synapse/federation/federation_client.py

@@ -33,6 +33,7 @@ from synapse.api.constants import (
 )
 from synapse.api.errors import (
     CodeMessageException,
+    Codes,
     FederationDeniedError,
     HttpResponseException,
     SynapseError,
@@ -792,10 +793,25 @@ class FederationClient(FederationBase):
             defer.returnValue(content)
         except HttpResponseException as e:
             if e.code in [400, 404]:
+                err = e.to_synapse_error()
+
+                # If we receive an error response that isn't a generic error, we
+                # assume that the remote understands the v2 invite API and this
+                # is a legitimate error.
+                if err.errcode != Codes.UNKNOWN:
+                    raise err
+
+                # Otherwise, we assume that the remote server doesn't understand
+                # the v2 invite API.
+
                 if room_version in (RoomVersions.V1, RoomVersions.V2):
                     pass  # We'll fall through
                 else:
-                    raise Exception("Remote server is too old")
+                    raise SynapseError(
+                        400,
+                        "User's homeserver does not support this room version",
+                        Codes.UNSUPPORTED_ROOM_VERSION,
+                    )
             elif e.code == 403:
                 raise e.to_synapse_error()
             else:

synapse/federation/federation_server.py

@@ -25,9 +25,10 @@ from twisted.internet import defer
 from twisted.internet.abstract import isIPAddress
 from twisted.python import failure
 
-from synapse.api.constants import EventTypes, Membership
+from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventTypes, Membership
 from synapse.api.errors import (
     AuthError,
+    Codes,
     FederationError,
     IncompatibleRoomVersionError,
     NotFoundError,
@@ -148,6 +149,22 @@ class FederationServer(FederationBase):
 
         logger.debug("[%s] Transaction is new", transaction.transaction_id)
 
+        # Reject if PDU count > 50 and EDU count > 100
+        if (len(transaction.pdus) > 50
+                or (hasattr(transaction, "edus") and len(transaction.edus) > 100)):
+
+            logger.info(
+                "Transaction PDU or EDU count too large. Returning 400",
+            )
+
+            response = {}
+            yield self.transaction_actions.set_response(
+                origin,
+                transaction,
+                400, response
+            )
+            defer.returnValue((400, response))
+
         received_pdus_counter.inc(len(transaction.pdus))
 
         origin_host, _ = parse_server_name(origin)
@@ -223,8 +240,9 @@ class FederationServer(FederationBase):
                         f = failure.Failure()
                         pdu_results[event_id] = {"error": str(e)}
                         logger.error(
-                            "Failed to handle PDU %s: %s",
-                            event_id, f.getTraceback().rstrip(),
+                            "Failed to handle PDU %s",
+                            event_id,
+                            exc_info=(f.type, f.value, f.getTracebackObject()),
                         )
 
         yield concurrently_execute(
@@ -370,6 +388,13 @@ class FederationServer(FederationBase):
 
     @defer.inlineCallbacks
     def on_invite_request(self, origin, content, room_version):
+        if room_version not in KNOWN_ROOM_VERSIONS:
+            raise SynapseError(
+                400,
+                "Homeserver does not support this room version",
+                Codes.UNSUPPORTED_ROOM_VERSION,
+            )
+
         format_ver = room_version_to_event_format(room_version)
 
         pdu = event_from_pdu_json(content, format_ver)
@@ -861,6 +886,9 @@ class ReplicationFederationHandlerRegistry(FederationHandlerRegistry):
     def on_edu(self, edu_type, origin, content):
         """Overrides FederationHandlerRegistry
         """
+        if not self.config.use_presence and edu_type == "m.presence":
+            return
+
         handler = self.edu_handlers.get(edu_type)
         if handler:
             return super(ReplicationFederationHandlerRegistry, self).on_edu(

synapse/federation/send_queue.py

@@ -159,8 +159,12 @@ class FederationRemoteSendQueue(object):
         # stream.
         pass
 
-    def send_edu(self, destination, edu_type, content, key=None):
+    def build_and_send_edu(self, destination, edu_type, content, key=None):
         """As per TransactionQueue"""
+        if destination == self.server_name:
+            logger.info("Not sending EDU to ourselves")
+            return
+
         pos = self._next_pos()
 
         edu = Edu(
@@ -465,15 +469,11 @@ def process_rows_for_federation(transaction_queue, rows):
 
     for destination, edu_map in iteritems(buff.keyed_edus):
         for key, edu in edu_map.items():
-            transaction_queue.send_edu(
-                edu.destination, edu.edu_type, edu.content, key=key,
-            )
+            transaction_queue.send_edu(edu, key)
 
     for destination, edu_list in iteritems(buff.edus):
         for edu in edu_list:
-            transaction_queue.send_edu(
-                edu.destination, edu.edu_type, edu.content, key=None,
-            )
+            transaction_queue.send_edu(edu, None)
 
     for destination in buff.device_destinations:
         transaction_queue.send_device_messages(destination)