WIP

Merge remote-tracking branch 'origin/release-v0.99.0' into develop
v0.99.0rc2
2019-01-31 18:31:17 +00:00 · 2019-01-30 17:02:32 +00:00 · 2019-01-30 16:31:07 +00:00 · 2019-01-30 16:26:13 +00:00 · 2019-01-30 16:23:28 +00:00 · 2019-01-30 16:22:37 +00:00
255 changed files with 11170 additions and 3920 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -4,8 +4,8 @@ jobs:
    machine: true
    steps:
      - checkout
-      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_TAG} .
-      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 .
      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
@@ -13,13 +13,9 @@ jobs:
    machine: true
    steps:
      - checkout
-      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_SHA1} .
-      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_SHA1}-py3 --build-arg PYTHON_VERSION=3.6 .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest-py3 --build-arg PYTHON_VERSION=3.6 .
      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      - run: docker tag matrixdotorg/synapse:${CIRCLE_SHA1} matrixdotorg/synapse:latest
-      - run: docker tag matrixdotorg/synapse:${CIRCLE_SHA1}-py3 matrixdotorg/synapse:latest-py3
-      - run: docker push matrixdotorg/synapse:${CIRCLE_SHA1}
-      - run: docker push matrixdotorg/synapse:${CIRCLE_SHA1}-py3
      - run: docker push matrixdotorg/synapse:latest
      - run: docker push matrixdotorg/synapse:latest-py3
  sytestpy2:
--- a/.circleci/merge_base_branch.sh
+++ b/.circleci/merge_base_branch.sh
@@ -20,7 +20,7 @@ else
 fi

 # Show what we are before
-git show -s
+git --no-pager show -s

 # Set up username so it can do a merge
 git config --global user.email bot@matrix.org
@@ -31,4 +31,4 @@ git fetch -u origin $GITBASE
 git merge --no-edit origin/$GITBASE

 # Show what we are after.
-git show -s
+git --no-pager show -s
--- a/.codecov.yml
+++ b/.codecov.yml
@@ -0,0 +1,15 @@
+comment:
+  layout: "diff"
+
+coverage:
+  status:
+    project:
+      default:
+        target: 0  # Target % coverage, can be auto. Turned off for now
+        threshold: null
+        base: auto
+    patch:
+      default:
+        target: 0
+        threshold: null
+        base: auto
--- a/.coveragerc
+++ b/.coveragerc
@@ -1,12 +1,7 @@
 [run]
 branch = True
 parallel = True
-source = synapse
-
-[paths]
-source=
-   coverage
+include = synapse/*

 [report]
 precision = 2
-ignore_errors = True
--- a/.dockerignore
+++ b/.dockerignore
@@ -5,3 +5,5 @@ demo/etc
 tox.ini
 .git/*
 .tox/*
+debian/matrix-synapse/
+debian/matrix-synapse-*/
--- a/.editorconfig
+++ b/.editorconfig
@@ -0,0 +1,9 @@
+# EditorConfig https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# 4 space indentation
+[*.py]
+indent_style = space
+indent_size = 4
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -3,5 +3,5 @@
 <!-- Please read CONTRIBUTING.rst before submitting your pull request -->

 * [ ] Pull request is based on the develop branch
-* [ ] Pull request includes a [changelog file](CONTRIBUTING.rst#changelog)
-* [ ] Pull request includes a [sign off](CONTRIBUTING.rst#sign-off)
+* [ ] Pull request includes a [changelog file](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst#changelog)
+* [ ] Pull request includes a [sign off](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst#sign-off)
--- a/.gitignore
+++ b/.gitignore
@@ -12,20 +12,23 @@ dbs/
 dist/
 docs/build/
 *.egg-info
+pip-wheel-metadata/

 cmdclient_config.json
 homeserver*.db
 homeserver*.log
 homeserver*.log.*
 homeserver*.pid
-homeserver*.yaml
+/homeserver*.yaml

 *.signing.key
 *.tls.crt
 *.tls.dh
 *.tls.key

-.coverage
+.coverage*
+coverage.*
+!.coveragerc
 htmlcov

 demo/*/*.db
@@ -57,3 +60,5 @@ env/

 .vscode/
 .ropeproject/
+*.deb
+/debs
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,6 +12,9 @@ cache:
    #
    - $HOME/.cache/pip/wheels

+addons:
+  postgresql: "9.4"
+
 # don't clone the whole repo history, one commit will do
 git:
  depth: 1
@@ -36,24 +39,24 @@ matrix:
    env: TOX_ENV="pep8,check_isort"

  - python: 2.7
-    env: TOX_ENV=py27 TRIAL_FLAGS="-j 2"
+    env: TOX_ENV=py27,codecov TRIAL_FLAGS="-j 2"

  - python: 2.7
    env: TOX_ENV=py27-old TRIAL_FLAGS="-j 2"

  - python: 2.7
-    env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
+    env: TOX_ENV=py27-postgres,codecov TRIAL_FLAGS="-j 4"
    services:
      - postgresql

  - python: 3.5
-    env: TOX_ENV=py35 TRIAL_FLAGS="-j 2"
+    env: TOX_ENV=py35,codecov TRIAL_FLAGS="-j 2"

  - python: 3.6
-    env: TOX_ENV=py36 TRIAL_FLAGS="-j 2"
+    env: TOX_ENV=py36,codecov TRIAL_FLAGS="-j 2"

  - python: 3.6
-    env: TOX_ENV=py36-postgres TRIAL_FLAGS="-j 4"
+    env: TOX_ENV=py36-postgres,codecov TRIAL_FLAGS="-j 4"
    services:
      - postgresql

@@ -68,6 +71,13 @@ matrix:

 install:
  - pip install tox
+  
+  # if we don't have python3.6 in this environment, travis unhelpfully gives us
+  # a `python3.6` on our path which does nothing but spit out a warning. Tox
+  # tries to run it (even if we're not running a py36 env), so the build logs
+  # then have warnings which look like errors. To reduce the noise, remove the
+  # non-functional python3.6.
+  - ( ! command -v python3.6 || python3.6 --version ) &>/dev/null || rm -f $(command -v python3.6)

 script:
  - tox -e $TOX_ENV
--- a/AUTHORS.rst
+++ b/AUTHORS.rst
@@ -65,4 +65,7 @@ Pierre Jaury <pierre at jaury.eu>
 * Docker packaging

 Serban Constantin <serban.constantin at gmail dot com>
- * Small bug fix
+ * Small bug fix
+
+Jason Robinson <jasonr at matrix.org>
+ * Minor fixes
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,233 @@
+Synapse 0.99.0rc2 (2019-01-30)
+==============================
+
+Bugfixes
+--------
+
+- Fix bug when rejecting remote invites. ([\#4527](https://github.com/matrix-org/synapse/issues/4527))
+- Fix incorrect rendering of server capabilities. ([81b7e7eed](https://github.com/matrix-org/synapse/commit/81b7e7eed323f55d6550e7a270a9dc2c4c7b0fe0))
+
+Improved Documentation
+----------------------
+
+- Add documentation on enabling ACME support when upgrading to v0.99. ([\#4528](https://github.com/matrix-org/synapse/issues/4528))
+
+
+Synapse 0.99.0rc1 (2019-01-30)
+==============================
+
+Synapse v0.99.x is a precursor to the upcoming Synapse v1.0 release. It contains foundational changes to room architecture and the federation security model necessary to support the upcoming r0 release of the Server to Server API.
+
+Features
+--------
+
+- Synapse's cipher string has been updated to require ECDH key exchange. Configuring and generating dh_params is no longer required, and they will be ignored. ([\#4229](https://github.com/matrix-org/synapse/issues/4229))
+- Synapse can now automatically provision TLS certificates via ACME (the protocol used by CAs like Let's Encrypt). ([\#4384](https://github.com/matrix-org/synapse/issues/4384), [\#4492](https://github.com/matrix-org/synapse/issues/4492), [\#4525](https://github.com/matrix-org/synapse/issues/4525))
+- Implement MSC1708 (.well-known routing for server-server federation) ([\#4408](https://github.com/matrix-org/synapse/issues/4408), [\#4409](https://github.com/matrix-org/synapse/issues/4409), [\#4426](https://github.com/matrix-org/synapse/issues/4426), [\#4427](https://github.com/matrix-org/synapse/issues/4427), [\#4428](https://github.com/matrix-org/synapse/issues/4428), [\#4464](https://github.com/matrix-org/synapse/issues/4464), [\#4468](https://github.com/matrix-org/synapse/issues/4468), [\#4487](https://github.com/matrix-org/synapse/issues/4487), [\#4488](https://github.com/matrix-org/synapse/issues/4488), [\#4489](https://github.com/matrix-org/synapse/issues/4489), [\#4497](https://github.com/matrix-org/synapse/issues/4497), [\#4511](https://github.com/matrix-org/synapse/issues/4511), [\#4516](https://github.com/matrix-org/synapse/issues/4516), [\#4520](https://github.com/matrix-org/synapse/issues/4520), [\#4521](https://github.com/matrix-org/synapse/issues/4521))
+- Search now includes results from predecessor rooms after a room upgrade. ([\#4415](https://github.com/matrix-org/synapse/issues/4415))
+- Config option to disable requesting MSISDN on registration. ([\#4423](https://github.com/matrix-org/synapse/issues/4423))
+- Add a metric for tracking event stream position of the user directory. ([\#4445](https://github.com/matrix-org/synapse/issues/4445))
+- Support exposing server capabilities in CS API (MSC1753, MSC1804) ([\#4472](https://github.com/matrix-org/synapse/issues/4472))
+- Add support for room version 3 ([\#4483](https://github.com/matrix-org/synapse/issues/4483), [\#4499](https://github.com/matrix-org/synapse/issues/4499), [\#4515](https://github.com/matrix-org/synapse/issues/4515), [\#4523](https://github.com/matrix-org/synapse/issues/4523))
+- Synapse will now reload TLS certificates from disk upon SIGHUP. ([\#4495](https://github.com/matrix-org/synapse/issues/4495), [\#4524](https://github.com/matrix-org/synapse/issues/4524))
+
+
+Bugfixes
+--------
+
+- Prevent users with access tokens predating the introduction of device IDs from creating spurious entries in the user_ips table. ([\#4369](https://github.com/matrix-org/synapse/issues/4369))
+- Fix typo in ALL_USER_TYPES definition to ensure type is a tuple ([\#4392](https://github.com/matrix-org/synapse/issues/4392))
+- Fix high CPU usage due to remote devicelist updates ([\#4397](https://github.com/matrix-org/synapse/issues/4397))
+- Fix potential bug where creating or joining a room could fail ([\#4404](https://github.com/matrix-org/synapse/issues/4404))
+- Fix bug when rejecting remote invites ([\#4405](https://github.com/matrix-org/synapse/issues/4405))
+- Fix incorrect logcontexts after a Deferred was cancelled ([\#4407](https://github.com/matrix-org/synapse/issues/4407))
+- Ensure encrypted room state is persisted across room upgrades. ([\#4411](https://github.com/matrix-org/synapse/issues/4411))
+- Copy over whether a room is a direct message and any associated room tags on room upgrade. ([\#4412](https://github.com/matrix-org/synapse/issues/4412))
+- Fix None guard in calling config.server.is_threepid_reserved ([\#4435](https://github.com/matrix-org/synapse/issues/4435))
+- Don't send IP addresses as SNI ([\#4452](https://github.com/matrix-org/synapse/issues/4452))
+- Fix UnboundLocalError in post_urlencoded_get_json ([\#4460](https://github.com/matrix-org/synapse/issues/4460))
+- Add a timeout to filtered room directory queries. ([\#4461](https://github.com/matrix-org/synapse/issues/4461))
+- Workaround for login error when using both LDAP and internal authentication. ([\#4486](https://github.com/matrix-org/synapse/issues/4486))
+- Fix a bug where setting a relative consent directory path would cause a crash. ([\#4512](https://github.com/matrix-org/synapse/issues/4512))
+
+
+Deprecations and Removals
+-------------------------
+
+- Synapse no longer generates self-signed TLS certificates when generating a configuration file. ([\#4509](https://github.com/matrix-org/synapse/issues/4509))
+
+
+Internal Changes
+----------------
+
+- Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+. ([\#4306](https://github.com/matrix-org/synapse/issues/4306), [\#4459](https://github.com/matrix-org/synapse/issues/4459), [\#4466](https://github.com/matrix-org/synapse/issues/4466), [\#4471](https://github.com/matrix-org/synapse/issues/4471), [\#4477](https://github.com/matrix-org/synapse/issues/4477), [\#4505](https://github.com/matrix-org/synapse/issues/4505))
+- Update README to use the new virtualenv everywhere ([\#4342](https://github.com/matrix-org/synapse/issues/4342))
+- Add better logging for unexpected errors while sending transactions ([\#4368](https://github.com/matrix-org/synapse/issues/4368))
+- Apply a unique index to the user_ips table, preventing duplicates. ([\#4370](https://github.com/matrix-org/synapse/issues/4370), [\#4432](https://github.com/matrix-org/synapse/issues/4432), [\#4434](https://github.com/matrix-org/synapse/issues/4434))
+- Silence travis-ci build warnings by removing non-functional python3.6 ([\#4377](https://github.com/matrix-org/synapse/issues/4377))
+- Fix a comment in the generated config file ([\#4387](https://github.com/matrix-org/synapse/issues/4387))
+- Add ground work for implementing future federation API versions ([\#4390](https://github.com/matrix-org/synapse/issues/4390))
+- Update dependencies on msgpack and pymacaroons to use the up-to-date packages. ([\#4399](https://github.com/matrix-org/synapse/issues/4399))
+- Tweak codecov settings to make them less loud. ([\#4400](https://github.com/matrix-org/synapse/issues/4400))
+- Implement server support for MSC1794 - Federation v2 Invite API ([\#4402](https://github.com/matrix-org/synapse/issues/4402))
+- debian package: symlink to explicit python version ([\#4433](https://github.com/matrix-org/synapse/issues/4433))
+- Add infrastructure to support different event formats ([\#4437](https://github.com/matrix-org/synapse/issues/4437), [\#4447](https://github.com/matrix-org/synapse/issues/4447), [\#4448](https://github.com/matrix-org/synapse/issues/4448), [\#4470](https://github.com/matrix-org/synapse/issues/4470), [\#4481](https://github.com/matrix-org/synapse/issues/4481), [\#4482](https://github.com/matrix-org/synapse/issues/4482), [\#4493](https://github.com/matrix-org/synapse/issues/4493), [\#4494](https://github.com/matrix-org/synapse/issues/4494), [\#4496](https://github.com/matrix-org/synapse/issues/4496), [\#4510](https://github.com/matrix-org/synapse/issues/4510), [\#4514](https://github.com/matrix-org/synapse/issues/4514))
+- Generate the debian config during build ([\#4444](https://github.com/matrix-org/synapse/issues/4444))
+- Clarify documentation for the `public_baseurl` config param ([\#4458](https://github.com/matrix-org/synapse/issues/4458), [\#4498](https://github.com/matrix-org/synapse/issues/4498))
+- Fix quoting for allowed_local_3pids example config ([\#4476](https://github.com/matrix-org/synapse/issues/4476))
+- Remove deprecated --process-dependency-links option from UPGRADE.rst ([\#4485](https://github.com/matrix-org/synapse/issues/4485))
+- Make it possible to set the log level for tests via an environment variable ([\#4506](https://github.com/matrix-org/synapse/issues/4506))
+- Reduce the log level of linearizer lock acquirement to DEBUG. ([\#4507](https://github.com/matrix-org/synapse/issues/4507))
+- Fix code to comply with linting in PyFlakes 3.7.1. ([\#4519](https://github.com/matrix-org/synapse/issues/4519))
+
+
+Synapse 0.34.1.1 (2019-01-11)
+=============================
+
+This release fixes CVE-2019-5885 and is recommended for all users of Synapse 0.34.1.
+
+This release is compatible with Python 2.7 and 3.5+. Python 3.7 is fully supported.
+
+Bugfixes
+--------
+
+- Fix spontaneous logout on upgrade
+  ([\#4374](https://github.com/matrix-org/synapse/issues/4374))
+
+
+Synapse 0.34.1 (2019-01-09)
+===========================
+
+Internal Changes
+----------------
+
+- Add better logging for unexpected errors while sending transactions ([\#4361](https://github.com/matrix-org/synapse/issues/4361), [\#4362](https://github.com/matrix-org/synapse/issues/4362))
+
+
+Synapse 0.34.1rc1 (2019-01-08)
+==============================
+
+Features
+--------
+
+- Special-case a support user for use in verifying behaviour of a given server. The support user does not appear in user directory or monthly active user counts. ([\#4141](https://github.com/matrix-org/synapse/issues/4141), [\#4344](https://github.com/matrix-org/synapse/issues/4344))
+- Support for serving .well-known files ([\#4262](https://github.com/matrix-org/synapse/issues/4262))
+- Rework SAML2 authentication ([\#4265](https://github.com/matrix-org/synapse/issues/4265), [\#4267](https://github.com/matrix-org/synapse/issues/4267))
+- SAML2 authentication: Initialise user display name from SAML2 data ([\#4272](https://github.com/matrix-org/synapse/issues/4272))
+- Synapse can now have its conditional/extra dependencies installed by pip. This functionality can be used by using `pip install matrix-synapse[feature]`, where feature is a comma separated list with the possible values `email.enable_notifs`, `matrix-synapse-ldap3`, `postgres`, `resources.consent`, `saml2`, `url_preview`, and `test`. If you want to install all optional dependencies, you can use "all" instead. ([\#4298](https://github.com/matrix-org/synapse/issues/4298), [\#4325](https://github.com/matrix-org/synapse/issues/4325), [\#4327](https://github.com/matrix-org/synapse/issues/4327))
+- Add routes for reading account data. ([\#4303](https://github.com/matrix-org/synapse/issues/4303))
+- Add opt-in support for v2 rooms ([\#4307](https://github.com/matrix-org/synapse/issues/4307))
+- Add a script to generate a clean config file ([\#4315](https://github.com/matrix-org/synapse/issues/4315))
+- Return server data in /login response ([\#4319](https://github.com/matrix-org/synapse/issues/4319))
+
+
+Bugfixes
+--------
+
+- Fix contains_url check to be consistent with other instances in code-base and check that value is an instance of string. ([\#3405](https://github.com/matrix-org/synapse/issues/3405))
+- Fix CAS login when username is not valid in an MXID ([\#4264](https://github.com/matrix-org/synapse/issues/4264))
+- Send CORS headers for /media/config ([\#4279](https://github.com/matrix-org/synapse/issues/4279))
+- Add 'sandbox' to CSP for media reprository ([\#4284](https://github.com/matrix-org/synapse/issues/4284))
+- Make the new landing page prettier. ([\#4294](https://github.com/matrix-org/synapse/issues/4294))
+- Fix deleting E2E room keys when using old SQLite versions. ([\#4295](https://github.com/matrix-org/synapse/issues/4295))
+- The metric synapse_admin_mau:current previously did not update when config.mau_stats_only was set to True ([\#4305](https://github.com/matrix-org/synapse/issues/4305))
+- Fixed per-room account data filters ([\#4309](https://github.com/matrix-org/synapse/issues/4309))
+- Fix indentation in default config ([\#4313](https://github.com/matrix-org/synapse/issues/4313))
+- Fix synapse:latest docker upload ([\#4316](https://github.com/matrix-org/synapse/issues/4316))
+- Fix test_metric.py compatibility with prometheus_client 0.5. Contributed by Maarten de Vries <maarten@de-vri.es>. ([\#4317](https://github.com/matrix-org/synapse/issues/4317))
+- Avoid packaging _trial_temp directory in -py3 debian packages ([\#4326](https://github.com/matrix-org/synapse/issues/4326))
+- Check jinja version for consent resource ([\#4327](https://github.com/matrix-org/synapse/issues/4327))
+- fix NPE in /messages by checking if all events were filtered out ([\#4330](https://github.com/matrix-org/synapse/issues/4330))
+- Fix `python -m synapse.config` on Python 3. ([\#4356](https://github.com/matrix-org/synapse/issues/4356))
+
+
+Deprecations and Removals
+-------------------------
+
+- Remove the deprecated v1/register API on Python 2. It was never ported to Python 3. ([\#4334](https://github.com/matrix-org/synapse/issues/4334))
+
+
+Internal Changes
+----------------
+
+- Getting URL previews of IP addresses no longer fails on Python 3. ([\#4215](https://github.com/matrix-org/synapse/issues/4215))
+- drop undocumented dependency on dateutil ([\#4266](https://github.com/matrix-org/synapse/issues/4266))
+- Update the example systemd config to use a virtualenv ([\#4273](https://github.com/matrix-org/synapse/issues/4273))
+- Update link to kernel DCO guide ([\#4274](https://github.com/matrix-org/synapse/issues/4274))
+- Make isort tox check print diff when it fails ([\#4283](https://github.com/matrix-org/synapse/issues/4283))
+- Log room_id in Unknown room errors ([\#4297](https://github.com/matrix-org/synapse/issues/4297))
+- Documentation improvements for coturn setup. Contributed by Krithin Sitaram. ([\#4333](https://github.com/matrix-org/synapse/issues/4333))
+- Update pull request template to use absolute links ([\#4341](https://github.com/matrix-org/synapse/issues/4341))
+- Update README to not lie about required restart when updating TLS certificates ([\#4343](https://github.com/matrix-org/synapse/issues/4343))
+- Update debian packaging for compatibility with transitional package ([\#4349](https://github.com/matrix-org/synapse/issues/4349))
+- Fix command hint to generate a config file when trying to start without a config file ([\#4353](https://github.com/matrix-org/synapse/issues/4353))
+- Add better logging for unexpected errors while sending transactions ([\#4358](https://github.com/matrix-org/synapse/issues/4358))
+
+
+Synapse 0.34.0 (2018-12-20)
+===========================
+
+Synapse 0.34.0 is the first release to fully support Python 3. Synapse will now
+run on Python versions 3.5 or 3.6 (as well as 2.7). Support for Python 3.7
+remains experimental.
+
+We recommend upgrading to Python 3, but make sure to read the [upgrade
+notes](UPGRADE.rst#upgrading-to-v0340) when doing so.
+
+Features
+--------
+
+- Add 'sandbox' to CSP for media reprository ([\#4284](https://github.com/matrix-org/synapse/issues/4284))
+- Make the new landing page prettier. ([\#4294](https://github.com/matrix-org/synapse/issues/4294))
+- Fix deleting E2E room keys when using old SQLite versions. ([\#4295](https://github.com/matrix-org/synapse/issues/4295))
+- Add a welcome page for the client API port. Credit to @krombel! ([\#4289](https://github.com/matrix-org/synapse/issues/4289))
+- Remove Matrix console from the default distribution ([\#4290](https://github.com/matrix-org/synapse/issues/4290))
+- Add option to track MAU stats (but not limit people) ([\#3830](https://github.com/matrix-org/synapse/issues/3830))
+- Add an option to enable recording IPs for appservice users ([\#3831](https://github.com/matrix-org/synapse/issues/3831))
+- Rename login type `m.login.cas` to `m.login.sso` ([\#4220](https://github.com/matrix-org/synapse/issues/4220))
+- Add an option to disable search for homeservers that may not be interested in it. ([\#4230](https://github.com/matrix-org/synapse/issues/4230))
+
+
+Bugfixes
+--------
+
+- Pushrules can now again be made with non-ASCII rule IDs. ([\#4165](https://github.com/matrix-org/synapse/issues/4165))
+- The media repository now no longer fails to decode UTF-8 filenames when downloading remote media. ([\#4176](https://github.com/matrix-org/synapse/issues/4176))
+- URL previews now correctly decode non-UTF-8 text if the header contains a `<meta http-equiv="Content-Type"` header. ([\#4183](https://github.com/matrix-org/synapse/issues/4183))
+- Fix an issue where public consent URLs had two slashes. ([\#4192](https://github.com/matrix-org/synapse/issues/4192))
+- Fallback auth now accepts the session parameter on Python 3. ([\#4197](https://github.com/matrix-org/synapse/issues/4197))
+- Remove riot.im from the list of trusted Identity Servers in the default configuration ([\#4207](https://github.com/matrix-org/synapse/issues/4207))
+- fix start up failure when mau_limit_reserved_threepids set and db is postgres ([\#4211](https://github.com/matrix-org/synapse/issues/4211))
+- Fix auto join failures for servers that require user consent ([\#4223](https://github.com/matrix-org/synapse/issues/4223))
+- Fix exception caused by non-ascii event IDs ([\#4241](https://github.com/matrix-org/synapse/issues/4241))
+- Pushers can now be unsubscribed from on Python 3. ([\#4250](https://github.com/matrix-org/synapse/issues/4250))
+- Fix UnicodeDecodeError when postgres is configured to give non-English errors ([\#4253](https://github.com/matrix-org/synapse/issues/4253))
+
+
+Internal Changes
+----------------
+
+- Debian packages utilising a virtualenv with bundled dependencies can now be built. ([\#4212](https://github.com/matrix-org/synapse/issues/4212))
+- Disable pager when running git-show in CI ([\#4291](https://github.com/matrix-org/synapse/issues/4291))
+- A coveragerc file has been added. ([\#4180](https://github.com/matrix-org/synapse/issues/4180))
+- Add a GitHub pull request template and add multiple issue templates ([\#4182](https://github.com/matrix-org/synapse/issues/4182))
+- Update README to reflect the fact that [\#1491](https://github.com/matrix-org/synapse/issues/1491) is fixed ([\#4188](https://github.com/matrix-org/synapse/issues/4188))
+- Run the AS senders as background processes to fix warnings ([\#4189](https://github.com/matrix-org/synapse/issues/4189))
+- Add some diagnostics to the tests to detect logcontext problems ([\#4190](https://github.com/matrix-org/synapse/issues/4190))
+- Add missing `jpeg` package prerequisite for OpenBSD in README. ([\#4193](https://github.com/matrix-org/synapse/issues/4193))
+- Add a note saying you need to manually reclaim disk space after using the Purge History API ([\#4200](https://github.com/matrix-org/synapse/issues/4200))
+- More logcontext checking in unittests ([\#4205](https://github.com/matrix-org/synapse/issues/4205))
+- Ignore `__pycache__` directories in the database schema folder ([\#4214](https://github.com/matrix-org/synapse/issues/4214))
+- Add note to UPGRADE.rst about removing riot.im from list of trusted identity servers ([\#4224](https://github.com/matrix-org/synapse/issues/4224))
+- Added automated coverage reporting to CI. ([\#4225](https://github.com/matrix-org/synapse/issues/4225))
+- Garbage-collect after each unit test to fix logcontext leaks ([\#4227](https://github.com/matrix-org/synapse/issues/4227))
+- add more detail to logging regarding "More than one row matched" error ([\#4234](https://github.com/matrix-org/synapse/issues/4234))
+- Drop sent_transactions table ([\#4244](https://github.com/matrix-org/synapse/issues/4244))
+- Add a basic .editorconfig ([\#4257](https://github.com/matrix-org/synapse/issues/4257))
+- Update README.rst and UPGRADE.rst for Python 3. ([\#4260](https://github.com/matrix-org/synapse/issues/4260))
+- Remove obsolete `verbose` and `log_file` settings from `homeserver.yaml` for Docker image. ([\#4261](https://github.com/matrix-org/synapse/issues/4261))
+
+
 Synapse 0.33.9 (2018-11-19)
 ===========================

@@ -71,7 +301,7 @@ Synapse 0.33.8rc2 (2018-10-31)
 Bugfixes
 --------

- Searches that request profile info now no longer fail with a 500. Fixes 
+- Searches that request profile info now no longer fail with a 500. Fixes
  a regression in 0.33.8rc1. ([\#4122](https://github.com/matrix-org/synapse/issues/4122))


--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -102,7 +102,7 @@ Sign off
 In order to have a concrete record that your contribution is intentional
 and you agree to license it under the same terms as the project's license, we've adopted the
 same lightweight approach that the Linux Kernel
-(https://www.kernel.org/doc/Documentation/SubmittingPatches), Docker
+`submitting patches process <https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin>`_, Docker
 (https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other
 projects use: the DCO (Developer Certificate of Origin:
 http://developercertificate.org/). This is a simple declaration that you wrote
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -15,6 +15,7 @@ recursive-include docs *
 recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
+recursive-include tests *.pem
 recursive-include tests *.py

 recursive-include synapse/res *
@@ -26,6 +27,7 @@ recursive-include synapse/static *.js
 exclude Dockerfile
 exclude .dockerignore
 exclude test_postgresql.sh
+exclude .editorconfig

 include pyproject.toml
 recursive-include changelog.d *
@@ -35,6 +37,8 @@ prune demo/etc
 prune docker
 prune .circleci
 prune .coveragerc
+prune debian
+prune .codecov.yml

 exclude jenkins*
 recursive-exclude jenkins *.sh
--- a/README.rst
+++ b/README.rst
@@ -4,15 +4,15 @@ Introduction
 ============

 Matrix is an ambitious new ecosystem for open federated Instant Messaging and
-VoIP.  The basics you need to know to get up and running are:
+VoIP. The basics you need to know to get up and running are:

- Everything in Matrix happens in a room.  Rooms are distributed and do not
-  exist on any single server.  Rooms can be located using convenience aliases
+- Everything in Matrix happens in a room. Rooms are distributed and do not
+  exist on any single server. Rooms can be located using convenience aliases
  like ``#matrix:matrix.org`` or ``#test:localhost:8448``.

 - Matrix user IDs look like ``@matthew:matrix.org`` (although in the future
  you will normally refer to yourself and others using a third party identifier
-  (3PID): email address, phone number, etc rather than manipulating Matrix user IDs)
+  (3PID): email address, phone number, etc rather than manipulating Matrix user IDs).

 The overall architecture is::

@@ -86,115 +86,222 @@ Synapse is the reference Python/Twisted Matrix homeserver implementation.
 System requirements:

 - POSIX-compliant system (tested on Linux & OS X)
- Python 2.7
+- Python 3.5, 3.6, 3.7, or 2.7
 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org

-Installing from source
----------------------
+The currently supported environment is [Ubuntu 18.04
+LTS](http://releases.ubuntu.com/18.04/).

-(Prebuilt packages are available for some platforms - see `Platform-Specific
-Instructions`_.)
+Recommended installation procedure
+----------------------------------

-Synapse is written in Python but some of the libraries it uses are written in
-C. So before we can install Synapse itself we need a working C compiler and the
-header files for Python C extensions.
+Building and running Synapse from source in a python3 environment is the
+recommended path for installation, as it is the most well-tested route.
+Binary packages are available for various platforms, but not officially
+supported by the Synapse team. See `Platform Specific Instructions`_ for
+details.
+
+Install prerequisites
+*********************

 Installing prerequisites on Ubuntu or Debian::

-    sudo apt-get install build-essential python2.7-dev libffi-dev \
-                         python-pip python-setuptools sqlite3 \
-                         libssl-dev python-virtualenv libjpeg-dev libxslt1-dev
+    sudo apt-get update && sudo apt-get dist-upgrade
+    sudo apt-get install build-essential python3-dev python3-venv \
+                         python3-pip python-setuptools libssl-dev \
+                         libjpeg-dev libffi-dev zlib1g-dev \
+                         libxslt1-dev postgresql libwebp-dev libpq-dev
+
+**TODO: Update and check non-debian distro pre-req's for new process**

 Installing prerequisites on ArchLinux::

-    sudo pacman -S base-devel python2 python-pip \
-                   python-setuptools python-virtualenv sqlite3
+    sudo pacman -S base-devel python python-pip \
+                   python-setuptools python-virtualenv

-Installing prerequisites on CentOS 7 or Fedora 25::
+Installing prerequisites on CentOS 7 or Fedora::

    sudo yum install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
                     lcms2-devel libwebp-devel tcl-devel tk-devel redhat-rpm-config \
                     python-virtualenv libffi-devel openssl-devel
    sudo yum groupinstall "Development Tools"

-Installing prerequisites on Mac OS X::
-
-    xcode-select --install
-    sudo easy_install pip
-    sudo pip install virtualenv
-    brew install pkg-config libffi
-
 Installing prerequisites on Raspbian::

-    sudo apt-get install build-essential python2.7-dev libffi-dev \
-                         python-pip python-setuptools sqlite3 \
-                         libssl-dev python-virtualenv libjpeg-dev
-    sudo pip install --upgrade pip
-    sudo pip install --upgrade ndg-httpsclient
-    sudo pip install --upgrade virtualenv
+    sudo apt-get update && sudo apt-get dist-upgrade
+    sudo apt-get install build-essential python3-dev python3-venv \
+                         python3-pip python-setuptools libssl-dev \
+                         libjpeg-dev libffi-dev zlib1g-dev \
+                         libxslt1-dev postgresql libwebp-dev libpq-dev

-Installing prerequisites on openSUSE::

-    sudo zypper in -t pattern devel_basis
-    sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \
-                   python-devel libffi-devel libopenssl-devel libjpeg62-devel
+Set up python environment
+*************************

-Installing prerequisites on OpenBSD::
+Add a new user for Synapse and log in as them::

-    doas pkg_add python libffi py-pip py-setuptools sqlite3 py-virtualenv \
-                 libxslt jpeg
+    useradd matrix
+    su -l matrix

-To install the Synapse homeserver run::
+Create a python3 virtualenv and install dependencies::

-    virtualenv -p python2.7 ~/.synapse
-    source ~/.synapse/bin/activate
-    pip install --upgrade pip
-    pip install --upgrade setuptools
-    pip install matrix-synapse
+    python3 -m venv matrix-synapse
+    ./matrix-synapse/bin/python -m pip install -U pip setuptools wheel
+    ./matrix-synapse/bin/python -m pip install -U matrix-synapse[all]

-This installs Synapse, along with the libraries it uses, into a virtual
-environment under ``~/.synapse``.  Feel free to pick a different directory
-if you prefer.
+Create a Synapse configuration directory. **Make sure you change
+``matrix.mydomain.com`` to your own domain**::

-This Synapse installation can then be later upgraded by using pip again with the
-update flag::
+    mkdir cfg
+    ./matrix-synapse/bin/python -m synapse.app.homeserver --generate-config \
+                                -H matrix.mydomain.com \ # Change
+                                -c cfg/homeserver.yaml \
+                                --report-stats=yes

-    source ~/.synapse/bin/activate
-    pip install -U matrix-synapse
+Installing postgres
+*******************
+
+`PostgreSQL <https://www.postgresql.org/>`_ is the recommended database backend
+supported by Synapse. If you are upgrading from SQLite, please consult the
+`documentation on how to switch
+<https://github.com/matrix-org/synapse/blob/master/docs/postgres.rst#porting-from-sqlite>`_
+for improved performance.
+
+Enable and start postgresql::
+
+    systemctl enable postgresql && systemctl start postgresql
+
+Assuming your postgres user is called ``postgres``, login and create a user.
+This will prompt for a password, make sure you set a strong passphrase::
+
+    su - postgres
+    createuser --pwprompt synapse_user
+
+Create a Synapse database::
+
+    CREATE DATABASE synapse
+     ENCODING 'UTF8'
+     LC_COLLATE='C'
+     LC_CTYPE='C'
+     template=template0
+     OWNER synapse_user;
+
+Finally, edit the ``database`` section in your ``cfg/homeserver.yaml`` file
+to point to the new database::
+
+    database:
+        name: psycopg2
+        args:
+            user: synapse_user
+            password: <password defined in the createuser step>
+            database: synapse
+            host: localhost
+            cp_min: 5
+            cp_max: 10
+
+More information can be found at `Using Postgres with Synapse
+<docs/postgres.rst>`_.
+
+Systemd
+*******
+
+Running Synapse under `systemd <https://en.wikipedia.org/wiki/Systemd>`_ is
+recommended, as it allows for simple management and automatic restarts in case
+of a server error. To integrate Synapse with systemd, create a file at
+`/etc/systemd/system/synapse.service` with the following contents::
+
+    [Unit]
+    Description="Synapse homeserver"
+
+    [Service]
+    ExecStart=/home/matrix/matrix-synapse/bin/python -m synapse.app.homeserver
+    PIDFile=/home/matrix/matrix-synapse/homeserver.pid
+    Type=forking
+    WorkingDirectory=/home/matrix/matrix-synapse/
+    Restart=always
+
+Then tell systemd to update service file information::
+
+    sudo systemctl daemon-reload
+
+Synapse should now be enabled to run under Systemd, but **don't start Synapse
+yet!**
+
+
+ACME setup
+**********
+
+Synapse requires valid TLS certificates for communication between servers
+(port ``8448`` by default) in addition to those that are client-facing (port
+``443``). Synapse **will provision server-to-server certificates
+automatically for you for free** through `Let's Encrypt
+<https://letsencrypt.org/>`_ if you tell it to.
+
+    Note: Synapse does not currently hot-renew Let's Encrypt certificates for
+    you, it only checks for certificates that need renewing on restart. This
+    functionality will be implemented promptly, but if in the meantime your
+    federation certificates expire, simply restarting Synapse should renew
+    them automatically.
+
+In order for Synapse to complete the ACME challenge to provision a
+certificate, it needs access to port 80. Typically listening on port 80 is
+only granted to applications running as root. There are thus two solutions to
+this problem.
+
+**Using a reverse proxy**
+
+A reverse proxy such as Apache or Nginx allows a single process (the web
+server) to listen on port 80 and redirect traffic to the appropriate program
+running on your server.
+
+
+
+**Authbind**
+
+``authbind`` allows a program which does not or should not run as root to
+bind to low-numbered ports in a controlled way. The setup is simpler, but
+requires a webserver not to already be running on port 80. **This includes
+every time Synapse renews a certificate**, which may be cumbersome if you
+usually run a web server on port 80. Nevertheless, if that isn't a concern,
+follow the instructions below.
+
+Install ``authbind``. This can be done on Ubuntu/Debian with::
+
+    sudo apt-get install authbind
+
+**Add authbind to the systemd script**
+
+
+**TODO: This right?** If you would like to use your own
+certificates, specifying them in Synapse's config file is sufficient.
+
+
+**TODO: Fit this in**
+These keys will allow your Home Server to identify itself to other Home
+Servers, so don't lose or delete them. It would be wise to back them up
+somewhere safe. (If, for whatever reason, you do need to change your Home
+Server's keys, you may find that other Home Servers have the old key cached.
+If you update the signing key, you should change the name of the key in the
+``<server name>.signing.key`` file (the second word) to something different.
+See `the spec`__ for more information on key management.)
+
+**TODO: Does this still work?** This Synapse installation can then be later
+upgraded by using pip again with the update flag::
+
+    source ~/synapse/env/bin/activate
+    pip install -U matrix-synapse[all]

 In case of problems, please see the _`Troubleshooting` section below.

-There is an offical synapse image available at
-https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with
-the docker-compose file available at `contrib/docker <contrib/docker>`_. Further information on
-this including configuration options is available in the README on
-hub.docker.com.
-
-Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
-Dockerfile to automate a synapse server in a single Docker image, at
-https://hub.docker.com/r/avhost/docker-matrix/tags/
-
-Slavi Pantaleev has created an Ansible playbook,
-which installs the offical Docker image of Matrix Synapse
-along with many other Matrix-related services (Postgres database, riot-web, coturn, mxisd, SSL support, etc.).
-For more details, see
-https://github.com/spantaleev/matrix-docker-ansible-deploy
+We have now created a "matrix" user with its own home directory that stores
+Synapse's data and configuration files, backed by a postgres database, all
+packaged into a isolated python virtual environment.

 Configuring Synapse
 -------------------

-Before you can start Synapse, you will need to generate a configuration
-file. To do this, run (in your virtualenv, as before)::
-
-    cd ~/.synapse
-    python -m synapse.app.homeserver \
-        --server-name my.domain.name \
-        --config-path homeserver.yaml \
-        --generate-config \
-        --report-stats=[yes|no]
-
-... substituting an appropriate value for ``--server-name``. The server name
-determines the "domain" part of user-ids for users on your server: these will
+Before starting Synapse, inspect the ``cfg/homeserver.yaml`` file. ``server_name``
+determines the "domain" part of user-ids for users on your server, which will
 all be of the format ``@user:my.domain.name``. It also determines how other
 matrix servers will reach yours for `Federation`_. For a test configuration,
 set this to the hostname of your server. For a more production-ready setup, you
@@ -202,16 +309,7 @@ will probably want to specify your domain (``example.com``) rather than a
 matrix-specific hostname here (in the same way that your email address is
 probably ``user@example.com`` rather than ``user@email.example.com``) - but
 doing so may require more advanced setup - see `Setting up
-Federation`_. Beware that the server name cannot be changed later.
-
-This command will generate you a config file that you can then customise, but it will
-also generate a set of keys for you. These keys will allow your Home Server to
-identify itself to other Home Servers, so don't lose or delete them. It would be
-wise to back them up somewhere safe. (If, for whatever reason, you do need to
-change your Home Server's keys, you may find that other Home Servers have the
-old key cached. If you update the signing key, you should change the name of the
-key in the ``<server name>.signing.key`` file (the second word) to something
-different. See `the spec`__ for more information on key management.)
+Federation`_. **Be aware that the server name cannot be changed later.**

 .. __: `key_management`_

@@ -219,10 +317,9 @@ The default configuration exposes two HTTP ports: 8008 and 8448. Port 8008 is
 configured without TLS; it should be behind a reverse proxy for TLS/SSL
 termination on port 443 which in turn should be used for clients. Port 8448
 is configured to use TLS with a self-signed certificate. If you would like
-to do initial test with a client without having to setup a reverse proxy,
-you can temporarly use another certificate. (Note that a self-signed
-certificate is fine for `Federation`_). You can do so by changing
-``tls_certificate_path``, ``tls_private_key_path`` and ``tls_dh_params_path``
+to do an initial test with a client without having to setup a reverse proxy,
+you can temporarly use another certificate. You can do so by changing
+``tls_certificate_path`` and ``tls_private_key_path``
 in ``homeserver.yaml``; alternatively, you can use a reverse-proxy, but be sure
 to read `Using a reverse proxy with Synapse`_ when doing so.

@@ -240,7 +337,7 @@ commandline script.

 To get started, it is easiest to use the command line to register new users::

-    $ source ~/.synapse/bin/activate
+    $ source ~/synapse/env/bin/activate
    $ synctl start # if not already running
    $ register_new_matrix_user -c homeserver.yaml https://localhost:8448
    New user localpart: erikj
@@ -265,13 +362,24 @@ a TURN server.  See `<docs/turn-howto.rst>`_ for details.
 Running Synapse
 ===============

-To actually run your new homeserver, pick a working directory for Synapse to
-run (e.g. ``~/.synapse``), and::
+**TODO: Needs update**

-    cd ~/.synapse
-    source ./bin/activate
+To actually run your new homeserver, pick a working directory for Synapse to
+run (e.g. ``~/synapse``), and::
+
+    cd ~/synapse
+    source env/bin/activate
    synctl start

+Upgrading an existing Synapse
+=============================
+
+The instructions for upgrading synapse are in `UPGRADE.rst`_.
+Please check these instructions as upgrading may require extra steps for some
+versions of synapse.
+
+.. _UPGRADE.rst: UPGRADE.rst
+

 Connecting to Synapse from a client
 ===================================
@@ -292,10 +400,6 @@ go back in your web client and proceed further.
 If all goes well you should at least be able to log in, create a room, and
 start sending messages.

-(The homeserver runs a web client by default at https://localhost:8448/, though
-as of the time of writing it is somewhat outdated and not really recommended -
-https://github.com/matrix-org/synapse/issues/1527).
-
 .. _`client-user-reg`:

 Registering a new user from a client
@@ -333,19 +437,24 @@ content served to web browsers a matrix API from being able to attack webapps ho
 on the same domain.  This is particularly true of sharing a matrix webclient and
 server on the same domain.

-See https://github.com/vector-im/vector-web/issues/1977 and
+See https://github.com/vector-im/riot-web/issues/1977 and
 https://developer.github.com/changes/2014-04-25-user-content-security for more details.


-Platform-Specific Instructions
-==============================
+Platform-Specific Packages
+==========================
+
+Note that the only officially supported installation method is what is listed
+in `Synapse installation`_. Instructions and packages for other platforms are
+listed below, but beware that they may be outdated.

 Debian
 ------

 Matrix provides official Debian packages via apt from https://matrix.org/packages/debian/.
+
 Note that these packages do not include a client - choose one from
-https://matrix.org/docs/projects/try-matrix-now.html (or build your own with one of our SDKs :)
+https://matrix.org/docs/projects/try-matrix-now.html (or build your own with one of our SDKs :).

 Fedora
 ------
@@ -375,40 +484,19 @@ ArchLinux

 The quickest way to get up and running with ArchLinux is probably with the community package
 https://www.archlinux.org/packages/community/any/matrix-synapse/, which should pull in most of
-the necessary dependencies. If the default web client is to be served (enabled by default in
-the generated config),
-https://www.archlinux.org/packages/community/any/python2-matrix-angular-sdk/ will also need to
-be installed.
-
-Alternatively, to install using pip a few changes may be needed as ArchLinux
-defaults to python 3, but synapse currently assumes python 2.7 by default:
+the necessary dependencies.

 pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 )::

-    sudo pip2.7 install --upgrade pip
-
-You also may need to explicitly specify python 2.7 again during the install
-request::
-
-    pip2.7 install https://github.com/matrix-org/synapse/tarball/master
+    sudo pip install --upgrade pip

 If you encounter an error with lib bcrypt causing an Wrong ELF Class:
 ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly
 compile it under the right architecture. (This should not be needed if
 installing under virtualenv)::

-    sudo pip2.7 uninstall py-bcrypt
-    sudo pip2.7 install py-bcrypt
-
-During setup of Synapse you need to call python2.7 directly again::
-
-    cd ~/.synapse
-    python2.7 -m synapse.app.homeserver \
-      --server-name machine.my.domain.name \
-      --config-path homeserver.yaml \
-      --generate-config
-
-...substituting your host and domain name as appropriate.
+    sudo pip uninstall py-bcrypt
+    sudo pip install py-bcrypt

 FreeBSD
 -------
@@ -418,7 +506,6 @@ Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Mo
 - Ports: ``cd /usr/ports/net-im/py-matrix-synapse && make install clean``
 - Packages: ``pkg install py27-matrix-synapse``

-
 OpenBSD
 -------

@@ -452,12 +539,33 @@ https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-
 Windows Install
 ---------------

-If you wish to run or develop Synapse on Windows, the Windows Subsystem For
-Linux provides a Linux environment on Windows 10 which is capable of using the
-Debian, Fedora, or source installation methods. More information about WSL can
-be found at https://docs.microsoft.com/en-us/windows/wsl/install-win10 for
-Windows 10 and https://docs.microsoft.com/en-us/windows/wsl/install-on-server
-for Windows Server.
+Running Synapse on Windows is not recommended or supported. However, if you
+wish to run Synapse on Windows, the Windows Subsystem For Linux provides a
+Linux environment on Windows 10 which is capable of using the Debian, Fedora,
+or source installation methods. More information about WSL can be found at
+https://docs.microsoft.com/en-us/windows/wsl/install-win10 for Windows 10 and
+https://docs.microsoft.com/en-us/windows/wsl/install-on-server for Windows
+Server.
+
+
+Alternative installation methods
+================================
+
+There is an offical synapse image available at
+https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with
+the docker-compose file available at `contrib/docker <contrib/docker>`_.
+Further information on this including configuration options is available in
+the README on hub.docker.com.
+
+Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
+Dockerfile to automate a synapse server in a single Docker image, at
+https://hub.docker.com/r/avhost/docker-matrix/tags/
+
+Slavi Pantaleev has created an Ansible playbook, which installs the offical
+Docker image of Matrix Synapse along with many other Matrix-related services
+(Postgres database, riot-web, coturn, mxisd, SSL support, etc.). For more
+details, see https://github.com/spantaleev/matrix-docker-ansible-deploy
+

 Troubleshooting
 ===============
@@ -475,7 +583,7 @@ You can fix this by manually upgrading pip and virtualenv::

    sudo pip install --upgrade virtualenv

-You can next rerun ``virtualenv -p python2.7 synapse`` to update the virtual env.
+You can next rerun ``virtualenv -p python3 synapse`` to update the virtual env.

 Installing may fail during installing virtualenv with ``InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.``
 You can fix this  by manually installing ndg-httpsclient::
@@ -501,7 +609,7 @@ failing, e.g.::
    pip install twisted

 Running out of File Handles
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+***************************

 If synapse runs out of filehandles, it typically fails badly - live-locking
 at 100% CPU, and/or failing to accept new TCP connections (blocking the
@@ -524,26 +632,6 @@ log lines and looking for any 'Processed request' lines which take more than
 a few seconds to execute.  Please let us know at #matrix-dev:matrix.org if
 you see this failure mode so we can help debug it, however.

-ArchLinux
-~~~~~~~~~
-
-If running `$ synctl start` fails with 'returned non-zero exit status 1',
-you will need to explicitly call Python2.7 - either running as::
-
-    python2.7 -m synapse.app.homeserver --daemonize -c homeserver.yaml
-
-...or by editing synctl with the correct python executable.
-
-
-Upgrading an existing Synapse
-=============================
-
-The instructions for upgrading synapse are in `UPGRADE.rst`_.
-Please check these instructions as upgrading may require extra steps for some
-versions of synapse.
-
-.. _UPGRADE.rst: UPGRADE.rst
-
 .. _federation:

 Setting up Federation
@@ -731,7 +819,7 @@ port:

 * Until v0.33.3, Synapse did not support SNI on the federation port
  (`bug #1491 <https://github.com/matrix-org/synapse/issues/1491>`_). This bug
-  is now fixed, but means that federating with older servers can be unreliable 
+  is now fixed, but means that federating with older servers can be unreliable
  when using name-based virtual hosting.

 Furthermore, a number of the normal reasons for using a reverse-proxy do not
@@ -763,8 +851,8 @@ caveats, you will need to do the following:
  tell other servers how to find you. See `Setting up Federation`_.

 When updating the SSL certificate, just update the file pointed to by
-``tls_certificate_path``: there is no need to restart synapse. (You may like to
-use a symbolic link to help make this process atomic.)
+``tls_certificate_path`` and then restart Synapse. (You may like to use a symbolic link
+to help make this process atomic.)

 The most common mistake when setting up federation is not to tell Synapse about
 your SSL certificate. To check it, you can visit
@@ -828,14 +916,13 @@ Password reset
 ==============

 If a user has registered an email address to their account using an identity
-server, they can request a password-reset token via clients such as Vector.
+server, they can request a password-reset token via clients such as Riot.

 A manual password reset can be done via direct database access as follows.

 First calculate the hash of the new password::

-    $ source ~/.synapse/bin/activate
-    $ ./scripts/hash_password
+    $ ~/synapse/env/bin/hash_password
    Password:
    Confirm password:
    $2a$12$xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
@@ -864,8 +951,7 @@ to install using pip and a virtualenv::

    virtualenv -p python2.7 env
    source env/bin/activate
-    python -m synapse.python_dependencies | xargs pip install
-    pip install lxml mock
+    python -m pip install -e .[all]

 This will run a process of downloading and installing all the needed
 dependencies into a virtual env.
@@ -873,7 +959,7 @@ dependencies into a virtual env.
 Once this is done, you may wish to run Synapse's unit tests, to
 check that everything is installed as it should be::

-    PYTHONPATH="." trial tests
+    python -m twisted.trial tests

 This should end with a 'PASSED' result::

--- a/UPGRADE.rst
+++ b/UPGRADE.rst
@@ -18,7 +18,7 @@ instructions that may be required are listed later in this document.

   .. code:: bash

-       pip install --upgrade --process-dependency-links matrix-synapse
+       pip install --upgrade matrix-synapse

       # restart synapse
       synctl restart
@@ -48,6 +48,118 @@ returned by the Client-Server API:
    # configured on port 443.
    curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"

+Upgrading to v0.99.0
+====================
+
+In preparation for Synapse v1.0, you must update your TLS certificates from
+self-signed ones to verifiable ones signed by a trusted root CA.
+
+If you do not already have a certificate for your domain, the easiest way to get
+one is with Synapse's new ACME support, which will use the ACME protocol to
+provision a certificate automatically. By default, certificates will be obtained
+from the publicly trusted CA Let's Encrypt.
+
+For a sample configuration, please inspect the new ACME section in the example
+generated config by running the ``generate-config`` executable. For example::
+
+  ~/synapse/env3/bin/generate-config
+
+You will need to provide Let's Encrypt (or other ACME provider) access to your
+Synapse ACME challenge responder on port 80, at the domain of your homeserver.
+This requires you either change the port of the ACME listener provided by
+Synapse to a high port and reverse proxy to it, or use a tool like authbind to
+allow Synapse to listen on port 80 without root access. (Do not run Synapse with
+root permissions!)
+
+You will need to back up or delete your self signed TLS certificate
+(``example.com.tls.crt`` and ``example.com.tls.key``), Synapse's ACME
+implementation will not overwrite them.
+
+You may wish to use alternate methods such as Certbot to obtain a certificate
+from Let's Encrypt, depending on your server configuration. Of course, if you
+already have a valid certificate for your homeserver's domain, that can be
+placed in Synapse's config directory without the need for ACME.
+
+Upgrading to v0.34.0
+====================
+
+1. This release is the first to fully support Python 3. Synapse will now run on
+   Python versions 3.5, or 3.6 (as well as 2.7). We recommend switching to
+   Python 3, as it has been shown to give performance improvements.
+
+   For users who have installed Synapse into a virtualenv, we recommend doing
+   this by creating a new virtualenv. For example::
+
+       virtualenv -p python3 ~/synapse/env3
+       source ~/synapse/env3/bin/activate
+       pip install matrix-synapse
+
+   You can then start synapse as normal, having activated the new virtualenv::
+
+       cd ~/synapse
+       source env3/bin/activate
+       synctl start
+
+   Users who have installed from distribution packages should see the relevant
+   package documentation. See below for notes on Debian packages.
+
+   * When upgrading to Python 3, you **must** make sure that your log files are
+     configured as UTF-8, by adding ``encoding: utf8`` to the
+     ``RotatingFileHandler`` configuration (if you have one) in your
+     ``<server>.log.config`` file. For example, if your ``log.config`` file
+     contains::
+
+       handlers:
+         file:
+           class: logging.handlers.RotatingFileHandler
+           formatter: precise
+           filename: homeserver.log
+           maxBytes: 104857600
+           backupCount: 10
+           filters: [context]
+         console:
+           class: logging.StreamHandler
+           formatter: precise
+           filters: [context]
+
+     Then you should update this to be::
+
+       handlers:
+         file:
+           class: logging.handlers.RotatingFileHandler
+           formatter: precise
+           filename: homeserver.log
+           maxBytes: 104857600
+           backupCount: 10
+           filters: [context]
+           encoding: utf8
+         console:
+           class: logging.StreamHandler
+           formatter: precise
+           filters: [context]
+
+     There is no need to revert this change if downgrading to Python 2.
+
+   We are also making available Debian packages which will run Synapse on
+   Python 3. You can switch to these packages with ``apt-get install
+   matrix-synapse-py3``, however, please read `debian/NEWS
+   <https://github.com/matrix-org/synapse/blob/release-v0.34.0/debian/NEWS>`_
+   before doing so. The existing ``matrix-synapse`` packages will continue to
+   use Python 2 for the time being.
+
+2. This release removes the ``riot.im`` from the default list of trusted
+   identity servers.
+
+   If ``riot.im`` is in your homeserver's list of
+   ``trusted_third_party_id_servers``, you should remove it. It was added in
+   case a hypothetical future identity server was put there. If you don't
+   remove it, users may be unable to deactivate their accounts.
+
+3. This release no longer installs the (unmaintained) Matrix Console web client
+   as part of the default installation. It is possible to re-enable it by
+   installing it separately and setting the ``web_client_location`` config
+   option, but please consider switching to another client.
+
 Upgrading to v0.33.7
 ====================

--- a/changelog.d/3830.feature
+++ b/changelog.d/3830.feature
@@ -1 +0,0 @@
-Add option to track MAU stats (but not limit people)
--- a/changelog.d/3902.feature
+++ b/changelog.d/3902.feature
@@ -0,0 +1 @@
+Include m.room.encryption on invites by default
--- a/changelog.d/4176.bugfix
+++ b/changelog.d/4176.bugfix
@@ -1 +0,0 @@
-The media repository now no longer fails to decode UTF-8 filenames when downloading remote media.
--- a/changelog.d/4180.misc
+++ b/changelog.d/4180.misc
@@ -1 +0,0 @@
-A coveragerc file, as well as the py36-coverage tox target, have been added.
--- a/changelog.d/4182.misc
+++ b/changelog.d/4182.misc
@@ -1 +0,0 @@
-Add a GitHub pull request template and add multiple issue templates
--- a/changelog.d/4183.bugfix
+++ b/changelog.d/4183.bugfix
@@ -1 +0,0 @@
-URL previews now correctly decode non-UTF-8 text if the header contains a `<meta http-equiv="Content-Type"` header.
--- a/changelog.d/4188.misc
+++ b/changelog.d/4188.misc
@@ -1 +0,0 @@
-Update README to reflect the fact that #1491 is fixed
--- a/changelog.d/4192.bugfix
+++ b/changelog.d/4192.bugfix
@@ -1 +0,0 @@
-Fix an issue where public consent URLs had two slashes.
--- a/changelog.d/4193.misc
+++ b/changelog.d/4193.misc
@@ -1 +0,0 @@
-Add missing `jpeg` package prerequisite for OpenBSD in README.
--- a/changelog.d/4197.bugfix
+++ b/changelog.d/4197.bugfix
@@ -1 +0,0 @@
-Fallback auth now accepts the session parameter on Python 3.
--- a/changelog.d/4200.misc
+++ b/changelog.d/4200.misc
@@ -1 +0,0 @@
-Add a note saying you need to manually reclaim disk space after using the Purge History API
--- a/changelog.d/4204.misc
+++ b/changelog.d/4204.misc
@@ -1 +0,0 @@
-Fix logcontext leaks in EmailPusher and in tests
--- a/changelog.d/4207.bugfix
+++ b/changelog.d/4207.bugfix
@@ -1 +0,0 @@
-Remove riot.im from the list of trusted Identity Servers in the default configuration
--- a/contrib/docker/docker-compose.yml
+++ b/contrib/docker/docker-compose.yml
@@ -37,7 +37,7 @@ services:
    labels:
      - traefik.enable=true
      - traefik.frontend.rule=Host:my.matrix.Host
-      - traefik.port=8448
+      - traefik.port=8008

  db:
    image: docker.io/postgres:10-alpine
--- a/contrib/systemd/matrix-synapse.service
+++ b/contrib/systemd/matrix-synapse.service
@@ -0,0 +1,31 @@
+# Example systemd configuration file for synapse. Copy into
+#    /etc/systemd/system/, update the paths if necessary, then:
+#
+#    systemctl enable matrix-synapse
+#    systemctl start matrix-synapse
+#
+# This assumes that Synapse has been installed in a virtualenv in
+# /opt/synapse/env.
+#
+# **NOTE:** This is an example service file that may change in the future. If you
+# wish to use this please copy rather than symlink it.
+
+[Unit]
+Description=Synapse Matrix homeserver
+
+[Service]
+Type=simple
+Restart=on-abort
+
+User=synapse
+Group=nogroup
+
+WorkingDirectory=/opt/synapse
+ExecStart=/opt/synapse/env/bin/python -m synapse.app.homeserver --config-path=/opt/synapse/homeserver.yaml
+
+# adjust the cache factor if necessary
+# Environment=SYNAPSE_CACHE_FACTOR=2.0
+
+[Install]
+WantedBy=multi-user.target
+
--- a/contrib/systemd/synapse.service
+++ b/contrib/systemd/synapse.service
@@ -1,22 +0,0 @@
-# This assumes that Synapse has been installed as a system package
-# (e.g. https://www.archlinux.org/packages/community/any/matrix-synapse/ for ArchLinux)
-# rather than in a user home directory or similar under virtualenv.
-
-# **NOTE:** This is an example service file that may change in the future. If you
-# wish to use this please copy rather than symlink it.
-
-[Unit]
-Description=Synapse Matrix homeserver
-
-[Service]
-Type=simple
-User=synapse
-Group=synapse
-WorkingDirectory=/var/lib/synapse
-ExecStart=/usr/bin/python2.7 -m synapse.app.homeserver --config-path=/etc/synapse/homeserver.yaml
-ExecStop=/usr/bin/synctl stop /etc/synapse/homeserver.yaml
-# EnvironmentFile=-/etc/sysconfig/synapse  # Can be used to e.g. set SYNAPSE_CACHE_FACTOR 
-
-[Install]
-WantedBy=multi-user.target
-
--- a/debian/.gitignore
+++ b/debian/.gitignore
@@ -0,0 +1,7 @@
+/matrix-synapse-py3.*.debhelper
+/matrix-synapse-py3.debhelper.log
+/matrix-synapse-py3.substvars
+/matrix-synapse-*/
+/files
+/debhelper-build-stamp
+/.debhelper
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -0,0 +1,32 @@
+matrix-synapse-py3 (0.34.0) stable; urgency=medium
+
+  matrix-synapse-py3 is intended as a drop-in replacement for the existing
+  matrix-synapse package. When the package is installed, matrix-synapse will be
+  automatically uninstalled. The replacement should be relatively seamless,
+  however, please note the following important differences to matrix-synapse:
+
+  * Most importantly, the matrix-synapse service now runs under Python 3 rather
+    than Python 2.7.
+
+  * Synapse is installed into its own virtualenv (in /opt/venvs/matrix-synapse)
+    instead of using the system python libraries. (This may mean that you can
+    remove a number of old dependencies with `apt autoremove`).
+
+  * If you have previously manually installed any custom python extensions
+    (such as matrix-synapse-rest-auth) into the system python directories, you
+    will need to reinstall them in the new virtualenv. Please consult the
+    documentation of the relevant extensions for further details.
+
+  matrix-synapse-py3 will take over responsibility for the existing
+  configuration files, including the matrix-synapse systemd service.
+
+  Beware, however, that `apt purge matrix-synapse` will *disable* the
+  matrix-synapse service (so that it will not be started on reboot), even
+  though that service is no longer being provided by the matrix-synapse
+  package. It can be re-enabled with `systemctl enable matrix-synapse`.
+
+  The matrix.org team will continue to provide Python 2 `matrix-synapse`
+  packages for the next couple of releases, to allow time for system
+  administrators to test the new packages.
+
+ -- Richard van der Hoff <richard@matrix.org>  Wed, 19 Dec 2018 14:00:00 +0000
--- a/debian/build_virtualenv
+++ b/debian/build_virtualenv
@@ -0,0 +1,91 @@
+#!/bin/bash
+#
+# runs dh_virtualenv to build the virtualenv in the build directory,
+# and then runs the trial tests against the installed synapse.
+
+set -e
+
+export DH_VIRTUALENV_INSTALL_ROOT=/opt/venvs
+
+# make sure that the virtualenv links to the specific version of python, by
+# dereferencing the python3 symlink.
+#
+# Otherwise, if somebody tries to install (say) the stretch package on buster,
+# they will get a confusing error about "No module named 'synapse'", because
+# python won't look in the right directory. At least this way, the error will
+# be a *bit* more obvious.
+#
+SNAKE=`readlink -e /usr/bin/python3`
+
+# try to set the CFLAGS so any compiled C extensions are compiled with the most
+# generic as possible x64 instructions, so that compiling it on a new Intel chip
+# doesn't enable features not available on older ones or AMD.
+#
+# TODO: add similar things for non-amd64, or figure out a more generic way to
+# do this.
+
+case `dpkg-architecture -q DEB_HOST_ARCH` in
+    amd64)
+        export CFLAGS=-march=x86-64
+        ;;
+esac
+
+# Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather
+# than the 2/3 compatible `virtualenv`.
+
+dh_virtualenv \
+    --install-suffix "matrix-synapse" \
+    --builtin-venv \
+    --setuptools \
+    --python "$SNAKE" \
+    --upgrade-pip \
+    --preinstall="lxml" \
+    --preinstall="mock" \
+    --extra-pip-arg="--no-cache-dir" \
+    --extra-pip-arg="--compile" \
+    --extras="all"
+
+PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
+VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"
+TARGET_PYTHON="${VIRTUALENV_DIR}/bin/python"
+
+# we copy the tests to a temporary directory so that we can put them on the
+# PYTHONPATH without putting the uninstalled synapse on the pythonpath.
+tmpdir=`mktemp -d`
+trap "rm -r $tmpdir" EXIT
+
+cp -r tests "$tmpdir"
+
+PYTHONPATH="$tmpdir" \
+    "${TARGET_PYTHON}" -B -m twisted.trial --reporter=text -j2 tests
+
+# build the config file
+"${TARGET_PYTHON}" -B "${VIRTUALENV_DIR}/bin/generate_config" \
+        --config-dir="/etc/matrix-synapse" \
+        --data-dir="/var/lib/matrix-synapse" |
+    perl -pe '
+# tweak the paths to the tls certs and signing keys
+/^tls_.*_path:/ and s/SERVERNAME/homeserver/;
+/^signing_key_path:/ and s/SERVERNAME/homeserver/;
+
+# tweak the pid file location
+/^pid_file:/ and s#:.*#: "/var/run/matrix-synapse.pid"#;
+
+# tweak the path to the log config
+/^log_config:/ and s/SERVERNAME\.log\.config/log.yaml/;
+
+# tweak the path to the media store
+/^media_store_path:/ and s#/media_store#/media#;
+
+# remove the server_name setting, which is set in a separate file
+/^server_name:/ and $_ = "#\n# This is set in /etc/matrix-synapse/conf.d/server_name.yaml for Debian installations.\n# $_";
+
+# remove the report_stats setting, which is set in a separate file
+/^# report_stats:/ and $_ = "";
+
+' > "${PACKAGE_BUILD_DIR}/etc/matrix-synapse/homeserver.yaml"
+
+
+# add a dependency on the right version of python to substvars.
+PYPKG=`basename $SNAKE`
+echo "synapse:pydepends=$PYPKG" >> debian/matrix-synapse-py3.substvars
--- a/debian/changelog
+++ b/debian/changelog
@@ -0,0 +1,669 @@
+matrix-synapse-py3 (0.34.1.1++1) stable; urgency=medium
+
+  * Update conflicts specifications to allow smoother transition from matrix-synapse.
+
+ -- Synapse Packaging team <packages@matrix.org>  Sat, 12 Jan 2019 12:58:35 +0000
+
+matrix-synapse-py3 (0.34.1.1) stable; urgency=high
+
+  * New synapse release 0.34.1.1
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 10 Jan 2019 15:04:52 +0000
+
+matrix-synapse-py3 (0.34.1+1) stable; urgency=medium
+
+  * Remove 'Breaks: matrix-synapse-ldap3'. (matrix-synapse-py3 includes
+    the matrix-synapse-ldap3 python files, which makes the
+    matrix-synapse-ldap3 debian package redundant but not broken.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 09 Jan 2019 15:30:00 +0000
+
+matrix-synapse-py3 (0.34.1) stable; urgency=medium
+
+  * New synapse release 0.34.1.
+  * Update Conflicts specifications to allow installation alongside our
+    matrix-synapse transitional package.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 09 Jan 2019 14:52:24 +0000
+
+matrix-synapse-py3 (0.34.0) stable; urgency=medium
+
+  * New synapse release 0.34.0.
+  * Synapse is now installed into a Python 3 virtual environment with
+    up-to-date dependencies.
+  * The matrix-synapse service will now be restarted when the package is
+    upgraded.
+    (Fixes https://github.com/matrix-org/package-synapse-debian/issues/18)
+
+ -- Synapse packaging team <packages@matrix.org>  Wed, 19 Dec 2018 14:00:00 +0000
+
+matrix-synapse (0.33.9-1matrix1) stretch; urgency=medium
+
+  [ Erik Johnston ]
+  * Remove dependency on python-pydenticon
+
+  [ Richard van der Hoff ]
+  * New upstream version 0.33.9
+  * Refresh patches for 0.33.9
+
+ -- Richard van der Hoff <richard@matrix.org>  Tue, 20 Nov 2018 10:26:05 +0000
+
+matrix-synapse (0.33.8-1) stretch; urgency=medium
+
+  * New upstream version 0.33.8
+
+ -- Erik Johnston <erik@matrix.org>  Thu, 01 Nov 2018 14:33:26 +0000
+
+matrix-synapse (0.33.7-1matrix1) stretch; urgency=medium
+
+  * New upstream version 0.33.7
+
+ -- Richard van der Hoff <richard@matrix.org>  Thu, 18 Oct 2018 16:18:26 +0100
+
+matrix-synapse (0.33.6-1matrix1) stretch; urgency=medium
+
+  * Imported Upstream version 0.33.6
+  * Remove redundant explicit dep on python-bcrypt
+  * Run the tests during build
+  * Add dependency on python-attr 16.0
+  * Refresh patches for 0.33.6
+
+ -- Richard van der Hoff <richard@matrix.org>  Thu, 04 Oct 2018 14:40:29 +0100
+
+matrix-synapse (0.33.5.1-1matrix1) stretch; urgency=medium
+
+  * Imported Upstream version 0.33.5.1
+
+ -- Richard van der Hoff <richard@matrix.org>  Mon, 24 Sep 2018 18:20:51 +0100
+
+matrix-synapse (0.33.5-1matrix1) stretch; urgency=medium
+
+  * Imported Upstream version 0.33.5
+
+ -- Richard van der Hoff <richard@matrix.org>  Mon, 24 Sep 2018 16:06:23 +0100
+
+matrix-synapse (0.33.4-1mx1) stretch; urgency=medium
+
+  * Imported Upstream version 0.33.4
+  * Avoid telling people to install packages with pip
+    (fixes https://github.com/matrix-org/synapse/issues/3743)
+
+ -- Richard van der Hoff <richard@matrix.org>  Fri, 07 Sep 2018 14:06:17 +0100
+
+matrix-synapse (0.33.3.1-1mx1) stretch; urgency=critical
+
+  [ Richard van der Hoff ]
+  * Imported Upstream version 0.33.3.1
+
+ -- Richard van der Hoff <richard@matrix.org>  Thu, 06 Sep 2018 11:20:37 +0100
+
+matrix-synapse (0.33.3-2) stretch; urgency=medium
+
+  * We now require python-twisted 17.1.0 or later
+  * Add recommendations for python-psycopg2 and python-lxml
+
+ -- Richard van der Hoff <richard@matrix.org>  Thu, 23 Aug 2018 19:04:08 +0100
+
+matrix-synapse (0.33.3-1) jessie; urgency=medium
+
+  * New upstream version 0.33.3
+
+ -- Richard van der Hoff <richard@matrix.org>  Wed, 22 Aug 2018 14:50:30 +0100
+
+matrix-synapse (0.33.2-1) jessie; urgency=medium
+
+  * New upstream version 0.33.2
+
+ -- Richard van der Hoff <richard@matrix.org>  Thu, 09 Aug 2018 15:40:42 +0100
+
+matrix-synapse (0.33.1-1) jessie; urgency=medium
+
+  * New upstream version 0.33.1
+
+ -- Erik Johnston <erik@matrix.org>  Thu, 02 Aug 2018 15:52:19 +0100
+
+matrix-synapse (0.33.0-1) jessie; urgency=medium
+
+  * New upstream version 0.33.0
+
+ -- Richard van der Hoff <richard@matrix.org>  Thu, 19 Jul 2018 13:38:41 +0100
+
+matrix-synapse (0.32.1-1) jessie; urgency=medium
+
+  * New upstream version 0.32.1
+
+ -- Richard van der Hoff <richard@matrix.org>  Fri, 06 Jul 2018 17:16:29 +0100
+
+matrix-synapse (0.32.0-1) jessie; urgency=medium
+
+  * New upstream version 0.32.0
+
+ -- Erik Johnston <erik@matrix.org>  Fri, 06 Jul 2018 15:34:06 +0100
+
+matrix-synapse (0.31.2-1) jessie; urgency=high
+
+  * New upstream version 0.31.2
+
+ -- Richard van der Hoff <richard@matrix.org>  Thu, 14 Jun 2018 16:49:07 +0100
+
+matrix-synapse (0.31.1-1) jessie; urgency=medium
+
+  * New upstream version 0.31.1
+  * Require python-prometheus-client >= 0.0.14
+
+ -- Richard van der Hoff <richard@matrix.org>  Fri, 08 Jun 2018 16:11:55 +0100
+
+matrix-synapse (0.31.0-1) jessie; urgency=medium
+
+  * New upstream version 0.31.0
+
+ -- Richard van der Hoff <richard@matrix.org>  Wed, 06 Jun 2018 17:23:10 +0100
+
+matrix-synapse (0.30.0-1) jessie; urgency=medium
+
+  [ Michael Kaye ]
+  * update homeserver.yaml to be somewhat more modern.
+
+  [ Erik Johnston ]
+  * New upstream version 0.30.0
+
+ -- Erik Johnston <erik@matrix.org>  Thu, 24 May 2018 16:43:16 +0100
+
+matrix-synapse (0.29.0-1) jessie; urgency=medium
+
+  * New upstream version 0.29.0
+
+ -- Erik Johnston <erik@matrix.org>  Wed, 16 May 2018 17:43:06 +0100
+
+matrix-synapse (0.28.1-1) jessie; urgency=medium
+
+  * New upstream version 0.28.1
+
+ -- Erik Johnston <erik@matrix.org>  Tue, 01 May 2018 19:21:39 +0100
+
+matrix-synapse (0.28.0-1) jessie; urgency=medium
+
+  * New upstream 0.28.0
+
+ -- Erik Johnston <erik@matrix.org>  Fri, 27 Apr 2018 13:15:49 +0100
+
+matrix-synapse (0.27.4-1) jessie; urgency=medium
+
+  * Bump canonicaljson version
+  * New upstream 0.27.4
+
+ -- Erik Johnston <erik@matrix.org>  Fri, 13 Apr 2018 13:37:47 +0100
+
+matrix-synapse (0.27.3-1) jessie; urgency=medium
+
+  * Report stats should default to off
+  * Refresh patches
+  * New upstream 0.27.3
+
+ -- Erik Johnston <erik@matrix.org>  Wed, 11 Apr 2018 11:43:47 +0100
+
+matrix-synapse (0.27.2-1) jessie; urgency=medium
+
+  * New upstream version 0.27.2
+
+ -- Erik Johnston <erik@matrix.org>  Mon, 26 Mar 2018 16:41:57 +0100
+
+matrix-synapse (0.27.1-1) jessie; urgency=medium
+
+  * New upstream version 0.27.1
+
+ -- Erik Johnston <erik@matrix.org>  Mon, 26 Mar 2018 16:22:03 +0100
+
+matrix-synapse (0.27.0-2) jessie; urgency=medium
+
+  * Fix bcrypt dependency
+
+ -- Erik Johnston <erik@matrix.org>  Mon, 26 Mar 2018 16:00:26 +0100
+
+matrix-synapse (0.27.0-1) jessie; urgency=medium
+
+  * New upstream version 0.27.0
+
+ -- Erik Johnston <erik@matrix.org>  Mon, 26 Mar 2018 15:07:52 +0100
+
+matrix-synapse (0.26.1-1) jessie; urgency=medium
+
+  * Ignore RC
+  * New upstream version 0.26.1
+
+ -- Erik Johnston <erik@matrix.org>  Fri, 16 Mar 2018 00:40:08 +0000
+
+matrix-synapse (0.26.0-1) jessie; urgency=medium
+
+  [ Richard van der Hoff ]
+  * Remove `level` for `file` log handler
+
+  [ Erik Johnston ]
+
+ -- Erik Johnston <erik@matrix.org>  Fri, 05 Jan 2018 11:21:26 +0000
+
+matrix-synapse (0.25.1-1) jessie; urgency=medium
+
+  * New upstream version 0.25.1
+
+ -- Erik Johnston <erik@matrix.org>  Mon, 20 Nov 2017 10:05:37 +0000
+
+matrix-synapse (0.25.0-1) jessie; urgency=medium
+
+  * New upstream version 0.25.0
+
+ -- Erik Johnston <erik@matrix.org>  Wed, 15 Nov 2017 11:36:32 +0000
+
+matrix-synapse (0.24.1-1) jessie; urgency=medium
+
+  * New upstream version 0.24.1
+
+ -- Erik Johnston <erik@matrix.org>  Tue, 24 Oct 2017 15:05:03 +0100
+
+matrix-synapse (0.24.0-1) jessie; urgency=medium
+
+  * New upstream version 0.24.0
+
+ -- Erik Johnston <erik@matrix.org>  Mon, 23 Oct 2017 14:11:46 +0100
+
+matrix-synapse (0.23.1-1) xenial; urgency=medium
+
+  * Imported upstream version 0.23.1
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 05 Oct 2017 15:28:25 +0100
+
+matrix-synapse (0.23.0-1) jessie; urgency=medium
+
+  * Fix patch after refactor
+  * Add patch to remove requirement on affinity package
+  * refresh webclient patch
+
+ -- Erik Johnston <erikj@matrix.org>  Mon, 02 Oct 2017 15:34:57 +0100
+
+matrix-synapse (0.22.1-1) jessie; urgency=medium
+
+  * Imported Upstream version 0.22.1
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 06 Jul 2017 18:14:13 +0100
+
+matrix-synapse (0.22.0-1) jessie; urgency=medium
+
+  * Imported upstream version 0.22.0
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 06 Jul 2017 10:47:45 +0100
+
+matrix-synapse (0.21.1-1) jessie; urgency=medium
+
+  * Imported upstream version 0.21.1
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 15 Jun 2017 13:31:13 +0100
+
+matrix-synapse (0.21.0-1) jessie; urgency=medium
+
+  * Imported upstream version 0.21.0
+  * Update patches
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 18 May 2017 14:16:54 +0100
+
+matrix-synapse (0.20.0-2) jessie; urgency=medium
+
+  * Depend on python-jsonschema
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 12 Apr 2017 10:41:46 +0100
+
+matrix-synapse (0.20.0-1) jessie; urgency=medium
+
+  * Imported upstream version 0.20.0
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 11 Apr 2017 12:58:26 +0100
+
+matrix-synapse (0.19.3-1) jessie; urgency=medium
+
+  * Imported upstream version 0.19.3
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 21 Mar 2017 13:45:41 +0000
+
+matrix-synapse (0.19.2-1) jessie; urgency=medium
+
+  [ Sunil Mohan Adapa ]
+  * Bump standards version to 3.9.8
+  * Add debian/copyright file
+  * Don't ignore errors in debian/config
+  * Reformat depenedencies in debian/control
+  * Internationalize strings in template file
+  * Update package description
+  * Add lsb-base as dependency
+  * Update questions for debconf style
+  * Add man pages for all binaries
+
+  [ Erik Johnston ]
+  * Imported upstream version 0.19.2
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 21 Feb 2017 13:55:00 +0000
+
+matrix-synapse (0.19.1-1) jessie; urgency=medium
+
+  * Imported upstream version 0.19.1
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 09 Feb 2017 11:53:27 +0000
+
+matrix-synapse (0.19.0-1) jessie; urgency=medium
+
+  This build requires python-twisted 0.19.0, which may need to be installed
+  from backports.
+
+  [ Bryce Chidester ]
+  * Add EnvironmentFile to the systemd service
+  * Create matrix-synapse.default
+
+  [ Erik Johnston ]
+  * Imported upstream version 0.19.0
+
+ -- Erik Johnston <erikj@matrix.org>  Sat, 04 Feb 2017 09:58:29 +0000
+
+matrix-synapse (0.18.7-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.18.4
+
+ -- Erik Johnston <erikj@matrix.org>  Mon, 09 Jan 2017 15:10:21 +0000
+
+matrix-synapse (0.18.5-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.18.5
+
+ -- Erik Johnston <erikj@matrix.org>  Fri, 16 Dec 2016 10:51:59 +0000
+
+matrix-synapse (0.18.4-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.18.4
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 22 Nov 2016 10:33:41 +0000
+
+matrix-synapse (0.18.3-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.18.3
+  * Remove upstreamed ldap3 patch
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 08 Nov 2016 15:01:49 +0000
+
+matrix-synapse (0.18.2-2) trusty; urgency=high
+
+  * Patch ldap3 support to workaround differences in python-ldap3 0.9,
+    bug allowed unauthorized logins if ldap3 0.9 was used.
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 08 Nov 2016 13:48:09 +0000
+
+matrix-synapse (0.18.2-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.18.2
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 01 Nov 2016 13:30:45 +0000
+
+matrix-synapse (0.18.1-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.18.1
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 05 Oct 2016 14:52:53 +0100
+
+matrix-synapse (0.18.0-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.18.0
+
+ -- Erik Johnston <erikj@matrix.org>  Mon, 19 Sep 2016 17:38:48 +0100
+
+matrix-synapse (0.17.3-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.17.3
+
+ -- Erik Johnston <erikj@matrix.org>  Fri, 09 Sep 2016 11:18:18 +0100
+
+matrix-synapse (0.17.2-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.17.2
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 08 Sep 2016 15:37:14 +0100
+
+matrix-synapse (0.17.1-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.17.1
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 24 Aug 2016 15:11:29 +0100
+
+matrix-synapse (0.17.0-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.17.0
+
+ -- Erik Johnston <erikj@matrix.org>  Mon, 08 Aug 2016 13:56:15 +0100
+
+matrix-synapse (0.16.1-r1-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.16.1-r1
+
+ -- Erik Johnston <erikj@matrix.org>  Fri, 08 Jul 2016 16:47:35 +0100
+
+matrix-synapse (0.16.1-2) trusty; urgency=critical
+
+  * Apply security patch
+
+ -- Erik Johnston <erikj@matrix.org>  Fri, 08 Jul 2016 11:05:27 +0100
+
+matrix-synapse (0.16.1-1) trusty; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 21 Jun 2016 14:56:48 +0100
+
+matrix-synapse (0.16.0-3) trusty; urgency=medium
+
+  * Don't require strict nacl==0.3.0 requirement
+
+ -- Erik Johnston <erikj@matrix.org>  Mon, 20 Jun 2016 13:24:22 +0100
+
+matrix-synapse (0.16.0-2) trusty; urgency=medium
+
+  * Also change the permissions of /etc/matrix-synapse
+  * Add apt webclient instructions
+  * Fix up patches
+  * Update default homeserver.yaml
+  * Add patch
+
+ -- Erik Johnston <erikj@matrix.org>  Fri, 10 Jun 2016 14:06:20 +0100
+
+matrix-synapse (0.16.0-1) trusty; urgency=medium
+
+  [ David A Roberts ]
+  * systemd
+
+  [ Erik Johnston ]
+  * Fixup postinst and matrix-synapse.service
+  * Handle email optional deps
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 09 Jun 2016 16:17:01 +0100
+
+matrix-synapse (0.14.0-1) trusty; urgency=medium
+
+  * Remove saml2 module requirements
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 30 Mar 2016 14:31:17 +0100
+
+matrix-synapse (0.13.3-1) trusty; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 11 Feb 2016 16:35:39 +0000
+
+matrix-synapse (0.13.2-1) trusty; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 11 Feb 2016 11:01:16 +0000
+
+matrix-synapse (0.13.0-1) trusty; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 10 Feb 2016 16:34:39 +0000
+
+matrix-synapse (0.12.0-2) trusty; urgency=medium
+
+  * Don't default `registerion_shared_secret` config option
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 06 Jan 2016 16:34:02 +0000
+
+matrix-synapse (0.12.0-1) stable; urgency=medium
+
+  * Imported Upstream version 0.12.0
+
+ -- Mark Haines <mark@matrix.org>  Mon, 04 Jan 2016 15:38:33 +0000
+
+matrix-synapse (0.11.1-1) unstable; urgency=medium
+
+  * Imported Upstream version 0.11.1
+
+ -- Erik Johnston <erikj@matrix.org>  Fri, 20 Nov 2015 17:56:52 +0000
+
+matrix-synapse (0.11.0-r2-1) stable; urgency=medium
+
+  * Imported Upstream version 0.11.0-r2
+  * Add gbp.conf
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 19 Nov 2015 13:52:36 +0000
+
+matrix-synapse (0.11.0-1) wheezy; urgency=medium
+
+  * Fix dependencies.
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 17 Nov 2015 16:28:06 +0000
+
+matrix-synapse (0.11.0-0) wheezy; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 17 Nov 2015 16:03:01 +0000
+
+matrix-synapse (0.10.0-2) wheezy; urgency=medium
+
+  * Rebuild for wheezy.
+
+ -- Erik Johnston <erikj@matrix.org>  Fri, 04 Sep 2015 14:21:03 +0100
+
+matrix-synapse (0.10.0-1) trusty; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 03 Sep 2015 10:08:34 +0100
+
+matrix-synapse (0.10.0~rc6-3) trusty; urgency=medium
+
+  * Create log directory.
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 02 Sep 2015 17:49:07 +0100
+
+matrix-synapse (0.10.0~rc6-2) trusty; urgency=medium
+
+  * Add patch to work around upstream bug in config directory handling.
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 02 Sep 2015 17:42:42 +0100
+
+matrix-synapse (0.10.0~rc6-1) trusty; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 02 Sep 2015 17:21:21 +0100
+
+matrix-synapse (0.10.0~rc5-3) trusty; urgency=medium
+
+  * Update init script to work.
+
+ -- Erik Johnston <erikj@matrix.org>  Fri, 28 Aug 2015 10:51:56 +0100
+
+matrix-synapse (0.10.0~rc5-2) trusty; urgency=medium
+
+  * Fix where python files are installed.
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 27 Aug 2015 11:55:39 +0100
+
+matrix-synapse (0.10.0~rc5-1) trusty; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 27 Aug 2015 11:26:54 +0100
+
+matrix-synapse (0.10.0~rc4-1) trusty; urgency=medium
+
+  * New upstream version.
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 27 Aug 2015 10:29:31 +0100
+
+matrix-synapse (0.10.0~rc3-7) trusty; urgency=medium
+
+  * Add debian/watch
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 26 Aug 2015 17:57:08 +0100
+
+matrix-synapse (0.10.0~rc3-6) trusty; urgency=medium
+
+  * Deps.
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 26 Aug 2015 17:07:13 +0100
+
+matrix-synapse (0.10.0~rc3-5) trusty; urgency=medium
+
+  * Deps.
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 26 Aug 2015 16:18:02 +0100
+
+matrix-synapse (0.10.0~rc3-4) trusty; urgency=medium
+
+  * More deps.
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 26 Aug 2015 14:09:27 +0100
+
+matrix-synapse (0.10.0~rc3-3) trusty; urgency=medium
+
+  * Update deps.
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 26 Aug 2015 13:49:20 +0100
+
+matrix-synapse (0.10.0~rc3-2) trusty; urgency=medium
+
+  * Add more deps.
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 26 Aug 2015 13:25:45 +0100
+
+matrix-synapse (0.10.0~rc3-1) trusty; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 25 Aug 2015 17:52:33 +0100
+
+matrix-synapse (0.9.3-1~trusty1) trusty; urgency=medium
+
+  * Rebuild for trusty.
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 20 Aug 2015 15:05:43 +0100
+
+matrix-synapse (0.9.3-1) wheezy; urgency=medium
+
+  * New upstream release
+  * Create a user, "matrix-synapse", to run as
+  * Log to /var/log/matrix-synapse/ directory
+  * Override the way synapse looks for the angular SDK (syweb) so it finds the
+    packaged one
+
+ -- Paul "LeoNerd" Evans <paul@matrix.org>  Fri, 07 Aug 2015 15:32:12 +0100
+
+matrix-synapse (0.9.2-2) wheezy; urgency=medium
+
+  * Supply a default config file
+  * Create directory in /var/lib
+  * Use debconf to ask the user for the server name at installation time
+
+ -- Paul "LeoNerd" Evans <paul@matrix.org>  Thu, 06 Aug 2015 15:28:00 +0100
+
+matrix-synapse (0.9.2-1) wheezy; urgency=low
+
+  * source package automatically created by stdeb 0.8.2
+
+ -- Paul "LeoNerd" Evans <paul@matrix.org>  Fri, 12 Jun 2015 14:32:03 +0100
--- a/debian/compat
+++ b/debian/compat
@@ -0,0 +1 @@
+9
--- a/debian/config
+++ b/debian/config
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+
+. /usr/share/debconf/confmodule
+
+db_input high matrix-synapse/server-name || true
+db_input high matrix-synapse/report-stats || true
+db_go
--- a/debian/control
+++ b/debian/control
@@ -0,0 +1,40 @@
+Source: matrix-synapse-py3
+Section: contrib/python
+Priority: extra
+Maintainer: Synapse Packaging team <packages@matrix.org>
+Build-Depends:
+ debhelper (>= 9),
+ dh-systemd,
+ dh-virtualenv (>= 1.1),
+ lsb-release,
+ python3-dev,
+ python3,
+ python3-setuptools,
+ python3-pip,
+ python3-venv,
+ tar,
+Standards-Version: 3.9.8
+Homepage: https://github.com/matrix-org/synapse
+
+Package: matrix-synapse-py3
+Architecture: amd64
+Provides: matrix-synapse
+Conflicts:
+ matrix-synapse (<< 0.34.0.1-0matrix2),
+ matrix-synapse (>= 0.34.0.1-1),
+Pre-Depends: dpkg (>= 1.16.1)
+Depends:
+ adduser,
+ debconf,
+ python3-distutils|libpython3-stdlib (<< 3.6),
+ ${misc:Depends},
+ ${synapse:pydepends},
+# some of our scripts use perl, but none of them are important,
+# so we put perl:Depends in Suggests rather than Depends.
+Suggests:
+ sqlite3,
+ ${perl:Depends},
+Description: Open federated Instant Messaging and VoIP server
+ Matrix is an ambitious new ecosystem for open federated Instant
+ Messaging and VoIP. Synapse is a reference Matrix server
+ implementation.
--- a/debian/copyright
+++ b/debian/copyright
@@ -0,0 +1,118 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: synapse
+Source: https://github.com/matrix-org/synapse
+
+Files: *
+Copyright: 2014-2017, OpenMarket Ltd, 2017-2018 New Vector Ltd
+License: Apache-2.0
+
+Files: synapse/config/saml2.py
+Copyright: 2015, Ericsson
+License: Apache-2.0
+
+Files: synapse/config/jwt.py
+Copyright: 2015, Niklas Riekenbrauck
+License: Apache-2.0
+
+Files: synapse/config/workers.py
+Copyright: 2016, matrix.org
+License: Apache-2.0
+
+Files: synapse/config/repository.py
+Copyright: 2014-2015, matrix.org
+License: Apache-2.0
+
+Files: contrib/jitsimeetbridge/unjingle/strophe/base64.js
+Copyright: Public Domain (Tyler Akins http://rumkin.com)
+License: public-domain
+ This code was written by Tyler Akins and has been placed in the
+ public domain.  It would be nice if you left this header intact.
+ Base64 code from Tyler Akins -- http://rumkin.com
+
+Files: contrib/jitsimeetbridge/unjingle/strophe/md5.js
+Copyright: 1999-2002, Paul Johnston & Contributors
+License: BSD-3-clause
+
+Files: contrib/jitsimeetbridge/unjingle/strophe/strophe.js
+Copyright: 2006-2008, OGG, LLC
+License: Expat
+
+Files: contrib/jitsimeetbridge/unjingle/strophe/XMLHttpRequest.js
+Copyright: 2010 passive.ly LLC
+License: Expat
+
+Files: contrib/jitsimeetbridge/unjingle/*.js
+Copyright: 2014 Jitsi
+License: Apache-2.0
+
+Files: debian/*
+Copyright: 2016-2017, Erik Johnston <erik@matrix.org>
+	   2017, Rahul De <rahulde@swecha.net>
+	   2017, Sunil Mohan Adapa <sunil@medhas.org>
+License: Apache-2.0
+
+License: Apache-2.0
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+ http://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ On Debian systems, the full text of the Apache License version
+ 2.0 can be found in the file
+ `/usr/share/common-licenses/Apache-2.0'.
+
+License: BSD-3-clause
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ .
+ Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following
+ disclaimer. Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided with
+ the distribution.
+ .
+ Neither the name of the author nor the names of its contributors may
+ be used to endorse or promote products derived from this software
+ without specific prior written permission.
+ .
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+License: Expat
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+ .
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
--- a/debian/dirs
+++ b/debian/dirs
@@ -0,0 +1,3 @@
+etc/matrix-synapse
+var/lib/matrix-synapse
+var/log/matrix-synapse
--- a/debian/hash_password.1
+++ b/debian/hash_password.1
@@ -0,0 +1,90 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "HASH_PASSWORD" "1" "February 2017" "" ""
+.
+.SH "NAME"
+\fBhash_password\fR \- Calculate the hash of a new password, so that passwords can be reset
+.
+.SH "SYNOPSIS"
+\fBhash_password\fR [\fB\-p\fR|\fB\-\-password\fR [password]] [\fB\-c\fR|\fB\-\-config\fR \fIfile\fR]
+.
+.SH "DESCRIPTION"
+\fBhash_password\fR calculates the hash of a supplied password using bcrypt\.
+.
+.P
+\fBhash_password\fR takes a password as an parameter either on the command line or the \fBSTDIN\fR if not supplied\.
+.
+.P
+It accepts an YAML file which can be used to specify parameters like the number of rounds for bcrypt and password_config section having the pepper value used for the hashing\. By default \fBbcrypt_rounds\fR is set to \fB10\fR\.
+.
+.P
+The hashed password is written on the \fBSTDOUT\fR\.
+.
+.SH "FILES"
+A sample YAML file accepted by \fBhash_password\fR is described below:
+.
+.P
+bcrypt_rounds: 17 password_config: pepper: "random hashing pepper"
+.
+.SH "OPTIONS"
+.
+.TP
+\fB\-p\fR, \fB\-\-password\fR
+Read the password form the command line if [password] is supplied\. If not, prompt the user and read the password form the \fBSTDIN\fR\. It is not recommended to type the password on the command line directly\. Use the STDIN instead\.
+.
+.TP
+\fB\-c\fR, \fB\-\-config\fR
+Read the supplied YAML \fIfile\fR containing the options \fBbcrypt_rounds\fR and the \fBpassword_config\fR section containing the \fBpepper\fR value\.
+.
+.SH "EXAMPLES"
+Hash from the command line:
+.
+.IP "" 4
+.
+.nf
+
+$ hash_password \-p "p@ssw0rd"
+$2b$12$VJNqWQYfsWTEwcELfoSi4Oa8eA17movHqqi8\.X8fWFpum7SxZ9MFe
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Hash from the STDIN:
+.
+.IP "" 4
+.
+.nf
+
+$ hash_password
+Password:
+Confirm password:
+$2b$12$AszlvfmJl2esnyhmn8m/kuR2tdXgROWtWxnX\.rcuAbM8ErLoUhybG
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Using a config file:
+.
+.IP "" 4
+.
+.nf
+
+$ hash_password \-c config\.yml
+Password:
+Confirm password:
+$2b$12$CwI\.wBNr\.w3kmiUlV3T5s\.GT2wH7uebDCovDrCOh18dFedlANK99O
+.
+.fi
+.
+.IP "" 0
+.
+.SH "COPYRIGHT"
+This man page was written by Rahul De <\fIrahulde@swecha\.net\fR> for Debian GNU/Linux distribution\.
+.
+.SH "SEE ALSO"
+synctl(1), synapse_port_db(1), register_new_matrix_user(1)
--- a/debian/hash_password.ronn
+++ b/debian/hash_password.ronn
@@ -0,0 +1,69 @@
+hash_password(1) -- Calculate the hash of a new password, so that passwords can be reset
+========================================================================================
+
+## SYNOPSIS
+
+`hash_password` [`-p`|`--password` [password]] [`-c`|`--config` <file>]
+
+## DESCRIPTION
+
+**hash_password** calculates the hash of a supplied password using bcrypt.
+
+`hash_password` takes a password as an parameter either on the command line
+or the `STDIN` if not supplied.
+
+It accepts an YAML file which can be used to specify parameters like the
+number of rounds for bcrypt and password_config section having the pepper
+value used for the hashing. By default `bcrypt_rounds` is set to **10**.
+
+The hashed password is written on the `STDOUT`.
+
+## FILES
+
+A sample YAML file accepted by `hash_password` is described below:
+
+  bcrypt_rounds: 17
+  password_config:
+    pepper: "random hashing pepper"
+
+## OPTIONS
+
+  * `-p`, `--password`:
+    Read the password form the command line if [password] is supplied.
+    If not, prompt the user and read the password form the `STDIN`.
+    It is not recommended to type the password on the command line
+    directly. Use the STDIN instead.
+
+  * `-c`, `--config`:
+    Read the supplied YAML <file> containing the options `bcrypt_rounds`
+    and the `password_config` section containing the `pepper` value.
+
+## EXAMPLES
+
+Hash from the command line:
+
+    $ hash_password -p "p@ssw0rd"
+    $2b$12$VJNqWQYfsWTEwcELfoSi4Oa8eA17movHqqi8.X8fWFpum7SxZ9MFe
+
+Hash from the STDIN:
+
+    $ hash_password
+    Password:
+    Confirm password:
+    $2b$12$AszlvfmJl2esnyhmn8m/kuR2tdXgROWtWxnX.rcuAbM8ErLoUhybG
+
+Using a config file:
+
+    $ hash_password -c config.yml
+    Password:
+    Confirm password:
+    $2b$12$CwI.wBNr.w3kmiUlV3T5s.GT2wH7uebDCovDrCOh18dFedlANK99O
+
+## COPYRIGHT
+
+This man page was written by Rahul De <<rahulde@swecha.net>>
+for Debian GNU/Linux distribution.
+
+## SEE ALSO
+
+synctl(1), synapse_port_db(1), register_new_matrix_user(1)
--- a/debian/install
+++ b/debian/install
@@ -0,0 +1 @@
+debian/log.yaml etc/matrix-synapse
--- a/debian/log.yaml
+++ b/debian/log.yaml
@@ -0,0 +1,36 @@
+
+version: 1
+
+formatters:
+  precise:
+   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s'
+
+filters:
+  context:
+    (): synapse.util.logcontext.LoggingContextFilter
+    request: ""
+
+handlers:
+  file:
+    class: logging.handlers.RotatingFileHandler
+    formatter: precise
+    filename: /var/log/matrix-synapse/homeserver.log
+    maxBytes: 104857600
+    backupCount: 10
+    filters: [context]
+    encoding: utf8
+  console:
+    class: logging.StreamHandler
+    formatter: precise
+    level: WARN
+
+loggers:
+    synapse:
+        level: INFO
+
+    synapse.storage.SQL:
+        level: INFO
+
+root:
+    level: INFO
+    handlers: [file, console]
--- a/debian/manpages
+++ b/debian/manpages
@@ -0,0 +1,4 @@
+debian/hash_password.1
+debian/register_new_matrix_user.1
+debian/synapse_port_db.1
+debian/synctl.1
--- a/debian/matrix-synapse-py3.links
+++ b/debian/matrix-synapse-py3.links
@@ -0,0 +1,4 @@
+opt/venvs/matrix-synapse/bin/hash_password usr/bin/hash_password
+opt/venvs/matrix-synapse/bin/register_new_matrix_user usr/bin/register_new_matrix_user
+opt/venvs/matrix-synapse/bin/synapse_port_db usr/bin/synapse_port_db
+opt/venvs/matrix-synapse/bin/synctl usr/bin/synctl
--- a/debian/matrix-synapse-py3.postinst
+++ b/debian/matrix-synapse-py3.postinst
@@ -0,0 +1,39 @@
+#!/bin/sh -e
+
+. /usr/share/debconf/confmodule
+
+CONFIGFILE_SERVERNAME="/etc/matrix-synapse/conf.d/server_name.yaml"
+CONFIGFILE_REPORTSTATS="/etc/matrix-synapse/conf.d/report_stats.yaml"
+USER="matrix-synapse"
+
+case "$1" in
+  configure|reconfigure)
+    # Set server name in config file
+    mkdir -p "/etc/matrix-synapse/conf.d/"
+    db_get matrix-synapse/server-name
+
+    if [ "$RET" ]; then
+        echo "server_name: $RET" > $CONFIGFILE_SERVERNAME
+    fi
+
+    db_get matrix-synapse/report-stats
+    if [ "$RET" ]; then
+        echo "report_stats: $RET" > $CONFIGFILE_REPORTSTATS
+    fi
+
+    if ! getent passwd $USER >/dev/null; then
+      adduser --quiet --system --no-create-home --home /var/lib/matrix-synapse $USER
+    fi
+
+    for DIR in /var/lib/matrix-synapse /var/log/matrix-synapse /etc/matrix-synapse; do
+      if ! dpkg-statoverride --list --quiet $DIR >/dev/null; then
+        dpkg-statoverride --force --quiet --update --add $USER nogroup 0755 $DIR
+      fi
+    done
+
+    ;;
+esac
+
+#DEBHELPER#
+
+exit 0
--- a/debian/matrix-synapse-py3.preinst
+++ b/debian/matrix-synapse-py3.preinst
@@ -0,0 +1,31 @@
+#!/bin/sh -e
+
+# Attempt to undo some of the braindamage caused by
+# https://github.com/matrix-org/package-synapse-debian/issues/18.
+#
+# Due to reasons [1], the old python2 matrix-synapse package will not stop the
+# service when the package is uninstalled. Our maintainer scripts will do the
+# right thing in terms of ensuring the service is enabled and unmasked, but
+# then do a `systemctl start matrix-synapse`, which of course does nothing -
+# leaving the old (py2) service running.
+#
+# There should normally be no reason for the service to be running during our
+# preinst, so we assume that if it *is* running, it's due to that situation,
+# and stop it.
+#
+# [1] dh_systemd_start doesn't do anything because it sees that there is an
+#     init.d script with the same name, so leaves it to dh_installinit.
+#
+#     dh_installinit doesn't do anything because somebody gave it a --no-start
+#     for unknown reasons.
+
+if [ -x /bin/systemctl ]; then
+    if /bin/systemctl --quiet is-active -- matrix-synapse; then
+        echo >&2 "stopping existing matrix-synapse service"
+        /bin/systemctl stop matrix-synapse || true
+    fi
+fi
+
+#DEBHELPER#
+
+exit 0
--- a/debian/matrix-synapse-py3.triggers
+++ b/debian/matrix-synapse-py3.triggers
@@ -0,0 +1,9 @@
+# Register interest in Python interpreter changes and
+# don't make the Python package dependent on the virtualenv package
+# processing (noawait)
+interest-noawait /usr/bin/python3.5
+interest-noawait /usr/bin/python3.6
+interest-noawait /usr/bin/python3.7
+
+# Also provide a symbolic trigger for all dh-virtualenv packages
+interest dh-virtualenv-interpreter-update
--- a/debian/matrix-synapse.default
+++ b/debian/matrix-synapse.default
@@ -0,0 +1,2 @@
+# Specify environment variables used when running Synapse
+# SYNAPSE_CACHE_FACTOR=1 (default)
--- a/debian/matrix-synapse.service
+++ b/debian/matrix-synapse.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=Synapse Matrix homeserver
+
+[Service]
+Type=simple
+User=matrix-synapse
+WorkingDirectory=/var/lib/matrix-synapse
+EnvironmentFile=/etc/default/matrix-synapse
+ExecStartPre=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys
+ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/
+Restart=always
+RestartSec=3
+
+[Install]
+WantedBy=multi-user.target
--- a/debian/po/POTFILES.in
+++ b/debian/po/POTFILES.in
@@ -0,0 +1 @@
+[type: gettext/rfc822deb] templates
--- a/debian/po/templates.pot
+++ b/debian/po/templates.pot
@@ -0,0 +1,56 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the matrix-synapse package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: matrix-synapse\n"
+"Report-Msgid-Bugs-To: matrix-synapse@packages.debian.org\n"
+"POT-Creation-Date: 2017-02-21 07:51+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"Language: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=CHARSET\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#. Type: string
+#. Description
+#: ../templates:1001
+msgid "Name of the server:"
+msgstr ""
+
+#. Type: string
+#. Description
+#: ../templates:1001
+msgid ""
+"The name that this homeserver will appear as, to clients and other servers "
+"via federation. This name should match the SRV record published in DNS."
+msgstr ""
+
+#. Type: boolean
+#. Description
+#: ../templates:2001
+msgid "Report anonymous statistics?"
+msgstr ""
+
+#. Type: boolean
+#. Description
+#: ../templates:2001
+msgid ""
+"Developers of Matrix and Synapse really appreciate helping the project out "
+"by reporting anonymized usage statistics from this homeserver. Only very "
+"basic aggregate data (e.g. number of users) will be reported, but it helps "
+"track the growth of the Matrix community, and helps in making Matrix a "
+"success, as well as to convince other networks that they should peer with "
+"Matrix."
+msgstr ""
+
+#. Type: boolean
+#. Description
+#: ../templates:2001
+msgid "Thank you."
+msgstr ""
--- a/debian/register_new_matrix_user.1
+++ b/debian/register_new_matrix_user.1
@@ -0,0 +1,72 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "REGISTER_NEW_MATRIX_USER" "1" "February 2017" "" ""
+.
+.SH "NAME"
+\fBregister_new_matrix_user\fR \- Used to register new users with a given home server when registration has been disabled
+.
+.SH "SYNOPSIS"
+\fBregister_new_matrix_user\fR options\.\.\.
+.
+.SH "DESCRIPTION"
+\fBregister_new_matrix_user\fR registers new users with a given home server when registration has been disabled\. For this to work, the home server must be configured with the \'registration_shared_secret\' option set\.
+.
+.P
+This accepts the user credentials like the username, password, is user an admin or not and registers the user onto the homeserver database\. Also, a YAML file containing the shared secret can be provided\. If not, the shared secret can be provided via the command line\.
+.
+.P
+By default it assumes the home server URL to be \fBhttps://localhost:8448\fR\. This can be changed via the \fBserver_url\fR command line option\.
+.
+.SH "FILES"
+A sample YAML file accepted by \fBregister_new_matrix_user\fR is described below:
+.
+.IP "" 4
+.
+.nf
+
+registration_shared_secret: "s3cr3t"
+.
+.fi
+.
+.IP "" 0
+.
+.SH "OPTIONS"
+.
+.TP
+\fB\-u\fR, \fB\-\-user\fR
+Local part of the new user\. Will prompt if omitted\.
+.
+.TP
+\fB\-p\fR, \fB\-\-password\fR
+New password for user\. Will prompt if omitted\. Supplying the password on the command line is not recommended\. Use the STDIN instead\.
+.
+.TP
+\fB\-a\fR, \fB\-\-admin\fR
+Register new user as an admin\. Will prompt if omitted\.
+.
+.TP
+\fB\-c\fR, \fB\-\-config\fR
+Path to server config file containing the shared secret\.
+.
+.TP
+\fB\-k\fR, \fB\-\-shared\-secret\fR
+Shared secret as defined in server config file\. This is an optional parameter as it can be also supplied via the YAML file\.
+.
+.TP
+\fBserver_url\fR
+URL of the home server\. Defaults to \'https://localhost:8448\'\.
+.
+.SH "EXAMPLES"
+.
+.nf
+
+$ register_new_matrix_user \-u user1 \-p p@ssword \-a \-c config\.yaml
+.
+.fi
+.
+.SH "COPYRIGHT"
+This man page was written by Rahul De <\fIrahulde@swecha\.net\fR> for Debian GNU/Linux distribution\.
+.
+.SH "SEE ALSO"
+synctl(1), synapse_port_db(1), hash_password(1)
--- a/debian/register_new_matrix_user.ronn
+++ b/debian/register_new_matrix_user.ronn
@@ -0,0 +1,61 @@
+register_new_matrix_user(1) -- Used to register new users with a given home server when registration has been disabled
+======================================================================================================================
+
+## SYNOPSIS
+
+`register_new_matrix_user` options...
+
+## DESCRIPTION
+
+**register_new_matrix_user** registers new users with a given home server when
+registration has been disabled. For this to work, the home server must be
+configured with the 'registration_shared_secret' option set.
+
+This accepts the user credentials like the username, password, is user an
+admin or not and registers the user onto the homeserver database. Also,
+a YAML file containing the shared secret can be provided. If not, the
+shared secret can be provided via the command line.
+
+By default it assumes the home server URL to be `https://localhost:8448`.
+This can be changed via the `server_url` command line option.
+
+## FILES
+
+A sample YAML file accepted by `register_new_matrix_user` is described below:
+
+    registration_shared_secret: "s3cr3t"
+
+## OPTIONS
+
+  * `-u`, `--user`:
+    Local part of the new user. Will prompt if omitted.
+
+  * `-p`, `--password`:
+    New password for user. Will prompt if omitted. Supplying the password
+    on the command line is not recommended. Use the STDIN instead.
+
+  * `-a`, `--admin`:
+    Register new user as an admin. Will prompt if omitted.
+
+  * `-c`, `--config`:
+    Path to server config file containing the shared secret.
+
+  * `-k`, `--shared-secret`:
+    Shared secret as defined in server config file. This is an optional
+    parameter as it can be also supplied via the YAML file.
+
+  * `server_url`:
+    URL of the home server. Defaults to 'https://localhost:8448'.
+
+## EXAMPLES
+
+    $ register_new_matrix_user -u user1 -p p@ssword -a -c config.yaml
+
+## COPYRIGHT
+
+This man page was written by Rahul De <<rahulde@swecha.net>>
+for Debian GNU/Linux distribution.
+
+## SEE ALSO
+
+synctl(1), synapse_port_db(1), hash_password(1)
--- a/debian/rules
+++ b/debian/rules
@@ -0,0 +1,22 @@
+#!/usr/bin/make -f
+#
+# Build Debian package using https://github.com/spotify/dh-virtualenv
+#
+
+override_dh_systemd_enable:
+	dh_systemd_enable --name=matrix-synapse
+
+override_dh_installinit:
+	dh_installinit --name=matrix-synapse
+
+override_dh_strip:
+
+override_dh_shlibdeps:
+
+override_dh_virtualenv:
+	./debian/build_virtualenv
+
+# We are restricted to compat level 9 (because xenial), so have to
+# enable the systemd bits manually.
+%:
+	dh $@ --with python-virtualenv --with systemd
--- a/debian/source/format
+++ b/debian/source/format
@@ -0,0 +1 @@
+3.0 (native)
--- a/debian/synapse_port_db.1
+++ b/debian/synapse_port_db.1
@@ -0,0 +1,98 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "SYNAPSE_PORT_DB" "1" "February 2017" "" ""
+.
+.SH "NAME"
+\fBsynapse_port_db\fR \- A script to port an existing synapse SQLite database to a new PostgreSQL database\.
+.
+.SH "SYNOPSIS"
+\fBsynapse_port_db\fR [\-v] \-\-sqlite\-database=\fIdbfile\fR \-\-postgres\-config=\fIyamlconfig\fR [\-\-curses] [\-\-batch\-size=\fIbatch\-size\fR]
+.
+.SH "DESCRIPTION"
+\fBsynapse_port_db\fR ports an existing synapse SQLite database to a new PostgreSQL database\.
+.
+.P
+SQLite database is specified with \fB\-\-sqlite\-database\fR option and PostgreSQL configuration required to connect to PostgreSQL database is provided using \fB\-\-postgres\-config\fR configuration\. The configuration is specified in YAML format\.
+.
+.SH "OPTIONS"
+.
+.TP
+\fB\-v\fR
+Print log messages in \fBdebug\fR level instead of \fBinfo\fR level\.
+.
+.TP
+\fB\-\-sqlite\-database\fR
+The snapshot of the SQLite database file\. This must not be currently used by a running synapse server\.
+.
+.TP
+\fB\-\-postgres\-config\fR
+The database config file for the PostgreSQL database\.
+.
+.TP
+\fB\-\-curses\fR
+Display a curses based progress UI\.
+.
+.SH "CONFIG FILE"
+The postgres configuration file must be a valid YAML file with the following options\.
+.
+.IP "\(bu" 4
+\fBdatabase\fR: Database configuration section\. This section header can be ignored and the options below may be specified as top level keys\.
+.
+.IP "\(bu" 4
+\fBname\fR: Connector to use when connecting to the database\. This value must be \fBpsycopg2\fR\.
+.
+.IP "\(bu" 4
+\fBargs\fR: DB API 2\.0 compatible arguments to send to the \fBpsycopg2\fR module\.
+.
+.IP "\(bu" 4
+\fBdbname\fR \- the database name
+.
+.IP "\(bu" 4
+\fBuser\fR \- user name used to authenticate
+.
+.IP "\(bu" 4
+\fBpassword\fR \- password used to authenticate
+.
+.IP "\(bu" 4
+\fBhost\fR \- database host address (defaults to UNIX socket if not provided)
+.
+.IP "\(bu" 4
+\fBport\fR \- connection port number (defaults to 5432 if not provided)
+.
+.IP "" 0
+
+.
+.IP "\(bu" 4
+\fBsynchronous_commit\fR: Optional\. Default is True\. If the value is \fBFalse\fR, enable asynchronous commit and don\'t wait for the server to call fsync before ending the transaction\. See: https://www\.postgresql\.org/docs/current/static/wal\-async\-commit\.html
+.
+.IP "" 0
+
+.
+.IP "" 0
+.
+.P
+Following example illustrates the configuration file format\.
+.
+.IP "" 4
+.
+.nf
+
+database:
+  name: psycopg2
+  args:
+    dbname: synapsedb
+    user: synapseuser
+    password: ORohmi9Eet=ohphi
+    host: localhost
+  synchronous_commit: false
+.
+.fi
+.
+.IP "" 0
+.
+.SH "COPYRIGHT"
+This man page was written by Sunil Mohan Adapa <\fIsunil@medhas\.org\fR> for Debian GNU/Linux distribution\.
+.
+.SH "SEE ALSO"
+synctl(1), hash_password(1), register_new_matrix_user(1)
--- a/debian/synapse_port_db.ronn
+++ b/debian/synapse_port_db.ronn
@@ -0,0 +1,87 @@
+synapse_port_db(1) -- A script to port an existing synapse SQLite database to a new PostgreSQL database.
+=============================================
+
+## SYNOPSIS
+
+`synapse_port_db` [-v] --sqlite-database=<dbfile> --postgres-config=<yamlconfig> [--curses] [--batch-size=<batch-size>]
+
+## DESCRIPTION
+
+**synapse_port_db** ports an existing synapse SQLite database to a new
+PostgreSQL database.
+
+SQLite database is specified with `--sqlite-database` option and
+PostgreSQL configuration required to connect to PostgreSQL database is
+provided using `--postgres-config` configuration.  The configuration
+is specified in YAML format.
+
+## OPTIONS
+
+  * `-v`:
+    Print log messages in `debug` level instead of `info` level.
+
+  * `--sqlite-database`:
+    The snapshot of the SQLite database file. This must not be
+    currently used by a running synapse server.
+
+  * `--postgres-config`:
+    The database config file for the PostgreSQL database.
+
+  * `--curses`:
+    Display a curses based progress UI.
+
+## CONFIG FILE
+
+The postgres configuration file must be a valid YAML file with the
+following options.
+
+  * `database`:
+    Database configuration section.  This section header can be
+    ignored and the options below may be specified as top level
+    keys.
+
+    * `name`:
+      Connector to use when connecting to the database.  This value must
+      be `psycopg2`.
+
+    * `args`:
+      DB API 2.0 compatible arguments to send to the `psycopg2` module.
+
+      * `dbname` - the database name 
+
+      * `user` - user name used to authenticate
+
+      * `password` - password used to authenticate
+
+      * `host` - database host address (defaults to UNIX socket if not
+        provided)
+
+      * `port` - connection port number (defaults to 5432 if not
+        provided)
+      
+
+    * `synchronous_commit`:
+      Optional.  Default is True.  If the value is `False`, enable
+      asynchronous commit and don't wait for the server to call fsync
+      before ending the transaction. See:
+      https://www.postgresql.org/docs/current/static/wal-async-commit.html
+
+Following example illustrates the configuration file format.
+
+    database:
+      name: psycopg2
+      args:
+        dbname: synapsedb
+        user: synapseuser
+        password: ORohmi9Eet=ohphi
+        host: localhost
+      synchronous_commit: false
+  
+## COPYRIGHT
+
+This man page was written by Sunil Mohan Adapa <<sunil@medhas.org>> for
+Debian GNU/Linux distribution.
+
+## SEE ALSO
+
+synctl(1), hash_password(1), register_new_matrix_user(1)
--- a/debian/synctl.1
+++ b/debian/synctl.1
@@ -0,0 +1,63 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "SYNCTL" "1" "February 2017" "" ""
+.
+.SH "NAME"
+\fBsynctl\fR \- Synapse server control interface
+.
+.SH "SYNOPSIS"
+Start, stop or restart synapse server\.
+.
+.P
+\fBsynctl\fR {start|stop|restart} [configfile] [\-w|\-\-worker=\fIWORKERCONFIG\fR] [\-a|\-\-all\-processes=\fIWORKERCONFIGDIR\fR]
+.
+.SH "DESCRIPTION"
+\fBsynctl\fR can be used to start, stop or restart Synapse server\. The control operation can be done on all processes or a single worker process\.
+.
+.SH "OPTIONS"
+.
+.TP
+\fBaction\fR
+The value of action should be one of \fBstart\fR, \fBstop\fR or \fBrestart\fR\.
+.
+.TP
+\fBconfigfile\fR
+Optional path of the configuration file to use\. Default value is \fBhomeserver\.yaml\fR\. The configuration file must exist for the operation to succeed\.
+.
+.TP
+\fB\-w\fR, \fB\-\-worker\fR:
+.
+.IP
+Perform start, stop or restart operations on a single worker\. Incompatible with \fB\-a\fR|\fB\-\-all\-processes\fR\. Value passed must be a valid worker\'s configuration file\.
+.
+.TP
+\fB\-a\fR, \fB\-\-all\-processes\fR:
+.
+.IP
+Perform start, stop or restart operations on all the workers in the given directory and the main synapse process\. Incompatible with \fB\-w\fR|\fB\-\-worker\fR\. Value passed must be a directory containing valid work configuration files\. All files ending with \fB\.yaml\fR extension shall be considered as configuration files and all other files in the directory are ignored\.
+.
+.SH "CONFIGURATION FILE"
+Configuration file may be generated as follows:
+.
+.IP "" 4
+.
+.nf
+
+$ python \-B \-m synapse\.app\.homeserver \-c config\.yaml \-\-generate\-config \-\-server\-name=<server name>
+.
+.fi
+.
+.IP "" 0
+.
+.SH "ENVIRONMENT"
+.
+.TP
+\fBSYNAPSE_CACHE_FACTOR\fR
+Synapse\'s architecture is quite RAM hungry currently \- a lot of recent room data and metadata is deliberately cached in RAM in order to speed up common requests\. This will be improved in future, but for now the easiest way to either reduce the RAM usage (at the risk of slowing things down) is to set the SYNAPSE_CACHE_FACTOR environment variable\. Roughly speaking, a SYNAPSE_CACHE_FACTOR of 1\.0 will max out at around 3\-4GB of resident memory \- this is what we currently run the matrix\.org on\. The default setting is currently 0\.1, which is probably around a ~700MB footprint\. You can dial it down further to 0\.02 if desired, which targets roughly ~512MB\. Conversely you can dial it up if you need performance for lots of users and have a box with a lot of RAM\.
+.
+.SH "COPYRIGHT"
+This man page was written by Sunil Mohan Adapa <\fIsunil@medhas\.org\fR> for Debian GNU/Linux distribution\.
+.
+.SH "SEE ALSO"
+synapse_port_db(1), hash_password(1), register_new_matrix_user(1)
--- a/debian/synctl.ronn
+++ b/debian/synctl.ronn
@@ -0,0 +1,70 @@
+synctl(1) -- Synapse server control interface
+=============================================
+
+## SYNOPSIS
+  Start, stop or restart synapse server.
+
+`synctl` {start|stop|restart} [configfile] [-w|--worker=<WORKERCONFIG>] [-a|--all-processes=<WORKERCONFIGDIR>]
+
+## DESCRIPTION
+
+**synctl** can be used to start, stop or restart Synapse server.  The
+control operation can be done on all processes or a single worker
+process.
+
+## OPTIONS
+
+  * `action`:
+    The value of action should be one of `start`, `stop` or `restart`.
+
+  * `configfile`:
+    Optional path of the configuration file to use.  Default value is
+    `homeserver.yaml`.  The configuration file must exist for the
+    operation to succeed.
+
+  * `-w`, `--worker`:
+
+    Perform start, stop or restart operations on a single worker.
+    Incompatible with `-a`|`--all-processes`.  Value passed must be a
+    valid worker's configuration file.
+
+  * `-a`, `--all-processes`:
+
+    Perform start, stop or restart operations on all the workers in
+    the given directory and the main synapse process. Incompatible
+    with `-w`|`--worker`.  Value passed must be a directory containing
+    valid work configuration files.  All files ending with `.yaml`
+    extension shall be considered as configuration files and all other
+    files in the directory are ignored.
+
+## CONFIGURATION FILE
+
+Configuration file may be generated as follows:
+
+    $ python -B -m synapse.app.homeserver -c config.yaml --generate-config --server-name=<server name>
+
+## ENVIRONMENT
+
+  * `SYNAPSE_CACHE_FACTOR`:
+    Synapse's architecture is quite RAM hungry currently - a lot of
+    recent room data and metadata is deliberately cached in RAM in
+    order to speed up common requests.  This will be improved in
+    future, but for now the easiest way to either reduce the RAM usage
+    (at the risk of slowing things down) is to set the
+    SYNAPSE_CACHE_FACTOR environment variable. Roughly speaking, a
+    SYNAPSE_CACHE_FACTOR of 1.0 will max out at around 3-4GB of
+    resident memory - this is what we currently run the matrix.org
+    on. The default setting is currently 0.1, which is probably around
+    a ~700MB footprint. You can dial it down further to 0.02 if
+    desired, which targets roughly ~512MB. Conversely you can dial it
+    up if you need performance for lots of users and have a box with a
+    lot of RAM.
+
+## COPYRIGHT
+
+This man page was written by Sunil Mohan Adapa <<sunil@medhas.org>> for
+Debian GNU/Linux distribution.
+
+## SEE ALSO
+
+synapse_port_db(1), hash_password(1), register_new_matrix_user(1)
--- a/debian/templates
+++ b/debian/templates
@@ -0,0 +1,19 @@
+Template: matrix-synapse/server-name
+Type: string
+_Description: Name of the server:
+ The name that this homeserver will appear as, to clients and other
+ servers via federation. This name should match the SRV record
+ published in DNS.
+
+Template: matrix-synapse/report-stats
+Type: boolean
+Default: false
+_Description: Report anonymous statistics?
+ Developers of Matrix and Synapse really appreciate helping the
+ project out by reporting anonymized usage statistics from this
+ homeserver. Only very basic aggregate data (e.g. number of users)
+ will be reported, but it helps track the growth of the Matrix
+ community, and helps in making Matrix a success, as well as to
+ convince other networks that they should peer with Matrix.
+ .
+ Thank you.
--- a/demo/demo.tls.dh
+++ b/demo/demo.tls.dh
@@ -1,9 +0,0 @@
-2048-bit DH parameters taken from rfc3526
-----BEGIN DH PARAMETERS-----
-MIIBCAKCAQEA///////////JD9qiIWjCNMTGYouA3BzRKQJOCIpnzHQCC76mOxOb
-IlFKCHmONATd75UZs806QxswKwpt8l8UN0/hNW1tUcJF5IW1dmJefsb0TELppjft
-awv/XLb0Brft7jhr+1qJn6WunyQRfEsf5kkoZlHs5Fs9wgB8uKFjvwWY2kg2HFXT
-mmkWP6j9JM9fg2VdI9yjrZYcYvNWIIVSu57VKQdwlpZtZww1Tkq8mATxdGwIyhgh
-fDKQXkYuNs474553LBgOhgObJ4Oi7Aeij7XFXfBvTFLJ3ivL9pVYFxg5lUl86pVq
-5RXSJhiY+gUQFXKOWoqsqmj//////////wIBAg==
-----END DH PARAMETERS-----
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -33,9 +33,7 @@ RUN pip install --prefix="/install" --no-warn-script-location \

 COPY . /synapse
 RUN pip install --prefix="/install" --no-warn-script-location \
-        lxml \
-        psycopg2 \
-        /synapse
+        /synapse[all]

 ###
 ### Stage 1: runtime
--- a/docker/Dockerfile-dhvirtualenv
+++ b/docker/Dockerfile-dhvirtualenv
@@ -0,0 +1,64 @@
+# A dockerfile which builds a docker image for building a debian package for
+# synapse. The distro to build for is passed as a docker build var.
+#
+# The default entrypoint expects the synapse source to be mounted as a
+# (read-only) volume at /synapse/source, and an output directory at /debs.
+#
+# A pair of environment variables (TARGET_USERID and TARGET_GROUPID) can be
+# passed to the docker container; if these are set, the build script will chown
+# the build products accordingly, to avoid ending up with things owned by root
+# in the host filesystem.
+
+# Get the distro we want to pull from as a dynamic build variable
+ARG distro=""
+
+###
+### Stage 0: build a dh-virtualenv
+###
+FROM ${distro} as builder
+
+RUN apt-get update -qq -o Acquire::Languages=none
+RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
+        -yqq --no-install-recommends \
+        build-essential \
+        ca-certificates \
+        devscripts \
+        equivs \
+        wget
+
+# fetch and unpack the package
+RUN wget -q -O /dh-virtuenv-1.1.tar.gz https://github.com/spotify/dh-virtualenv/archive/1.1.tar.gz
+RUN tar xvf /dh-virtuenv-1.1.tar.gz
+
+# install its build deps
+RUN cd dh-virtualenv-1.1/ \
+    && env DEBIAN_FRONTEND=noninteractive mk-build-deps -ri -t "apt-get -yqq --no-install-recommends"
+
+# build it
+RUN cd dh-virtualenv-1.1 && dpkg-buildpackage -us -uc -b
+
+###
+### Stage 1
+###
+FROM ${distro}
+
+# Install the build dependencies
+RUN apt-get update -qq -o Acquire::Languages=none \
+    && env DEBIAN_FRONTEND=noninteractive apt-get install \
+        -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
+        build-essential \
+        debhelper \
+        devscripts \
+        dh-systemd \
+        lsb-release \
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        python3-venv \
+        sqlite3
+
+COPY --from=builder /dh-virtualenv_1.1-1_all.deb /
+RUN apt-get install -yq /dh-virtualenv_1.1-1_all.deb
+
+WORKDIR /synapse/source
+ENTRYPOINT ["bash","/synapse/source/docker/build_debian.sh"]
--- a/docker/build_debian.sh
+++ b/docker/build_debian.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# The script to build the Debian package, as ran inside the Docker image.
+
+set -ex
+
+DIST=`lsb_release -c -s`
+
+# we get a read-only copy of the source: make a writeable copy
+cp -aT /synapse/source /synapse/build
+cd /synapse/build
+
+# add an entry to the changelog for this distribution
+dch -M -l "+$DIST" "build for $DIST"
+dch -M -r "" --force-distribution --distribution "$DIST"
+
+dpkg-buildpackage -us -uc
+
+ls -l ..
+
+# copy the build results out, setting perms if necessary
+shopt -s nullglob
+for i in ../*.deb ../*.dsc ../*.tar.xz ../*.changes ../*.buildinfo; do
+    [ -z "$TARGET_USERID" ] || chown "$TARGET_USERID" "$i"
+    [ -z "$TARGET_GROUPID" ] || chgrp "$TARGET_GROUPID" "$i"
+    mv "$i" /debs
+done
--- a/docker/conf/homeserver.yaml
+++ b/docker/conf/homeserver.yaml
@@ -4,7 +4,6 @@

 tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt"
 tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key"
-tls_dh_params_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.dh"
 no_tls: {{ "True" if SYNAPSE_NO_TLS else "False" }}
 tls_fingerprints: []

@@ -14,6 +13,7 @@ server_name: "{{ SYNAPSE_SERVER_NAME }}"
 pid_file: /homeserver.pid
 web_client: False
 soft_file_limit: 0
+log_config: "/compiled/log.config"

 ## Ports ##

@@ -67,9 +67,6 @@ database:
 ## Performance ##

 event_cache_size: "{{ SYNAPSE_EVENT_CACHE_SIZE or "10K" }}"
-verbose: 0
-log_file: "/data/homeserver.log"
-log_config: "/compiled/log.config"

 ## Ratelimiting ##

--- a/docs/admin_api/register_api.rst
+++ b/docs/admin_api/register_api.rst
@@ -39,13 +39,13 @@ As an example::
    }

 The MAC is the hex digest output of the HMAC-SHA1 algorithm, with the key being
-the shared secret and the content being the nonce, user, password, and either
-the string "admin" or "notadmin", each separated by NULs. For an example of
-generation in Python::
+the shared secret and the content being the nonce, user, password, either the
+string "admin" or "notadmin", and optionally the user_type
+each separated by NULs. For an example of generation in Python::

  import hmac, hashlib

-  def generate_mac(nonce, user, password, admin=False):
+  def generate_mac(nonce, user, password, admin=False, user_type=None):

      mac = hmac.new(
        key=shared_secret,
@@ -59,5 +59,8 @@ generation in Python::
      mac.update(password.encode('utf8'))
      mac.update(b"\x00")
      mac.update(b"admin" if admin else b"notadmin")
+      if user_type:
+          mac.update(b"\x00")
+          mac.update(user_type.encode('utf8'))

      return mac.hexdigest()
--- a/docs/log_contexts.rst
+++ b/docs/log_contexts.rst
@@ -163,7 +163,7 @@ the logcontext was set, this will make things work out ok: provided
 It's all too easy to forget to ``yield``: for instance if we forgot that
 ``do_some_stuff`` returned a deferred, we might plough on regardless. This
 leads to a mess; it will probably work itself out eventually, but not before
-a load of stuff has been logged against the wrong content. (Normally, other
+a load of stuff has been logged against the wrong context. (Normally, other
 things will break, more obviously, if you forget to ``yield``, so this tends
 not to be a major problem in practice.)

@@ -440,3 +440,59 @@ To conclude: I think this scheme would have worked equally well, with less
 danger of messing it up, and probably made some more esoteric code easier to
 write. But again — changing the conventions of the entire Synapse codebase is
 not a sensible option for the marginal improvement offered.
+
+
+A note on garbage-collection of Deferred chains
+-----------------------------------------------
+
+It turns out that our logcontext rules do not play nicely with Deferred
+chains which get orphaned and garbage-collected.
+
+Imagine we have some code that looks like this:
+
+.. code:: python
+
+    listener_queue = []
+
+    def on_something_interesting():
+        for d in listener_queue:
+            d.callback("foo")
+
+    @defer.inlineCallbacks
+    def await_something_interesting():
+        new_deferred = defer.Deferred()
+        listener_queue.append(new_deferred)
+
+        with PreserveLoggingContext():
+            yield new_deferred
+
+Obviously, the idea here is that we have a bunch of things which are waiting
+for an event. (It's just an example of the problem here, but a relatively
+common one.)
+
+Now let's imagine two further things happen. First of all, whatever was
+waiting for the interesting thing goes away. (Perhaps the request times out,
+or something *even more* interesting happens.)
+
+Secondly, let's suppose that we decide that the interesting thing is never
+going to happen, and we reset the listener queue:
+
+.. code:: python
+
+    def reset_listener_queue():
+        listener_queue.clear()
+
+So, both ends of the deferred chain have now dropped their references, and the
+deferred chain is now orphaned, and will be garbage-collected at some point.
+Note that ``await_something_interesting`` is a generator function, and when
+Python garbage-collects generator functions, it gives them a chance to clean
+up by making the ``yield`` raise a ``GeneratorExit`` exception. In our case,
+that means that the ``__exit__`` handler of ``PreserveLoggingContext`` will
+carefully restore the request context, but there is now nothing waiting for
+its return, so the request context is never cleared.
+
+To reiterate, this problem only arises when *both* ends of a deferred chain
+are dropped. Dropping the the reference to a deferred you're supposed to be
+calling is probably bad practice, so this doesn't actually happen too much.
+Unfortunately, when it does happen, it will lead to leaked logcontexts which
+are incredibly hard to track down.
--- a/docs/turn-howto.rst
+++ b/docs/turn-howto.rst
@@ -40,7 +40,6 @@ You may be able to setup coturn via your package manager,  or set it up manually
 4. Create or edit the config file in ``/etc/turnserver.conf``. The relevant
    lines, with example values, are::

-      lt-cred-mech
      use-auth-secret
      static-auth-secret=[your secret key here]
      realm=turn.myserver.org
@@ -52,7 +51,7 @@ You may be able to setup coturn via your package manager,  or set it up manually

 5. Consider your security settings.  TURN lets users request a relay
    which will connect to arbitrary IP addresses and ports.  At the least
-    we recommend:
+    we recommend::

       # VoIP traffic is all UDP. There is no reason to let users connect to arbitrary TCP endpoints via the relay.
       no-tcp-relay
@@ -106,7 +105,7 @@ Your home server configuration file needs the following extra keys:
    to refresh credentials. The TURN REST API specification recommends
    one day (86400000).

-  4. "turn_allow_guests": Whether to allow guest users to use the TURN
+ 4. "turn_allow_guests": Whether to allow guest users to use the TURN
    server.  This is enabled by default, as otherwise VoIP will not
    work reliably for guests.  However, it does introduce a security risk
    as it lets guests connect to arbitrary endpoints without having gone
--- a/scripts-dev/build_debian_packages
+++ b/scripts-dev/build_debian_packages
@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+
+# Build the Debian packages using Docker images.
+#
+# This script builds the Docker images and then executes them sequentially, each
+# one building a Debian package for the targeted operating system. It is
+# designed to be a "single command" to produce all the images.
+#
+# By default, builds for all known distributions, but a list of distributions
+# can be passed on the commandline for debugging.
+
+import argparse
+import os
+import signal
+import subprocess
+import sys
+import threading
+from concurrent.futures import ThreadPoolExecutor
+
+DISTS = (
+    "debian:stretch",
+    "debian:buster",
+    "debian:sid",
+    "ubuntu:xenial",
+    "ubuntu:bionic",
+    "ubuntu:cosmic",
+)
+
+DESC = '''\
+Builds .debs for synapse, using a Docker image for the build environment.
+
+By default, builds for all known distributions, but a list of distributions
+can be passed on the commandline for debugging.
+'''
+
+
+class Builder(object):
+    def __init__(self, redirect_stdout=False):
+        self.redirect_stdout = redirect_stdout
+        self.active_containers = set()
+        self._lock = threading.Lock()
+        self._failed = False
+
+    def run_build(self, dist):
+        """Build deb for a single distribution"""
+
+        if self._failed:
+            print("not building %s due to earlier failure" % (dist, ))
+            raise Exception("failed")
+
+        try:
+            self._inner_build(dist)
+        except Exception as e:
+            print("build of %s failed: %s" % (dist, e), file=sys.stderr)
+            self._failed = True
+            raise
+
+    def _inner_build(self, dist):
+        projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+        os.chdir(projdir)
+
+        tag = dist.split(":", 1)[1]
+
+        # Make the dir where the debs will live.
+        #
+        # Note that we deliberately put this outside the source tree, otherwise
+        # we tend to get source packages which are full of debs. (We could hack
+        # around that with more magic in the build_debian.sh script, but that
+        # doesn't solve the problem for natively-run dpkg-buildpakage).
+        debsdir = os.path.join(projdir, '../debs')
+        os.makedirs(debsdir, exist_ok=True)
+
+        if self.redirect_stdout:
+            logfile = os.path.join(debsdir, "%s.buildlog" % (tag, ))
+            print("building %s: directing output to %s" % (dist, logfile))
+            stdout = open(logfile, "w")
+        else:
+            stdout = None
+
+        # first build a docker image for the build environment
+        subprocess.check_call([
+            "docker", "build",
+            "--tag", "dh-venv-builder:" + tag,
+            "--build-arg", "distro=" + dist,
+            "-f", "docker/Dockerfile-dhvirtualenv",
+            "docker",
+        ], stdout=stdout, stderr=subprocess.STDOUT)
+
+        container_name = "synapse_build_" + tag
+        with self._lock:
+            self.active_containers.add(container_name)
+
+        # then run the build itself
+        subprocess.check_call([
+            "docker", "run",
+            "--rm",
+            "--name", container_name,
+            "--volume=" + projdir + ":/synapse/source:ro",
+            "--volume=" + debsdir + ":/debs",
+            "-e", "TARGET_USERID=%i" % (os.getuid(), ),
+            "-e", "TARGET_GROUPID=%i" % (os.getgid(), ),
+            "dh-venv-builder:" + tag,
+        ], stdout=stdout, stderr=subprocess.STDOUT)
+
+        with self._lock:
+            self.active_containers.remove(container_name)
+
+        if stdout is not None:
+            stdout.close()
+            print("Completed build of %s" % (dist, ))
+
+    def kill_containers(self):
+        with self._lock:
+            active = list(self.active_containers)
+
+        for c in active:
+            print("killing container %s" % (c,))
+            subprocess.run([
+                "docker", "kill", c,
+            ], stdout=subprocess.DEVNULL)
+            with self._lock:
+                self.active_containers.remove(c)
+
+
+def run_builds(dists, jobs=1):
+    builder = Builder(redirect_stdout=(jobs > 1))
+
+    def sig(signum, _frame):
+        print("Caught SIGINT")
+        builder.kill_containers()
+    signal.signal(signal.SIGINT, sig)
+
+    with ThreadPoolExecutor(max_workers=jobs) as e:
+        res = e.map(builder.run_build, dists)
+
+    # make sure we consume the iterable so that exceptions are raised.
+    for r in res:
+        pass
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description=DESC,
+    )
+    parser.add_argument(
+        '-j', '--jobs', type=int, default=1,
+        help='specify the number of builds to run in parallel',
+    )
+    parser.add_argument(
+        'dist', nargs='*', default=DISTS,
+        help='a list of distributions to build for. Default: %(default)s',
+    )
+    args = parser.parse_args()
+    run_builds(dists=args.dist, jobs=args.jobs)
--- a/scripts/generate_config
+++ b/scripts/generate_config
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+import argparse
+import sys
+
+from synapse.config.homeserver import HomeServerConfig
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--config-dir",
+        default="CONFDIR",
+
+        help="The path where the config files are kept. Used to create filenames for "
+             "things like the log config and the signing key. Default: %(default)s",
+    )
+
+    parser.add_argument(
+        "--data-dir",
+        default="DATADIR",
+        help="The path where the data files are kept. Used to create filenames for "
+             "things like the database and media store. Default: %(default)s",
+    )
+
+    parser.add_argument(
+        "--server-name",
+        default="SERVERNAME",
+        help="The server name. Used to initialise the server_name config param, but also "
+             "used in the names of some of the config files. Default: %(default)s",
+    )
+
+    parser.add_argument(
+        "--report-stats",
+        action="store",
+        help="Whether the generated config reports anonymized usage statistics",
+        choices=["yes", "no"],
+    )
+
+    parser.add_argument(
+        "--generate-secrets",
+        action="store_true",
+        help="Enable generation of new secrets for things like the macaroon_secret_key."
+             "By default, these parameters will be left unset."
+    )
+
+    parser.add_argument(
+        "-o", "--output-file",
+        type=argparse.FileType('w'),
+        default=sys.stdout,
+        help="File to write the configuration to. Default: stdout",
+    )
+
+    args = parser.parse_args()
+
+    report_stats = args.report_stats
+    if report_stats is not None:
+        report_stats = report_stats == "yes"
+
+    conf = HomeServerConfig().generate_config(
+        config_dir_path=args.config_dir,
+        data_dir_path=args.data_dir,
+        server_name=args.server_name,
+        generate_secrets=args.generate_secrets,
+        report_stats=report_stats,
+    )
+
+    args.output_file.write(conf)
--- a/setup.py
+++ b/setup.py
@@ -84,13 +84,25 @@ version = exec_file(("synapse", "__init__.py"))["__version__"]
 dependencies = exec_file(("synapse", "python_dependencies.py"))
 long_description = read_file(("README.rst",))

+REQUIREMENTS = dependencies['REQUIREMENTS']
+CONDITIONAL_REQUIREMENTS = dependencies['CONDITIONAL_REQUIREMENTS']
+
+# Make `pip install matrix-synapse[all]` install all the optional dependencies.
+ALL_OPTIONAL_REQUIREMENTS = set()
+
+for optional_deps in CONDITIONAL_REQUIREMENTS.values():
+    ALL_OPTIONAL_REQUIREMENTS = set(optional_deps) | ALL_OPTIONAL_REQUIREMENTS
+
+CONDITIONAL_REQUIREMENTS["all"] = list(ALL_OPTIONAL_REQUIREMENTS)
+
+
 setup(
    name="matrix-synapse",
    version=version,
    packages=find_packages(exclude=["tests", "tests.*"]),
    description="Reference homeserver for the Matrix decentralised comms protocol",
-    install_requires=dependencies['requirements'](include_conditional=True).keys(),
-    dependency_links=dependencies["DEPENDENCY_LINKS"].values(),
+    install_requires=REQUIREMENTS,
+    extras_require=CONDITIONAL_REQUIREMENTS,
    include_package_data=True,
    zip_safe=False,
    long_description=long_description,
--- a/synapse/init.py
+++ b/synapse/init.py
@@ -27,4 +27,4 @@ try:
 except ImportError:
    pass

-__version__ = "0.33.9"
+__version__ = "0.99.0rc2"
--- a/synapse/_scripts/register_new_matrix_user.py
+++ b/synapse/_scripts/register_new_matrix_user.py
@@ -35,6 +35,7 @@ def request_registration(
    server_location,
    shared_secret,
    admin=False,
+    user_type=None,
    requests=_requests,
    _print=print,
    exit=sys.exit,
@@ -45,7 +46,7 @@ def request_registration(
    # Get the nonce
    r = requests.get(url, verify=False)

-    if r.status_code is not 200:
+    if r.status_code != 200:
        _print("ERROR! Received %d %s" % (r.status_code, r.reason))
        if 400 <= r.status_code < 500:
            try:
@@ -65,6 +66,9 @@ def request_registration(
    mac.update(password.encode('utf8'))
    mac.update(b"\x00")
    mac.update(b"admin" if admin else b"notadmin")
+    if user_type:
+        mac.update(b"\x00")
+        mac.update(user_type.encode('utf8'))

    mac = mac.hexdigest()

@@ -74,12 +78,13 @@ def request_registration(
        "password": password,
        "mac": mac,
        "admin": admin,
+        "user_type": user_type,
    }

    _print("Sending registration request...")
    r = requests.post(url, json=data, verify=False)

-    if r.status_code is not 200:
+    if r.status_code != 200:
        _print("ERROR! Received %d %s" % (r.status_code, r.reason))
        if 400 <= r.status_code < 500:
            try:
@@ -91,7 +96,7 @@ def request_registration(
    _print("Success!")


-def register_new_user(user, password, server_location, shared_secret, admin):
+def register_new_user(user, password, server_location, shared_secret, admin, user_type):
    if not user:
        try:
            default_user = getpass.getuser()
@@ -129,7 +134,8 @@ def register_new_user(user, password, server_location, shared_secret, admin):
        else:
            admin = False

-    request_registration(user, password, server_location, shared_secret, bool(admin))
+    request_registration(user, password, server_location, shared_secret,
+                         bool(admin), user_type)


 def main():
@@ -154,6 +160,12 @@ def main():
        default=None,
        help="New password for user. Will prompt if omitted.",
    )
+    parser.add_argument(
+        "-t",
+        "--user_type",
+        default=None,
+        help="User type as specified in synapse.api.constants.UserTypes",
+    )
    admin_group = parser.add_mutually_exclusive_group()
    admin_group.add_argument(
        "-a",
@@ -208,7 +220,8 @@ def main():
    if args.admin or args.no_admin:
        admin = args.admin

-    register_new_user(args.user, args.password, args.server_url, secret, admin)
+    register_new_user(args.user, args.password, args.server_url, secret,
+                      admin, args.user_type)


 if __name__ == "__main__":
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -65,7 +65,7 @@ class Auth(object):
        register_cache("cache", "token_cache", self.token_cache)

    @defer.inlineCallbacks
-    def check_from_context(self, event, context, do_sig_check=True):
+    def check_from_context(self, room_version, event, context, do_sig_check=True):
        prev_state_ids = yield context.get_prev_state_ids(self.store)
        auth_events_ids = yield self.compute_auth_events(
            event, prev_state_ids, for_verification=True,
@@ -74,12 +74,16 @@ class Auth(object):
        auth_events = {
            (e.type, e.state_key): e for e in itervalues(auth_events)
        }
-        self.check(event, auth_events=auth_events, do_sig_check=do_sig_check)
+        self.check(
+            room_version, event,
+            auth_events=auth_events, do_sig_check=do_sig_check,
+        )

-    def check(self, event, auth_events, do_sig_check=True):
+    def check(self, room_version, event, auth_events, do_sig_check=True):
        """ Checks if this event is correctly authed.

        Args:
+            room_version (str): version of the room
            event: the event being checked.
            auth_events (dict: event-key -> event): the existing room state.

@@ -88,7 +92,9 @@ class Auth(object):
            True if the auth checks pass.
        """
        with Measure(self.clock, "auth.check"):
-            event_auth.check(event, auth_events, do_sig_check=do_sig_check)
+            event_auth.check(
+                room_version, event, auth_events, do_sig_check=do_sig_check
+            )

    @defer.inlineCallbacks
    def check_joined_room(self, room_id, user_id, current_state=None):
@@ -188,17 +194,33 @@ class Auth(object):
        """
        # Can optionally look elsewhere in the request (e.g. headers)
        try:
-            user_id, app_service = yield self._get_appservice_user_id(request)
-            if user_id:
-                request.authenticated_entity = user_id
-                defer.returnValue(
-                    synapse.types.create_requester(user_id, app_service=app_service)
-                )
+            ip_addr = self.hs.get_ip_from_request(request)
+            user_agent = request.requestHeaders.getRawHeaders(
+                b"User-Agent",
+                default=[b""]
+            )[0].decode('ascii', 'surrogateescape')

            access_token = self.get_access_token_from_request(
                request, self.TOKEN_NOT_FOUND_HTTP_STATUS
            )

+            user_id, app_service = yield self._get_appservice_user_id(request)
+            if user_id:
+                request.authenticated_entity = user_id
+
+                if ip_addr and self.hs.config.track_appservice_user_ips:
+                    yield self.store.insert_client_ip(
+                        user_id=user_id,
+                        access_token=access_token,
+                        ip=ip_addr,
+                        user_agent=user_agent,
+                        device_id="dummy-device",  # stubbed
+                    )
+
+                defer.returnValue(
+                    synapse.types.create_requester(user_id, app_service=app_service)
+                )
+
            user_info = yield self.get_user_by_access_token(access_token, rights)
            user = user_info["user"]
            token_id = user_info["token_id"]
@@ -208,11 +230,6 @@ class Auth(object):
            # stubbed out.
            device_id = user_info.get("device_id")

-            ip_addr = self.hs.get_ip_from_request(request)
-            user_agent = request.requestHeaders.getRawHeaders(
-                b"User-Agent",
-                default=[b""]
-            )[0].decode('ascii', 'surrogateescape')
            if user and access_token and ip_addr:
                yield self.store.insert_client_ip(
                    user_id=user.to_string(),
@@ -289,20 +306,28 @@ class Auth(object):
        Raises:
            AuthError if no user by that token exists or the token is invalid.
        """
+
+        if rights == "access":
+            # first look in the database
+            r = yield self._look_up_user_by_access_token(token)
+            if r:
+                defer.returnValue(r)
+
+        # otherwise it needs to be a valid macaroon
        try:
            user_id, guest = self._parse_and_validate_macaroon(token, rights)
-        except _InvalidMacaroonException:
-            # doesn't look like a macaroon: treat it as an opaque token which
-            # must be in the database.
-            # TODO: it would be nice to get rid of this, but apparently some
-            # people use access tokens which aren't macaroons
-            r = yield self._look_up_user_by_access_token(token)
-            defer.returnValue(r)
-
-        try:
            user = UserID.from_string(user_id)

-            if guest:
+            if rights == "access":
+                if not guest:
+                    # non-guest access tokens must be in the database
+                    logger.warning("Unrecognised access token - not in store.")
+                    raise AuthError(
+                        self.TOKEN_NOT_FOUND_HTTP_STATUS,
+                        "Unrecognised access token.",
+                        errcode=Codes.UNKNOWN_TOKEN,
+                    )
+
                # Guest access tokens are not stored in the database (there can
                # only be one access token per guest, anyway).
                #
@@ -343,31 +368,15 @@ class Auth(object):
                    "device_id": None,
                }
            else:
-                # This codepath exists for several reasons:
-                #   * so that we can actually return a token ID, which is used
-                #     in some parts of the schema (where we probably ought to
-                #     use device IDs instead)
-                #   * the only way we currently have to invalidate an
-                #     access_token is by removing it from the database, so we
-                #     have to check here that it is still in the db
-                #   * some attributes (notably device_id) aren't stored in the
-                #     macaroon. They probably should be.
-                # TODO: build the dictionary from the macaroon once the
-                # above are fixed
-                ret = yield self._look_up_user_by_access_token(token)
-                if ret["user"] != user:
-                    logger.error(
-                        "Macaroon user (%s) != DB user (%s)",
-                        user,
-                        ret["user"]
-                    )
-                    raise AuthError(
-                        self.TOKEN_NOT_FOUND_HTTP_STATUS,
-                        "User mismatch in macaroon",
-                        errcode=Codes.UNKNOWN_TOKEN
-                    )
+                raise RuntimeError("Unknown rights setting %s", rights)
            defer.returnValue(ret)
-        except (pymacaroons.exceptions.MacaroonException, TypeError, ValueError):
+        except (
+            _InvalidMacaroonException,
+            pymacaroons.exceptions.MacaroonException,
+            TypeError,
+            ValueError,
+        ) as e:
+            logger.warning("Invalid macaroon in auth: %s %s", type(e), e)
            raise AuthError(
                self.TOKEN_NOT_FOUND_HTTP_STATUS, "Invalid macaroon passed.",
                errcode=Codes.UNKNOWN_TOKEN
@@ -497,11 +506,8 @@ class Auth(object):
    def _look_up_user_by_access_token(self, token):
        ret = yield self.store.get_user_by_access_token(token)
        if not ret:
-            logger.warn("Unrecognised access token - not in store.")
-            raise AuthError(
-                self.TOKEN_NOT_FOUND_HTTP_STATUS, "Unrecognised access token.",
-                errcode=Codes.UNKNOWN_TOKEN
-            )
+            defer.returnValue(None)
+
        # we use ret.get() below because *lots* of unit tests stub out
        # get_user_by_access_token in a way where it only returns a couple of
        # the fields.
@@ -544,17 +550,6 @@ class Auth(object):
        """
        return self.store.is_server_admin(user)

-    @defer.inlineCallbacks
-    def add_auth_events(self, builder, context):
-        prev_state_ids = yield context.get_prev_state_ids(self.store)
-        auth_ids = yield self.compute_auth_events(builder, prev_state_ids)
-
-        auth_events_entries = yield self.store.add_event_hashes(
-            auth_ids
-        )
-
-        builder.auth_events = auth_events_entries
-
    @defer.inlineCallbacks
    def compute_auth_events(self, event, current_state_ids, for_verification=False):
        if event.type == EventTypes.Create:
@@ -571,7 +566,7 @@ class Auth(object):
        key = (EventTypes.JoinRules, "", )
        join_rule_event_id = current_state_ids.get(key)

-        key = (EventTypes.Member, event.user_id, )
+        key = (EventTypes.Member, event.sender, )
        member_event_id = current_state_ids.get(key)

        key = (EventTypes.Create, "", )
@@ -621,7 +616,7 @@ class Auth(object):

        defer.returnValue(auth_ids)

-    def check_redaction(self, event, auth_events):
+    def check_redaction(self, room_version, event, auth_events):
        """Check whether the event sender is allowed to redact the target event.

        Returns:
@@ -634,7 +629,7 @@ class Auth(object):
            AuthError if the event sender is definitely not allowed to redact
            the target event.
        """
-        return event_auth.check_redaction(event, auth_events)
+        return event_auth.check_redaction(room_version, event, auth_events)

    @defer.inlineCallbacks
    def check_can_change_room_list(self, room_id, user):
@@ -791,9 +786,10 @@ class Auth(object):
            threepid should never be set at the same time.
        """

-        # Never fail an auth check for the server notices users
+        # Never fail an auth check for the server notices users or support user
        # This can be a problem where event creation is prohibited due to blocking
-        if user_id == self.hs.config.server_notices_mxid:
+        is_support = yield self.store.is_support_user(user_id)
+        if user_id == self.hs.config.server_notices_mxid or is_support:
            return

        if self.hs.config.hs_disabled:
@@ -818,7 +814,9 @@ class Auth(object):
            elif threepid:
                # If the user does not exist yet, but is signing up with a
                # reserved threepid then pass auth check
-                if is_threepid_reserved(self.hs.config, threepid):
+                if is_threepid_reserved(
+                    self.hs.config.mau_limits_reserved_threepids, threepid
+                ):
                    return
            # Else if there is no room in the MAU bucket, bail
            current_mau = yield self.store.get_monthly_active_count()
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -68,10 +68,12 @@ class EventTypes(object):
    Aliases = "m.room.aliases"
    Redaction = "m.room.redaction"
    ThirdPartyInvite = "m.room.third_party_invite"
+    Encryption = "m.room.encryption"

    RoomHistoryVisibility = "m.room.history_visibility"
    CanonicalAlias = "m.room.canonical_alias"
    RoomAvatar = "m.room.avatar"
+    RoomEncryption = "m.room.encryption"
    GuestAccess = "m.room.guest_access"

    # These are used for validation
@@ -102,10 +104,16 @@ class ThirdPartyEntityKind(object):

 class RoomVersions(object):
    V1 = "1"
-    VDH_TEST = "vdh-test-version"
+    V2 = "2"
+    V3 = "3"
    STATE_V2_TEST = "state-v2-test"


+class RoomDisposition(object):
+    STABLE = "stable"
+    UNSTABLE = "unstable"
+
+
 # the version we will give rooms which are created on this server
 DEFAULT_ROOM_VERSION = RoomVersions.V1

@@ -113,9 +121,34 @@ DEFAULT_ROOM_VERSION = RoomVersions.V1
 # until we have a working v2.
 KNOWN_ROOM_VERSIONS = {
    RoomVersions.V1,
-    RoomVersions.VDH_TEST,
+    RoomVersions.V2,
+    RoomVersions.V3,
    RoomVersions.STATE_V2_TEST,
+    RoomVersions.V3,
 }

+
+class EventFormatVersions(object):
+    """This is an internal enum for tracking the version of the event format,
+    independently from the room version.
+    """
+    V1 = 1
+    V2 = 2
+
+
+KNOWN_EVENT_FORMAT_VERSIONS = {
+    EventFormatVersions.V1,
+    EventFormatVersions.V2,
+}
+
+
 ServerNoticeMsgType = "m.server_notice"
 ServerNoticeLimitReached = "m.server_notice.usage_limit_reached"
+
+
+class UserTypes(object):
+    """Allows for user type specific behaviour. With the benefit of hindsight
+    'admin' and 'guest' users should also be UserTypes. Normal users are type None
+    """
+    SUPPORT = "support"
+    ALL_USER_TYPES = (SUPPORT,)
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -348,6 +348,24 @@ class IncompatibleRoomVersionError(SynapseError):
        )


+class RequestSendFailed(RuntimeError):
+    """Sending a HTTP request over federation failed due to not being able to
+    talk to the remote server for some reason.
+
+    This exception is used to differentiate "expected" errors that arise due to
+    networking (e.g. DNS failures, connection timeouts etc), versus unexpected
+    errors (like programming errors).
+    """
+    def __init__(self, inner_exception, can_retry):
+        super(RequestSendFailed, self).__init__(
+            "Failed to send request: %s: %s" % (
+                type(inner_exception).__name__, inner_exception,
+            )
+        )
+        self.inner_exception = inner_exception
+        self.can_retry = can_retry
+
+
 def cs_error(msg, code=Codes.UNKNOWN, **kwargs):
    """ Utility method for constructing an error response for client-server
    interactions.
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -12,6 +12,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from six import text_type
+
 import jsonschema
 from canonicaljson import json
 from jsonschema import FormatChecker
@@ -353,7 +355,7 @@ class Filter(object):
            sender = event.user_id
            room_id = None
            ev_type = "m.presence"
-            is_url = False
+            contains_url = False
        else:
            sender = event.get("sender", None)
            if not sender:
@@ -368,13 +370,16 @@ class Filter(object):

            room_id = event.get("room_id", None)
            ev_type = event.get("type", None)
-            is_url = "url" in event.get("content", {})
+
+            content = event.get("content", {})
+            # check if there is a string url field in the content for filtering purposes
+            contains_url = isinstance(content.get("url"), text_type)

        return self.check_fields(
            room_id,
            sender,
            ev_type,
-            is_url,
+            contains_url,
        )

    def check_fields(self, room_id, sender, event_type, contains_url):
@@ -439,6 +444,20 @@ class Filter(object):
    def include_redundant_members(self):
        return self.filter_json.get("include_redundant_members", False)

+    def with_room_ids(self, room_ids):
+        """Returns a new filter with the given room IDs appended.
+
+        Args:
+            room_ids (iterable[unicode]): The room_ids to add
+
+        Returns:
+            filter: A new filter including the given rooms and the old
+                    filter's rooms.
+        """
+        newFilter = Filter(self.filter_json)
+        newFilter.rooms += room_ids
+        return newFilter
+

 def _matches_wildcard(actual_value, filter_value):
    if filter_value.endswith("*"):
--- a/synapse/api/urls.py
+++ b/synapse/api/urls.py
@@ -24,7 +24,9 @@ from synapse.config import ConfigError

 CLIENT_PREFIX = "/_matrix/client/api/v1"
 CLIENT_V2_ALPHA_PREFIX = "/_matrix/client/v2_alpha"
-FEDERATION_PREFIX = "/_matrix/federation/v1"
+FEDERATION_PREFIX = "/_matrix/federation"
+FEDERATION_V1_PREFIX = FEDERATION_PREFIX + "/v1"
+FEDERATION_V2_PREFIX = FEDERATION_PREFIX + "/v2"
 STATIC_PREFIX = "/_matrix/static"
 WEB_CLIENT_PREFIX = "/_matrix/client"
 CONTENT_REPO_PREFIX = "/_matrix/content"
--- a/synapse/app/init.py
+++ b/synapse/app/init.py
@@ -12,22 +12,38 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import logging
 import sys

 from synapse import python_dependencies  # noqa: E402

 sys.dont_write_bytecode = True

+logger = logging.getLogger(__name__)

 try:
    python_dependencies.check_requirements()
-except python_dependencies.MissingRequirementError as e:
-    message = "\n".join([
-        "Missing Requirement: %s" % (str(e),),
-        "To install run:",
-        "    pip install --upgrade --force \"%s\"" % (e.dependency,),
-        "",
-    ])
-    sys.stderr.writelines(message)
+except python_dependencies.DependencyException as e:
+    sys.stderr.writelines(e.message)
    sys.exit(1)
+
+
+def check_bind_error(e, address, bind_addresses):
+    """
+    This method checks an exception occurred while binding on 0.0.0.0.
+    If :: is specified in the bind addresses a warning is shown.
+    The exception is still raised otherwise.
+
+    Binding on both 0.0.0.0 and :: causes an exception on Linux and macOS
+    because :: binds on both IPv4 and IPv6 (as per RFC 3493).
+    When binding on 0.0.0.0 after :: this can safely be ignored.
+
+    Args:
+        e (Exception): Exception that was caught.
+        address (str): Address on which binding was attempted.
+        bind_addresses (list): Addresses on which the service listens.
+    """
+    if address == '0.0.0.0' and '::' in bind_addresses:
+        logger.warn('Failed to listen on 0.0.0.0, continuing because listening on [::]')
+    else:
+        raise e
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -22,6 +22,7 @@ from daemonize import Daemonize

 from twisted.internet import error, reactor

+from synapse.app import check_bind_error
 from synapse.util import PreserveLoggingContext
 from synapse.util.rlimit import change_resource_limit

@@ -143,6 +144,9 @@ def listen_metrics(bind_addresses, port):
 def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50):
    """
    Create a TCP socket for a port and several addresses
+
+    Returns:
+        list (empty)
    """
    for address in bind_addresses:
        try:
@@ -155,42 +159,33 @@ def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50):
        except error.CannotListenError as e:
            check_bind_error(e, address, bind_addresses)

+    logger.info("Synapse now listening on TCP port %d", port)
+    return []
+

 def listen_ssl(
    bind_addresses, port, factory, context_factory, reactor=reactor, backlog=50
 ):
    """
-    Create an SSL socket for a port and several addresses
+    Create an TLS-over-TCP socket for a port and several addresses
+
+    Returns:
+        list of twisted.internet.tcp.Port listening for TLS connections
    """
+    r = []
    for address in bind_addresses:
        try:
-            reactor.listenSSL(
-                port,
-                factory,
-                context_factory,
-                backlog,
-                address
+            r.append(
+                reactor.listenSSL(
+                    port,
+                    factory,
+                    context_factory,
+                    backlog,
+                    address
+                )
            )
        except error.CannotListenError as e:
            check_bind_error(e, address, bind_addresses)

-
-def check_bind_error(e, address, bind_addresses):
-    """
-    This method checks an exception occurred while binding on 0.0.0.0.
-    If :: is specified in the bind addresses a warning is shown.
-    The exception is still raised otherwise.
-
-    Binding on both 0.0.0.0 and :: causes an exception on Linux and macOS
-    because :: binds on both IPv4 and IPv6 (as per RFC 3493).
-    When binding on 0.0.0.0 after :: this can safely be ignored.
-
-    Args:
-        e (Exception): Exception that was caught.
-        address (str): Address on which binding was attempted.
-        bind_addresses (list): Addresses on which the service listens.
-    """
-    if address == '0.0.0.0' and '::' in bind_addresses:
-        logger.warn('Failed to listen on 0.0.0.0, continuing because listening on [::]')
-    else:
-        raise e
+    logger.info("Synapse now listening on port %d (TLS)", port)
+    return r
--- a/synapse/app/client_reader.py
+++ b/synapse/app/client_reader.py
@@ -164,23 +164,23 @@ def start(config_options):

    database_engine = create_engine(config.database_config)

-    tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
-
    ss = ClientReaderServer(
        config.server_name,
        db_config=config.database_config,
-        tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
        config=config,
        version_string="Synapse/" + get_version_string(synapse),
        database_engine=database_engine,
    )

    ss.setup()
-    ss.start_listening(config.worker_listeners)

    def start():
+        ss.config.read_certificate_from_disk()
+        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
+        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+            config
+        )
+        ss.start_listening(config.worker_listeners)
        ss.get_datastore().start_profiling()

    reactor.callWhenRunning(start)
--- a/synapse/app/event_creator.py
+++ b/synapse/app/event_creator.py
@@ -185,23 +185,23 @@ def start(config_options):

    database_engine = create_engine(config.database_config)

-    tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
-
    ss = EventCreatorServer(
        config.server_name,
        db_config=config.database_config,
-        tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
        config=config,
        version_string="Synapse/" + get_version_string(synapse),
        database_engine=database_engine,
    )

    ss.setup()
-    ss.start_listening(config.worker_listeners)

    def start():
+        ss.config.read_certificate_from_disk()
+        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
+        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+            config
+        )
+        ss.start_listening(config.worker_listeners)
        ss.get_datastore().start_profiling()

    reactor.callWhenRunning(start)
--- a/synapse/app/federation_reader.py
+++ b/synapse/app/federation_reader.py
@@ -151,23 +151,23 @@ def start(config_options):

    database_engine = create_engine(config.database_config)

-    tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
-
    ss = FederationReaderServer(
        config.server_name,
        db_config=config.database_config,
-        tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
        config=config,
        version_string="Synapse/" + get_version_string(synapse),
        database_engine=database_engine,
    )

    ss.setup()
-    ss.start_listening(config.worker_listeners)

    def start():
+        ss.config.read_certificate_from_disk()
+        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
+        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+            config
+        )
+        ss.start_listening(config.worker_listeners)
        ss.get_datastore().start_profiling()

    reactor.callWhenRunning(start)
--- a/synapse/app/federation_sender.py
+++ b/synapse/app/federation_sender.py
@@ -183,24 +183,24 @@ def start(config_options):
    # Force the pushers to start since they will be disabled in the main config
    config.send_federation = True

-    tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
-
-    ps = FederationSenderServer(
+    ss = FederationSenderServer(
        config.server_name,
        db_config=config.database_config,
-        tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
        config=config,
        version_string="Synapse/" + get_version_string(synapse),
        database_engine=database_engine,
    )

-    ps.setup()
-    ps.start_listening(config.worker_listeners)
+    ss.setup()

    def start():
-        ps.get_datastore().start_profiling()
+        ss.config.read_certificate_from_disk()
+        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
+        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+            config
+        )
+        ss.start_listening(config.worker_listeners)
+        ss.get_datastore().start_profiling()

    reactor.callWhenRunning(start)
    _base.start_worker_reactor("synapse-federation-sender", config)
--- a/synapse/app/frontend_proxy.py
+++ b/synapse/app/frontend_proxy.py
@@ -241,23 +241,23 @@ def start(config_options):

    database_engine = create_engine(config.database_config)

-    tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
-
    ss = FrontendProxyServer(
        config.server_name,
        db_config=config.database_config,
-        tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
        config=config,
        version_string="Synapse/" + get_version_string(synapse),
        database_engine=database_engine,
    )

    ss.setup()
-    ss.start_listening(config.worker_listeners)

    def start():
+        ss.config.read_certificate_from_disk()
+        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
+        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+            config
+        )
+        ss.start_listening(config.worker_listeners)
        ss.get_datastore().start_profiling()

    reactor.callWhenRunning(start)
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -13,10 +13,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import gc
 import logging
 import os
+import signal
 import sys
+import traceback

 from six import iteritems

@@ -25,6 +28,7 @@ from prometheus_client import Gauge

 from twisted.application import service
 from twisted.internet import defer, reactor
+from twisted.protocols.tls import TLSMemoryBIOFactory
 from twisted.web.resource import EncodingResourceWrapper, NoResource
 from twisted.web.server import GzipEncoderFactory
 from twisted.web.static import File
@@ -54,12 +58,13 @@ from synapse.metrics import RegistryProxy
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.module_api import ModuleApi
-from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, check_requirements
+from synapse.python_dependencies import check_requirements
 from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
 from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
 from synapse.rest import ClientRestResource
 from synapse.rest.key.v2 import KeyApiV2Resource
 from synapse.rest.media.v0.content_repository import ContentRepoResource
+from synapse.rest.well_known import WellKnownResource
 from synapse.server import HomeServer
 from synapse.storage import DataStore, are_all_users_on_domain
 from synapse.storage.engines import IncorrectDatabaseSetup, create_engine
@@ -79,38 +84,9 @@ def gz_wrap(r):
    return EncodingResourceWrapper(r, [GzipEncoderFactory()])


-def build_resource_for_web_client(hs):
-    webclient_path = hs.get_config().web_client_location
-    if not webclient_path:
-        try:
-            import syweb
-        except ImportError:
-            quit_with_error(
-                "Could not find a webclient.\n\n"
-                "Please either install the matrix-angular-sdk or configure\n"
-                "the location of the source to serve via the configuration\n"
-                "option `web_client_location`\n\n"
-                "To install the `matrix-angular-sdk` via pip, run:\n\n"
-                "    pip install '%(dep)s'\n"
-                "\n"
-                "You can also disable hosting of the webclient via the\n"
-                "configuration option `web_client`\n"
-                % {"dep": CONDITIONAL_REQUIREMENTS["web_client"].keys()[0]}
-            )
-        syweb_path = os.path.dirname(syweb.__file__)
-        webclient_path = os.path.join(syweb_path, "webclient")
-    # GZip is disabled here due to
-    # https://twistedmatrix.com/trac/ticket/7678
-    # (It can stay enabled for the API resources: they call
-    # write() with the whole body and then finish() straight
-    # after and so do not trigger the bug.
-    # GzipFile was removed in commit 184ba09
-    # return GzipFile(webclient_path)  # TODO configurable?
-    return File(webclient_path)  # TODO configurable?
-
-
 class SynapseHomeServer(HomeServer):
    DATASTORE_CLASS = DataStore
+    _listening_services = []

    def _listener_http(self, config, listener_config):
        port = listener_config["port"]
@@ -119,7 +95,9 @@ class SynapseHomeServer(HomeServer):
        site_tag = listener_config.get("tag", port)

        if tls and config.no_tls:
-            return
+            raise ConfigError(
+                "Listener on port %i has TLS enabled, but no_tls is set" % (port,),
+            )

        resources = {}
        for res in listener_config["resources"]:
@@ -137,15 +115,18 @@ class SynapseHomeServer(HomeServer):
            handler = handler_cls(config, module_api)
            resources[path] = AdditionalResource(self, handler.handle_request)

+        # try to find something useful to redirect '/' to
        if WEB_CLIENT_PREFIX in resources:
            root_resource = RootRedirect(WEB_CLIENT_PREFIX)
+        elif STATIC_PREFIX in resources:
+            root_resource = RootRedirect(STATIC_PREFIX)
        else:
            root_resource = NoResource()

        root_resource = create_resource_tree(resources, root_resource)

        if tls:
-            listen_ssl(
+            return listen_ssl(
                bind_addresses,
                port,
                SynapseSite(
@@ -159,7 +140,7 @@ class SynapseHomeServer(HomeServer):
            )

        else:
-            listen_tcp(
+            return listen_tcp(
                bind_addresses,
                port,
                SynapseSite(
@@ -170,7 +151,6 @@ class SynapseHomeServer(HomeServer):
                    self.version_string,
                )
            )
-        logger.info("Synapse now listening on port %d", port)

    def _configure_named_resource(self, name, compress=False):
        """Build a resource map for a named resource
@@ -195,8 +175,13 @@ class SynapseHomeServer(HomeServer):
                "/_matrix/client/unstable": client_resource,
                "/_matrix/client/v2_alpha": client_resource,
                "/_matrix/client/versions": client_resource,
+                "/.well-known/matrix/client": WellKnownResource(self),
            })

+            if self.get_config().saml2_enabled:
+                from synapse.rest.saml2 import SAML2Resource
+                resources["/_matrix/saml2"] = SAML2Resource(self)
+
        if name == "consent":
            from synapse.rest.consent.consent_resource import ConsentResource
            consent_resource = ConsentResource(self)
@@ -237,7 +222,16 @@ class SynapseHomeServer(HomeServer):
            resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self)

        if name == "webclient":
-            resources[WEB_CLIENT_PREFIX] = build_resource_for_web_client(self)
+            webclient_path = self.get_config().web_client_location
+
+            if webclient_path is None:
+                logger.warning(
+                    "Not enabling webclient resource, as web_client_location is unset."
+                )
+            else:
+                # GZip is disabled here due to
+                # https://twistedmatrix.com/trac/ticket/7678
+                resources[WEB_CLIENT_PREFIX] = File(webclient_path)

        if name == "metrics" and self.get_config().enable_metrics:
            resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
@@ -252,7 +246,9 @@ class SynapseHomeServer(HomeServer):

        for listener in config.listeners:
            if listener["type"] == "http":
-                self._listener_http(config, listener)
+                self._listening_services.extend(
+                    self._listener_http(config, listener)
+                )
            elif listener["type"] == "manhole":
                listen_tcp(
                    listener["bind_addresses"],
@@ -332,24 +328,28 @@ def setup(config_options):
        # generating config files and shouldn't try to continue.
        sys.exit(0)

-    synapse.config.logger.setup_logging(config, use_worker_options=False)
+    sighup_callbacks = []
+    synapse.config.logger.setup_logging(
+        config,
+        use_worker_options=False,
+        register_sighup=sighup_callbacks.append
+    )

-    # check any extra requirements we have now we have a config
-    check_requirements(config)
+    def handle_sighup(*args, **kwargs):
+        for i in sighup_callbacks:
+            i(*args, **kwargs)
+
+    if hasattr(signal, "SIGHUP"):
+        signal.signal(signal.SIGHUP, handle_sighup)

    events.USE_FROZEN_DICTS = config.use_frozen_dicts

-    tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
-
    database_engine = create_engine(config.database_config)
    config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection

    hs = SynapseHomeServer(
        config.server_name,
        db_config=config.database_config,
-        tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
        config=config,
        version_string="Synapse/" + get_version_string(synapse),
        database_engine=database_engine,
@@ -376,12 +376,78 @@ def setup(config_options):
    logger.info("Database prepared in %s.", config.database_config['name'])

    hs.setup()
-    hs.start_listening()

+    def refresh_certificate(*args):
+        """
+        Refresh the TLS certificates that Synapse is using by re-reading them
+        from disk and updating the TLS context factories to use them.
+        """
+        logging.info("Reloading certificate from disk...")
+        hs.config.read_certificate_from_disk()
+        hs.tls_server_context_factory = context_factory.ServerContextFactory(config)
+        hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+            config
+        )
+        logging.info("Certificate reloaded.")
+
+        logging.info("Updating context factories...")
+        for i in hs._listening_services:
+            if isinstance(i.factory, TLSMemoryBIOFactory):
+                i.factory = TLSMemoryBIOFactory(
+                    hs.tls_server_context_factory,
+                    False,
+                    i.factory.wrappedFactory
+                )
+        logging.info("Context factories updated.")
+
+    sighup_callbacks.append(refresh_certificate)
+
+    @defer.inlineCallbacks
    def start():
-        hs.get_pusherpool().start()
-        hs.get_datastore().start_profiling()
-        hs.get_datastore().start_doing_background_updates()
+        try:
+            # Check if the certificate is still valid.
+            cert_days_remaining = hs.config.is_disk_cert_valid()
+
+            if hs.config.acme_enabled:
+                # If ACME is enabled, we might need to provision a certificate
+                # before starting.
+                acme = hs.get_acme_handler()
+
+                # Start up the webservices which we will respond to ACME
+                # challenges with.
+                yield acme.start_listening()
+
+                # We want to reprovision if cert_days_remaining is None (meaning no
+                # certificate exists), or the days remaining number it returns
+                # is less than our re-registration threshold.
+                if (cert_days_remaining is None) or (
+                    not cert_days_remaining > hs.config.acme_reprovision_threshold
+                ):
+                    yield acme.provision_certificate()
+
+            # Read the certificate from disk and build the context factories for
+            # TLS.
+            hs.config.read_certificate_from_disk()
+            hs.tls_server_context_factory = context_factory.ServerContextFactory(config)
+            hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+                config
+            )
+
+            # It is now safe to start your Synapse.
+            hs.start_listening()
+            hs.get_pusherpool().start()
+            hs.get_datastore().start_profiling()
+            hs.get_datastore().start_doing_background_updates()
+        except Exception as e:
+            # If a DeferredList failed (like in listening on the ACME listener),
+            # we need to print the subfailure explicitly.
+            if isinstance(e, defer.FirstError):
+                e.subFailure.printTraceback(sys.stderr)
+                sys.exit(1)
+
+            # Something else went wrong when starting. Print it and bail out.
+            traceback.print_exc(file=sys.stderr)
+            sys.exit(1)

    reactor.callWhenRunning(start)

@@ -549,7 +615,7 @@ def run(hs):
        )

    start_generate_monthly_active_users()
-    if hs.config.limit_usage_by_mau:
+    if hs.config.limit_usage_by_mau or hs.config.mau_stats_only:
        clock.looping_call(start_generate_monthly_active_users, 5 * 60 * 1000)
    # End of monthly active user settings

--- a/synapse/app/media_repository.py
+++ b/synapse/app/media_repository.py
@@ -151,23 +151,23 @@ def start(config_options):

    database_engine = create_engine(config.database_config)

-    tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
-
    ss = MediaRepositoryServer(
        config.server_name,
        db_config=config.database_config,
-        tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
        config=config,
        version_string="Synapse/" + get_version_string(synapse),
        database_engine=database_engine,
    )

    ss.setup()
-    ss.start_listening(config.worker_listeners)

    def start():
+        ss.config.read_certificate_from_disk()
+        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
+        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+            config
+        )
+        ss.start_listening(config.worker_listeners)
        ss.get_datastore().start_profiling()

    reactor.callWhenRunning(start)
--- a/synapse/app/user_dir.py
+++ b/synapse/app/user_dir.py
@@ -211,24 +211,24 @@ def start(config_options):
    # Force the pushers to start since they will be disabled in the main config
    config.update_user_directory = True

-    tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
-
-    ps = UserDirectoryServer(
+    ss = UserDirectoryServer(
        config.server_name,
        db_config=config.database_config,
-        tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
        config=config,
        version_string="Synapse/" + get_version_string(synapse),
        database_engine=database_engine,
    )

-    ps.setup()
-    ps.start_listening(config.worker_listeners)
+    ss.setup()

    def start():
-        ps.get_datastore().start_profiling()
+        ss.config.read_certificate_from_disk()
+        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
+        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+            config
+        )
+        ss.start_listening(config.worker_listeners)
+        ss.get_datastore().start_profiling()

    reactor.callWhenRunning(start)

--- a/synapse/appservice/scheduler.py
+++ b/synapse/appservice/scheduler.py
@@ -53,8 +53,8 @@ import logging
 from twisted.internet import defer

 from synapse.appservice import ApplicationServiceState
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.logcontext import run_in_background
-from synapse.util.metrics import Measure

 logger = logging.getLogger(__name__)

@@ -104,14 +104,23 @@ class _ServiceQueuer(object):
        self.clock = clock

    def enqueue(self, service, event):
-        # if this service isn't being sent something
        self.queued_events.setdefault(service.id, []).append(event)
-        run_in_background(self._send_request, service)
+
+        # start a sender for this appservice if we don't already have one
+
+        if service.id in self.requests_in_flight:
+            return
+
+        run_as_background_process(
+            "as-sender-%s" % (service.id, ),
+            self._send_request, service,
+        )

    @defer.inlineCallbacks
    def _send_request(self, service):
-        if service.id in self.requests_in_flight:
-            return
+        # sanity-check: we shouldn't get here if this service already has a sender
+        # running.
+        assert(service.id not in self.requests_in_flight)

        self.requests_in_flight.add(service.id)
        try:
@@ -119,12 +128,10 @@ class _ServiceQueuer(object):
                events = self.queued_events.pop(service.id, [])
                if not events:
                    return
-
-                with Measure(self.clock, "servicequeuer.send"):
-                    try:
-                        yield self.txn_ctrl.send(service, events)
-                    except Exception:
-                        logger.exception("AS request failed")
+                try:
+                    yield self.txn_ctrl.send(service, events)
+                except Exception:
+                    logger.exception("AS request failed")
        finally:
            self.requests_in_flight.discard(service.id)

@@ -223,7 +230,12 @@ class _Recoverer(object):
        self.backoff_counter = 1

    def recover(self):
-        self.clock.call_later((2 ** self.backoff_counter), self.retry)
+        def _retry():
+            run_as_background_process(
+                "as-recoverer-%s" % (self.service.id,),
+                self.retry,
+            )
+        self.clock.call_later((2 ** self.backoff_counter), _retry)

    def _backoff(self):
        # cap the backoff to be around 8.5min => (2^9) = 512 secs
--- a/synapse/config/main.py
+++ b/synapse/config/main.py
@@ -16,7 +16,7 @@ from synapse.config._base import ConfigError

 if __name__ == "__main__":
    import sys
-    from homeserver import HomeServerConfig
+    from synapse.config.homeserver import HomeServerConfig

    action = sys.argv[1]

--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -134,10 +134,6 @@ class Config(object):
        with open(file_path) as file_stream:
            return file_stream.read()

-    @staticmethod
-    def default_path(name):
-        return os.path.abspath(os.path.join(os.path.curdir, name))
-
    @staticmethod
    def read_config_file(file_path):
        with open(file_path) as file_stream:
@@ -151,8 +147,39 @@ class Config(object):
        return results

    def generate_config(
-        self, config_dir_path, server_name, is_generating_file, report_stats=None
+        self,
+        config_dir_path,
+        data_dir_path,
+        server_name,
+        generate_secrets=False,
+        report_stats=None,
    ):
+        """Build a default configuration file
+
+        This is used both when the user explicitly asks us to generate a config file
+        (eg with --generate_config), and before loading the config at runtime (to give
+        a base which the config files override)
+
+        Args:
+            config_dir_path (str): The path where the config files are kept. Used to
+                create filenames for things like the log config and the signing key.
+
+            data_dir_path (str): The path where the data files are kept. Used to create
+                filenames for things like the database and media store.
+
+            server_name (str): The server name. Used to initialise the server_name
+                config param, but also used in the names of some of the config files.
+
+            generate_secrets (bool): True if we should generate new secrets for things
+                like the macaroon_secret_key. If False, these parameters will be left
+                unset.
+
+            report_stats (bool|None): Initial setting for the report_stats setting.
+                If None, report_stats will be left unset.
+
+        Returns:
+            str: the yaml config file
+        """
        default_config = "# vim:ft=yaml\n"

        default_config += "\n\n".join(
@@ -160,15 +187,14 @@ class Config(object):
            for conf in self.invoke_all(
                "default_config",
                config_dir_path=config_dir_path,
+                data_dir_path=data_dir_path,
                server_name=server_name,
-                is_generating_file=is_generating_file,
+                generate_secrets=generate_secrets,
                report_stats=report_stats,
            )
        )

-        config = yaml.load(default_config)
-
-        return default_config, config
+        return default_config

    @classmethod
    def load_config(cls, description, argv):
@@ -274,12 +300,14 @@ class Config(object):
                if not cls.path_exists(config_dir_path):
                    os.makedirs(config_dir_path)
                with open(config_path, "w") as config_file:
-                    config_str, config = obj.generate_config(
+                    config_str = obj.generate_config(
                        config_dir_path=config_dir_path,
+                        data_dir_path=os.getcwd(),
                        server_name=server_name,
                        report_stats=(config_args.report_stats == "yes"),
-                        is_generating_file=True,
+                        generate_secrets=True,
                    )
+                    config = yaml.load(config_str)
                    obj.invoke_all("generate_files", config)
                    config_file.write(config_str)
                print(
@@ -339,7 +367,7 @@ class Config(object):
        if not keys_directory:
            keys_directory = os.path.dirname(config_files[-1])

-        config_dir_path = os.path.abspath(keys_directory)
+        self.config_dir_path = os.path.abspath(keys_directory)

        specified_config = {}
        for config_file in config_files:
@@ -350,11 +378,13 @@ class Config(object):
            raise ConfigError(MISSING_SERVER_NAME)

        server_name = specified_config["server_name"]
-        _, config = self.generate_config(
-            config_dir_path=config_dir_path,
+        config_string = self.generate_config(
+            config_dir_path=self.config_dir_path,
+            data_dir_path=os.getcwd(),
            server_name=server_name,
-            is_generating_file=False,
+            generate_secrets=False,
        )
+        config = yaml.load(config_string)
        config.pop("log_config")
        config.update(specified_config)

--- a/synapse/config/api.py
+++ b/synapse/config/api.py
@@ -24,6 +24,7 @@ class ApiConfig(Config):
            EventTypes.JoinRules,
            EventTypes.CanonicalAlias,
            EventTypes.RoomAvatar,
+            EventTypes.RoomEncryption,
            EventTypes.Name,
        ])

@@ -36,5 +37,6 @@ class ApiConfig(Config):
            - "{JoinRules}"
            - "{CanonicalAlias}"
            - "{RoomAvatar}"
+            - "{RoomEncryption}"
            - "{Name}"
        """.format(**vars(EventTypes))
--- a/synapse/config/appservice.py
+++ b/synapse/config/appservice.py
@@ -33,11 +33,16 @@ class AppServiceConfig(Config):
    def read_config(self, config):
        self.app_service_config_files = config.get("app_service_config_files", [])
        self.notify_appservices = config.get("notify_appservices", True)
+        self.track_appservice_user_ips = config.get("track_appservice_user_ips", False)

    def default_config(cls, **kwargs):
        return """\
        # A list of application service config file to use
        app_service_config_files: []
+
+        # Whether or not to track application service IP addresses. Implicitly
+        # enables MAU tracking for application service users.
+        track_appservice_user_ips: False
        """


--- a/synapse/config/consent_config.py
+++ b/synapse/config/consent_config.py
@@ -13,6 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+from os import path
+
+from synapse.config import ConfigError
+
 from ._base import Config

 DEFAULT_CONFIG = """\
@@ -85,7 +89,15 @@ class ConsentConfig(Config):
        if consent_config is None:
            return
        self.user_consent_version = str(consent_config["version"])
-        self.user_consent_template_dir = consent_config["template_dir"]
+        self.user_consent_template_dir = self.abspath(
+            consent_config["template_dir"]
+        )
+        if not path.isdir(self.user_consent_template_dir):
+            raise ConfigError(
+                "Could not find template directory '%s'" % (
+                    self.user_consent_template_dir,
+                ),
+            )
        self.user_consent_server_notice_content = consent_config.get(
            "server_notice_content",
        )
--- a/synapse/config/database.py
+++ b/synapse/config/database.py
@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os

 from ._base import Config

@@ -45,8 +46,8 @@ class DatabaseConfig(Config):

        self.set_databasepath(config.get("database_path"))

-    def default_config(self, **kwargs):
-        database_path = self.abspath("homeserver.db")
+    def default_config(self, data_dir_path, **kwargs):
+        database_path = os.path.join(data_dir_path, "homeserver.db")
        return """\
        # Database configuration
        database:
--- a/synapse/config/homeserver.py
+++ b/synapse/config/homeserver.py
@@ -32,7 +32,7 @@ from .ratelimiting import RatelimitConfig
 from .registration import RegistrationConfig
 from .repository import ContentRepositoryConfig
 from .room_directory import RoomDirectoryConfig
-from .saml2 import SAML2Config
+from .saml2_config import SAML2Config
 from .server import ServerConfig
 from .server_notices_config import ServerNoticesConfig
 from .spam_checker import SpamCheckerConfig
@@ -53,10 +53,3 @@ class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig,
                       ServerNoticesConfig, RoomDirectoryConfig,
                       ):
    pass
-
-
-if __name__ == '__main__':
-    import sys
-    sys.stdout.write(
-        HomeServerConfig().generate_config(sys.argv[1], sys.argv[2], True)[0]
-    )
--- a/synapse/config/key.py
+++ b/synapse/config/key.py
@@ -57,8 +57,8 @@ class KeyConfig(Config):
            # Unfortunately, there are people out there that don't have this
            # set. Lets just be "nice" and derive one from their secret key.
            logger.warn("Config is missing missing macaroon_secret_key")
-            seed = self.signing_key[0].seed
-            self.macaroon_secret_key = hashlib.sha256(seed)
+            seed = bytes(self.signing_key[0])
+            self.macaroon_secret_key = hashlib.sha256(seed).digest()

        self.expire_access_token = config.get("expire_access_token", False)

@@ -66,26 +66,32 @@ class KeyConfig(Config):
        # falsification of values
        self.form_secret = config.get("form_secret", None)

-    def default_config(self, config_dir_path, server_name, is_generating_file=False,
+    def default_config(self, config_dir_path, server_name, generate_secrets=False,
                       **kwargs):
        base_key_name = os.path.join(config_dir_path, server_name)

-        if is_generating_file:
-            macaroon_secret_key = random_string_with_symbols(50)
-            form_secret = '"%s"' % random_string_with_symbols(50)
+        if generate_secrets:
+            macaroon_secret_key = 'macaroon_secret_key: "%s"' % (
+                random_string_with_symbols(50),
+            )
+            form_secret = 'form_secret: "%s"' % random_string_with_symbols(50)
        else:
-            macaroon_secret_key = None
-            form_secret = 'null'
+            macaroon_secret_key = "# macaroon_secret_key: <PRIVATE STRING>"
+            form_secret = "# form_secret: <PRIVATE STRING>"

        return """\
-        macaroon_secret_key: "%(macaroon_secret_key)s"
+        # a secret which is used to sign access tokens. If none is specified,
+        # the registration_shared_secret is used, if one is given; otherwise,
+        # a secret key is derived from the signing key.
+        %(macaroon_secret_key)s

        # Used to enable access token expiration.
        expire_access_token: False

        # a secret which is used to calculate HMACs for form values, to stop
-        # falsification of values
-        form_secret: %(form_secret)s
+        # falsification of values. Must be specified for the User Consent
+        # forms to work.
+        %(form_secret)s

        ## Signing Keys ##

--- a/synapse/config/logger.py
+++ b/synapse/config/logger.py
@@ -80,9 +80,7 @@ class LoggingConfig(Config):
        self.log_file = self.abspath(config.get("log_file"))

    def default_config(self, config_dir_path, server_name, **kwargs):
-        log_config = self.abspath(
-            os.path.join(config_dir_path, server_name + ".log.config")
-        )
+        log_config = os.path.join(config_dir_path, server_name + ".log.config")
        return """
        # A yaml python logging config file
        log_config: "%(log_config)s"
@@ -129,7 +127,7 @@ class LoggingConfig(Config):
                )


-def setup_logging(config, use_worker_options=False):
+def setup_logging(config, use_worker_options=False, register_sighup=None):
    """ Set up python logging

    Args:
@@ -138,7 +136,16 @@ def setup_logging(config, use_worker_options=False):

        use_worker_options (bool): True to use 'worker_log_config' and
            'worker_log_file' options instead of 'log_config' and 'log_file'.
+
+        register_sighup (func | None): Function to call to register a
+            sighup handler.
    """
+    if not register_sighup:
+        if getattr(signal, "SIGHUP"):
+            register_sighup = lambda x: signal.signal(signal.SIGHUP, x)
+        else:
+            register_sighup = lambda x: None
+
    log_config = (config.worker_log_config if use_worker_options
                  else config.log_config)
    log_file = (config.worker_log_file if use_worker_options
@@ -200,13 +207,7 @@ def setup_logging(config, use_worker_options=False):

        load_log_config()

-    # TODO(paul): obviously this is a terrible mechanism for
-    #   stealing SIGHUP, because it means no other part of synapse
-    #   can use it instead. If we want to catch SIGHUP anywhere
-    #   else as well, I'd suggest we find a nicer way to broadcast
-    #   it around.
-    if getattr(signal, "SIGHUP"):
-        signal.signal(signal.SIGHUP, sighup)
+    register_sighup(sighup)

    # make sure that the first thing we log is a thing we can grep backwards
    # for
--- a/Show More
+++ b/Show More
				`@@ -1 +0,0 @@`
				`Add option to track MAU stats (but not limit people)`
				`@@ -0,0 +1 @@`
				`Include m.room.encryption on invites by default`
				`@@ -1 +0,0 @@`
				`The media repository now no longer fails to decode UTF-8 filenames when downloading remote media.`
				`@@ -1 +0,0 @@`
				`A coveragerc file, as well as the py36-coverage tox target, have been added.`
				`@@ -1 +0,0 @@`
				`Add a GitHub pull request template and add multiple issue templates`
				`@@ -1 +0,0 @@`
				URL previews now correctly decode non-UTF-8 text if the header contains a `<meta http-equiv="Content-Type"` header.
				`@@ -1 +0,0 @@`
				`Update README to reflect the fact that #1491 is fixed`
				`@@ -1 +0,0 @@`
				`Fix an issue where public consent URLs had two slashes.`
				`@@ -1 +0,0 @@`
				Add missing `jpeg` package prerequisite for OpenBSD in README.
				`@@ -1 +0,0 @@`
				`Fallback auth now accepts the session parameter on Python 3.`