Merge branch 'develop' into bwindels/registerasregularuser

.circleci/config.yml

@@ -4,125 +4,112 @@ jobs:
     machine: true
     steps:
       - checkout
-      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_TAG} .
-      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 .
+      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:$CIRCLE_TAG .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
-      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
+      - run: docker push matrixdotorg/synapse:$CIRCLE_TAG
   dockerhubuploadlatest:
     machine: true
     steps:
       - checkout
-      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_SHA1} .
-      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_SHA1}-py3 --build-arg PYTHON_VERSION=3.6 .
+      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:$CIRCLE_SHA1 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      - run: docker tag matrixdotorg/synapse:${CIRCLE_SHA1} matrixdotorg/synapse:latest
-      - run: docker tag matrixdotorg/synapse:${CIRCLE_SHA1}-py3 matrixdotorg/synapse:latest-py3
-      - run: docker push matrixdotorg/synapse:${CIRCLE_SHA1}
-      - run: docker push matrixdotorg/synapse:${CIRCLE_SHA1}-py3
+      - run: docker tag matrixdotorg/synapse:$CIRCLE_SHA1 matrixdotorg/synapse:latest
+      - run: docker push matrixdotorg/synapse:$CIRCLE_SHA1
       - run: docker push matrixdotorg/synapse:latest
-      - run: docker push matrixdotorg/synapse:latest-py3
   sytestpy2:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy2
-    working_directory: /src
+    machine: true
     steps:
       - checkout
-      - run: /synapse_sytest.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy2
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2
       - store_artifacts:
-          path: /logs
+          path: ~/project/logs
           destination: logs
       - store_test_results:
-          path: /logs
+          path: logs
   sytestpy2postgres:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy2
-    working_directory: /src
+    machine: true
     steps:
       - checkout
-      - run: POSTGRES=1 /synapse_sytest.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy2
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2
       - store_artifacts:
-          path: /logs
+          path: ~/project/logs
           destination: logs
       - store_test_results:
-          path: /logs
+          path: logs
   sytestpy2merged:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy2
-    working_directory: /src
+    machine: true
     steps:
       - checkout
       - run: bash .circleci/merge_base_branch.sh
-      - run: /synapse_sytest.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy2
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2
       - store_artifacts:
-          path: /logs
+          path: ~/project/logs
           destination: logs
       - store_test_results:
-          path: /logs
+          path: logs
+
   sytestpy2postgresmerged:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy2
-    working_directory: /src
+    machine: true
     steps:
       - checkout
       - run: bash .circleci/merge_base_branch.sh
-      - run: POSTGRES=1 /synapse_sytest.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy2
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2
       - store_artifacts:
-          path: /logs
+          path: ~/project/logs
           destination: logs
       - store_test_results:
-          path: /logs
+          path: logs
 
   sytestpy3:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy3
-    working_directory: /src
+    machine: true
     steps:
       - checkout
-      - run: /synapse_sytest.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
       - store_artifacts:
-          path: /logs
+          path: ~/project/logs
           destination: logs
       - store_test_results:
-          path: /logs
+          path: logs
   sytestpy3postgres:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy3
-    working_directory: /src
+    machine: true
     steps:
       - checkout
-      - run: POSTGRES=1 /synapse_sytest.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3
       - store_artifacts:
-          path: /logs
+          path: ~/project/logs
           destination: logs
       - store_test_results:
-          path: /logs
+          path: logs
   sytestpy3merged:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy3
-    working_directory: /src
+    machine: true
     steps:
       - checkout
       - run: bash .circleci/merge_base_branch.sh
-      - run: /synapse_sytest.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
       - store_artifacts:
-          path: /logs
+          path: ~/project/logs
           destination: logs
       - store_test_results:
-          path: /logs
+          path: logs
   sytestpy3postgresmerged:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy3
-    working_directory: /src
+    machine: true
     steps:
       - checkout
       - run: bash .circleci/merge_base_branch.sh
-      - run: POSTGRES=1 /synapse_sytest.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3
       - store_artifacts:
-          path: /logs
+          path: ~/project/logs
           destination: logs
       - store_test_results:
-          path: /logs
+          path: logs
 
 workflows:
   version: 2
@@ -163,7 +150,7 @@ workflows:
       - dockerhubuploadrelease:
           filters:
             tags:
-              only: /v[0-9].[0-9]+.[0-9]+.*/
+              only: /^v[0-9].[0-9]+.[0-9]+(.[0-9]+)?/
             branches:
               ignore: /.*/
       - dockerhubuploadlatest:

.circleci/merge_base_branch.sh

@@ -9,16 +9,13 @@ source $BASH_ENV
 
 if [[ -z "${CIRCLE_PR_NUMBER}" ]]
 then
-    echo "Can't figure out what the PR number is! Assuming merge target is develop."
-
-    # It probably hasn't had a PR opened yet. Since all PRs land on develop, we
-    # can probably assume it's based on it and will be merged into it.
-    GITBASE="develop"
-else
-    # Get the reference, using the GitHub API
-    GITBASE=`wget -O- https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
+    echo "Can't figure out what the PR number is!"
+    exit 1
 fi
 
+# Get the reference, using the GitHub API
+GITBASE=`curl -q https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
+
 # Show what we are before
 git show -s
 
@@ -31,4 +28,4 @@ git fetch -u origin $GITBASE
 git merge --no-edit origin/$GITBASE
 
 # Show what we are after.
-git show -s
+git show -s

.coveragerc

@@ -1,12 +0,0 @@
-[run]
-branch = True
-parallel = True
-source = synapse
-
-[paths]
-source=
-   coverage
-
-[report]
-precision = 2
-ignore_errors = True

.github/ISSUE_TEMPLATE.md

@@ -1,9 +1,3 @@
----
-name: Bug report
-about: Create a report to help us improve
-
----
-
 <!-- 
 
 **IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**: 
@@ -17,50 +11,38 @@ the necessary data to fix your issue.
 You can also preview your report before submitting it. You may remove sections
 that aren't relevant to your particular case.
 
-Text between <!-- and --> marks will be invisible in the report.
+Text between <!-- and --​> marks will be invisible in the report.
 
 -->
 
 ### Description
 
-<!-- Describe here the problem that you are experiencing -->
+Describe here the problem that you are experiencing, or the feature you are requesting.
 
 ### Steps to reproduce
 
-- list the steps
+- For bugs, list the steps
 - that reproduce the bug
 - using hyphens as bullet points
 
-<!-- 
 Describe how what happens differs from what you expected.
 
-If you can identify any relevant log snippets from _homeserver.log_, please include
+<!-- If you can identify any relevant log snippets from _homeserver.log_, please include
 those (please be careful to remove any personal or private data). Please surround them with
-``` (three backticks, on a line on their own), so that they are formatted legibly.
--->
+``` (three backticks, on a line on their own), so that they are formatted legibly. -->
 
 ### Version information
 
 <!-- IMPORTANT: please answer the following questions, to help us narrow down the problem -->
 
-<!-- Was this issue identified on matrix.org or another homeserver? -->
-- **Homeserver**: 
+- **Homeserver**: Was this issue identified on matrix.org or another homeserver?
 
 If not matrix.org:
-
-<!-- 
-What version of Synapse is running? 
+- **Version**:        What version of Synapse is running? <!-- 
 You can find the Synapse version by inspecting the server headers (replace matrix.org with
 your own homeserver domain):
 $ curl -v https://matrix.org/_matrix/client/versions 2>&1 | grep "Server:"
 -->
-- **Version**: 
-
-- **Install method**: 
-<!-- examples: package manager/git clone/pip  -->
-
-- **Platform**: 
-<!--
-Tell us about the environment in which your homeserver is operating
-distro, hardware, if it's running in a vm/container, etc.
--->
+- **Install method**: package manager/git clone/pip      
+- **Platform**:       Tell us about the environment in which your homeserver is operating
+                      - distro, hardware, if it's running in a vm/container, etc.

.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md

@@ -1,9 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-
----
-
-**Description:**
-
-<!-- Describe here the feature you are requesting. -->

.github/ISSUE_TEMPLATE/SUPPORT_REQUEST.md

@@ -1,9 +0,0 @@
----
-name: Support request
-about: I need support for Synapse
-
----
-
-# Please ask for support in [**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org)
-
-## Don't file an issue as a support request.

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +0,0 @@
-### Pull Request Checklist
-
-<!-- Please read CONTRIBUTING.rst before submitting your pull request -->
-
-* [ ] Pull request is based on the develop branch
-* [ ] Pull request includes a [changelog file](CONTRIBUTING.rst#changelog)
-* [ ] Pull request includes a [sign off](CONTRIBUTING.rst#sign-off)

.github/SUPPORT.md

@@ -1,3 +0,0 @@
-[**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org) is the official support room for Matrix, and can be accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html 
-
-It can also be access via IRC bridge at irc://irc.freenode.net/matrix or on the web here: https://webchat.freenode.net/?channels=matrix

.travis.yml

@@ -1,45 +1,27 @@
 sudo: false
 language: python
 
-cache:
-  directories:
-    # we only bother to cache the wheels; parts of the http cache get
-    # invalidated every build (because they get served with a max-age of 600
-    # seconds), which means that we end up re-uploading the whole cache for
-    # every build, which is time-consuming In any case, it's not obvious that
-    # downloading the cache from S3 would be much faster than downloading the
-    # originals from pypi.
-    #
-    - $HOME/.cache/pip/wheels
+# tell travis to cache ~/.cache/pip
+cache: pip
 
-# don't clone the whole repo history, one commit will do
-git:
-  depth: 1
+before_script:
+  - git remote set-branches --add origin develop
+  - git fetch origin develop
 
-# only build branches we care about (PRs are built seperately)
-branches:
-  only:
-    - master
-    - develop
-    - /^release-v/
-
-# When running the tox environments that call Twisted Trial, we can pass the -j
-# flag to run the tests concurrently. We set this to 2 for CPU bound tests
-# (SQLite) and 4 for I/O bound tests (PostgreSQL).
 matrix:
   fast_finish: true
   include:
   - python: 2.7
     env: TOX_ENV=packaging
 
-  - python: 3.6
-    env: TOX_ENV="pep8,check_isort"
+  - python: 2.7
+    env: TOX_ENV=pep8
 
   - python: 2.7
-    env: TOX_ENV=py27 TRIAL_FLAGS="-j 2"
+    env: TOX_ENV=py27
 
   - python: 2.7
-    env: TOX_ENV=py27-old TRIAL_FLAGS="-j 2"
+    env: TOX_ENV=py27-old
 
   - python: 2.7
     env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
@@ -47,24 +29,21 @@ matrix:
       - postgresql
 
   - python: 3.5
-    env: TOX_ENV=py35 TRIAL_FLAGS="-j 2"
+    env: TOX_ENV=py35
 
   - python: 3.6
-    env: TOX_ENV=py36 TRIAL_FLAGS="-j 2"
+    env: TOX_ENV=py36
 
   - python: 3.6
     env: TOX_ENV=py36-postgres TRIAL_FLAGS="-j 4"
     services:
       - postgresql
 
-  - # we only need to check for the newsfragment if it's a PR build
-    if: type = pull_request
-    python: 3.6
+  - python: 3.6
+    env: TOX_ENV=check_isort
+
+  - python: 3.6
     env: TOX_ENV=check-newsfragment
-    script:
-      - git remote set-branches --add origin develop
-      - git fetch origin develop
-      - tox -e $TOX_ENV
 
 install:
   - pip install tox

CHANGES.md

@@ -1,270 +1,11 @@
-Synapse 0.33.9 (2018-11-19)
-===========================
-
-No significant changes.
-
-
-Synapse 0.33.9rc1 (2018-11-14)
-==============================
-
-Features
---------
-
-- Include flags to optionally add `m.login.terms` to the registration flow when consent tracking is enabled. ([\#4004](https://github.com/matrix-org/synapse/issues/4004), [\#4133](https://github.com/matrix-org/synapse/issues/4133), [\#4142](https://github.com/matrix-org/synapse/issues/4142), [\#4184](https://github.com/matrix-org/synapse/issues/4184))
-- Support for replacing rooms with new ones ([\#4091](https://github.com/matrix-org/synapse/issues/4091), [\#4099](https://github.com/matrix-org/synapse/issues/4099), [\#4100](https://github.com/matrix-org/synapse/issues/4100), [\#4101](https://github.com/matrix-org/synapse/issues/4101))
-
-
-Bugfixes
---------
-
-- Fix exceptions when using the email mailer on Python 3. ([\#4095](https://github.com/matrix-org/synapse/issues/4095))
-- Fix e2e key backup with more than 9 backup versions ([\#4113](https://github.com/matrix-org/synapse/issues/4113))
-- Searches that request profile info now no longer fail with a 500. ([\#4122](https://github.com/matrix-org/synapse/issues/4122))
-- fix return code of empty key backups ([\#4123](https://github.com/matrix-org/synapse/issues/4123))
-- If the typing stream ID goes backwards (as on a worker when the master restarts), the worker's typing handler will no longer erroneously report rooms containing new typing events. ([\#4127](https://github.com/matrix-org/synapse/issues/4127))
-- Fix table lock of device_lists_remote_cache which could freeze the application ([\#4132](https://github.com/matrix-org/synapse/issues/4132))
-- Fix exception when using state res v2 algorithm ([\#4135](https://github.com/matrix-org/synapse/issues/4135))
-- Generating the user consent URI no longer fails on Python 3. ([\#4140](https://github.com/matrix-org/synapse/issues/4140), [\#4163](https://github.com/matrix-org/synapse/issues/4163))
-- Loading URL previews from the DB cache on Postgres will no longer cause Unicode type errors when responding to the request, and URL previews will no longer fail if the remote server returns a Content-Type header with the chartype in quotes. ([\#4157](https://github.com/matrix-org/synapse/issues/4157))
-- The hash_password script now works on Python 3. ([\#4161](https://github.com/matrix-org/synapse/issues/4161))
-- Fix noop checks when updating device keys, reducing spurious device list update notifications. ([\#4164](https://github.com/matrix-org/synapse/issues/4164))
-
-
-Deprecations and Removals
--------------------------
-
-- The disused and un-specced identicon generator has been removed. ([\#4106](https://github.com/matrix-org/synapse/issues/4106))
-- The obsolete and non-functional /pull federation endpoint has been removed. ([\#4118](https://github.com/matrix-org/synapse/issues/4118))
-- The deprecated v1 key exchange endpoints have been removed. ([\#4119](https://github.com/matrix-org/synapse/issues/4119))
-- Synapse will no longer fetch keys using the fallback deprecated v1 key exchange method and will now always use v2. ([\#4120](https://github.com/matrix-org/synapse/issues/4120))
-
-
-Internal Changes
-----------------
-
-- Fix build of Docker image with docker-compose ([\#3778](https://github.com/matrix-org/synapse/issues/3778))
-- Delete unreferenced state groups during history purge ([\#4006](https://github.com/matrix-org/synapse/issues/4006))
-- The "Received rdata" log messages on workers is now logged at DEBUG, not INFO. ([\#4108](https://github.com/matrix-org/synapse/issues/4108))
-- Reduce replication traffic for device lists ([\#4109](https://github.com/matrix-org/synapse/issues/4109))
-- Fix `synapse_replication_tcp_protocol_*_commands` metric label to be full command name, rather than just the first character ([\#4110](https://github.com/matrix-org/synapse/issues/4110))
-- Log some bits about room creation ([\#4121](https://github.com/matrix-org/synapse/issues/4121))
-- Fix `tox` failure on old systems ([\#4124](https://github.com/matrix-org/synapse/issues/4124))
-- Add STATE_V2_TEST room version ([\#4128](https://github.com/matrix-org/synapse/issues/4128))
-- Clean up event accesses and tests ([\#4137](https://github.com/matrix-org/synapse/issues/4137))
-- The default logging config will now set an explicit log file encoding of UTF-8. ([\#4138](https://github.com/matrix-org/synapse/issues/4138))
-- Add helpers functions for getting prev and auth events of an event ([\#4139](https://github.com/matrix-org/synapse/issues/4139))
-- Add some tests for the HTTP pusher. ([\#4149](https://github.com/matrix-org/synapse/issues/4149))
-- add purge_history.sh and purge_remote_media.sh scripts to contrib/ ([\#4155](https://github.com/matrix-org/synapse/issues/4155))
-- HTTP tests have been refactored to contain less boilerplate. ([\#4156](https://github.com/matrix-org/synapse/issues/4156))
-- Drop incoming events from federation for unknown rooms ([\#4165](https://github.com/matrix-org/synapse/issues/4165))
-
-
-Synapse 0.33.8 (2018-11-01)
-===========================
-
-No significant changes.
-
-
-Synapse 0.33.8rc2 (2018-10-31)
-==============================
-
-Bugfixes
---------
-
-- Searches that request profile info now no longer fail with a 500. Fixes 
-  a regression in 0.33.8rc1. ([\#4122](https://github.com/matrix-org/synapse/issues/4122))
-
-
-Synapse 0.33.8rc1 (2018-10-29)
-==============================
-
-Features
---------
-
-- Servers with auto-join rooms will now automatically create those rooms when the first user registers ([\#3975](https://github.com/matrix-org/synapse/issues/3975))
-- Add config option to control alias creation ([\#4051](https://github.com/matrix-org/synapse/issues/4051))
-- The register_new_matrix_user script is now ported to Python 3. ([\#4085](https://github.com/matrix-org/synapse/issues/4085))
-- Configure Docker image to listen on both ipv4 and ipv6. ([\#4089](https://github.com/matrix-org/synapse/issues/4089))
-
-
-Bugfixes
---------
-
-- Fix HTTP error response codes for federated group requests. ([\#3969](https://github.com/matrix-org/synapse/issues/3969))
-- Fix issue where Python 3 users couldn't paginate /publicRooms ([\#4046](https://github.com/matrix-org/synapse/issues/4046))
-- Fix URL previewing to work in Python 3.7 ([\#4050](https://github.com/matrix-org/synapse/issues/4050))
-- synctl will use the right python executable to run worker processes ([\#4057](https://github.com/matrix-org/synapse/issues/4057))
-- Manhole now works again on Python 3, instead of failing with a "couldn't match all kex parts" when connecting. ([\#4060](https://github.com/matrix-org/synapse/issues/4060), [\#4067](https://github.com/matrix-org/synapse/issues/4067))
-- Fix some metrics being racy and causing exceptions when polled by Prometheus. ([\#4061](https://github.com/matrix-org/synapse/issues/4061))
-- Fix bug which prevented email notifications from being sent unless an absolute path was given for `email_templates`. ([\#4068](https://github.com/matrix-org/synapse/issues/4068))
-- Correctly account for cpu usage by background threads ([\#4074](https://github.com/matrix-org/synapse/issues/4074))
-- Fix race condition where config defined reserved users were not being added to
-  the monthly active user list prior to the homeserver reactor firing up ([\#4081](https://github.com/matrix-org/synapse/issues/4081))
-- Fix bug which prevented backslashes being used in event field filters ([\#4083](https://github.com/matrix-org/synapse/issues/4083))
-
-
-Internal Changes
-----------------
-
-- Add information about the [matrix-docker-ansible-deploy](https://github.com/spantaleev/matrix-docker-ansible-deploy) playbook ([\#3698](https://github.com/matrix-org/synapse/issues/3698))
-- Add initial implementation of new state resolution algorithm ([\#3786](https://github.com/matrix-org/synapse/issues/3786))
-- Reduce database load when fetching state groups ([\#4011](https://github.com/matrix-org/synapse/issues/4011))
-- Various cleanups in the federation client code ([\#4031](https://github.com/matrix-org/synapse/issues/4031))
-- Run the CircleCI builds in docker containers ([\#4041](https://github.com/matrix-org/synapse/issues/4041))
-- Only colourise synctl output when attached to tty ([\#4049](https://github.com/matrix-org/synapse/issues/4049))
-- Refactor room alias creation code ([\#4063](https://github.com/matrix-org/synapse/issues/4063))
-- Make the Python scripts in the top-level scripts folders meet pep8 and pass flake8. ([\#4068](https://github.com/matrix-org/synapse/issues/4068))
-- The README now contains example for the Caddy web server. Contributed by steamp0rt. ([\#4072](https://github.com/matrix-org/synapse/issues/4072))
-- Add psutil as an explicit dependency ([\#4073](https://github.com/matrix-org/synapse/issues/4073))
-- Clean up threading and logcontexts in pushers ([\#4075](https://github.com/matrix-org/synapse/issues/4075))
-- Correctly manage logcontexts during startup to fix some "Unexpected logging context" warnings ([\#4076](https://github.com/matrix-org/synapse/issues/4076))
-- Give some more things logcontexts ([\#4077](https://github.com/matrix-org/synapse/issues/4077))
-- Clean up some bits of code which were flagged by the linter ([\#4082](https://github.com/matrix-org/synapse/issues/4082))
-
-
-Synapse 0.33.7 (2018-10-18)
-===========================
-
-**Warning**: This release removes the example email notification templates from
-`res/templates` (they are now internal to the python package). This should only
-affect you if you (a) deploy your Synapse instance from a git checkout or a
-github snapshot URL, and (b) have email notifications enabled.
-
-If you have email notifications enabled, you should ensure that
-`email.template_dir` is either configured to point at a directory where you
-have installed customised templates, or leave it unset to use the default
-templates.
-
-Synapse 0.33.7rc2 (2018-10-17)
-==============================
-
-Features
---------
-
-- Ship the example email templates as part of the package ([\#4052](https://github.com/matrix-org/synapse/issues/4052))
-
-Bugfixes
---------
-
-- Fix bug which made get_missing_events return too few events ([\#4045](https://github.com/matrix-org/synapse/issues/4045))
-
-
-Synapse 0.33.7rc1 (2018-10-15)
-==============================
-
-Features
---------
-
-- Add support for end-to-end key backup (MSC1687) ([\#4019](https://github.com/matrix-org/synapse/issues/4019))
-
-
-Bugfixes
---------
-
-- Fix bug in event persistence logic which caused 'NoneType is not iterable' ([\#3995](https://github.com/matrix-org/synapse/issues/3995))
-- Fix exception in background metrics collection ([\#3996](https://github.com/matrix-org/synapse/issues/3996))
-- Fix exception handling in fetching remote profiles ([\#3997](https://github.com/matrix-org/synapse/issues/3997))
-- Fix handling of rejected threepid invites ([\#3999](https://github.com/matrix-org/synapse/issues/3999))
-- Workers now start on Python 3. ([\#4027](https://github.com/matrix-org/synapse/issues/4027))
-- Synapse now starts on Python 3.7. ([\#4033](https://github.com/matrix-org/synapse/issues/4033))
-
-
-Internal Changes
-----------------
-
-- Log exceptions in looping calls ([\#4008](https://github.com/matrix-org/synapse/issues/4008))
-- Optimisation for serving federation requests ([\#4017](https://github.com/matrix-org/synapse/issues/4017))
-- Add metric to count number of non-empty sync responses ([\#4022](https://github.com/matrix-org/synapse/issues/4022))
-
-
-Synapse 0.33.6 (2018-10-04)
-===========================
-
-Internal Changes
-----------------
-
-- Pin to prometheus_client<0.4 to avoid renaming all of our metrics ([\#4002](https://github.com/matrix-org/synapse/issues/4002))
-
-
-Synapse 0.33.6rc1 (2018-10-03)
-==============================
-
-Features
---------
-
-- Adding the ability to change MAX_UPLOAD_SIZE for the docker container variables. ([\#3883](https://github.com/matrix-org/synapse/issues/3883))
-- Report "python_version" in the phone home stats ([\#3894](https://github.com/matrix-org/synapse/issues/3894))
-- Always LL ourselves if we're in a room ([\#3916](https://github.com/matrix-org/synapse/issues/3916))
-- Include eventid in log lines when processing incoming federation transactions ([\#3959](https://github.com/matrix-org/synapse/issues/3959))
-- Remove spurious check which made 'localhost' servers not work ([\#3964](https://github.com/matrix-org/synapse/issues/3964))
-
-
-Bugfixes
---------
-
-- Fix problem when playing media from Chrome using direct URL (thanks @remjey!) ([\#3578](https://github.com/matrix-org/synapse/issues/3578))
-- support registering regular users non-interactively with register_new_matrix_user script ([\#3836](https://github.com/matrix-org/synapse/issues/3836))
-- Fix broken invite email links for self hosted riots ([\#3868](https://github.com/matrix-org/synapse/issues/3868))
-- Don't ratelimit autojoins ([\#3879](https://github.com/matrix-org/synapse/issues/3879))
-- Fix 500 error when deleting unknown room alias ([\#3889](https://github.com/matrix-org/synapse/issues/3889))
-- Fix some b'abcd' noise in logs and metrics ([\#3892](https://github.com/matrix-org/synapse/issues/3892), [\#3895](https://github.com/matrix-org/synapse/issues/3895))
-- When we join a room, always try the server we used for the alias lookup first, to avoid unresponsive and out-of-date servers. ([\#3899](https://github.com/matrix-org/synapse/issues/3899))
-- Fix incorrect server-name indication for outgoing federation requests ([\#3907](https://github.com/matrix-org/synapse/issues/3907))
-- Fix adding client IPs to the database failing on Python 3. ([\#3908](https://github.com/matrix-org/synapse/issues/3908))
-- Fix bug where things occaisonally were not being timed out correctly. ([\#3910](https://github.com/matrix-org/synapse/issues/3910))
-- Fix bug where outbound federation would stop talking to some servers when using workers ([\#3914](https://github.com/matrix-org/synapse/issues/3914))
-- Fix some instances of ExpiringCache not expiring cache items ([\#3932](https://github.com/matrix-org/synapse/issues/3932), [\#3980](https://github.com/matrix-org/synapse/issues/3980))
-- Fix out-of-bounds error when LLing yourself ([\#3936](https://github.com/matrix-org/synapse/issues/3936))
-- Sending server notices regarding user consent now works on Python 3. ([\#3938](https://github.com/matrix-org/synapse/issues/3938))
-- Fix exceptions from metrics handler ([\#3956](https://github.com/matrix-org/synapse/issues/3956))
-- Fix error message for events with m.room.create missing from auth_events ([\#3960](https://github.com/matrix-org/synapse/issues/3960))
-- Fix errors due to concurrent monthly_active_user upserts ([\#3961](https://github.com/matrix-org/synapse/issues/3961))
-- Fix exceptions when processing incoming events over federation ([\#3968](https://github.com/matrix-org/synapse/issues/3968))
-- Replaced all occurences of e.message with str(e). Contributed by Schnuffle ([\#3970](https://github.com/matrix-org/synapse/issues/3970))
-- Fix lazy loaded sync in the presence of rejected state events ([\#3986](https://github.com/matrix-org/synapse/issues/3986))
-- Fix error when logging incomplete HTTP requests ([\#3990](https://github.com/matrix-org/synapse/issues/3990))
-
-
-Internal Changes
-----------------
-
-- Unit tests can now be run under PostgreSQL in Docker using ``test_postgresql.sh``. ([\#3699](https://github.com/matrix-org/synapse/issues/3699))
-- Speed up calculation of typing updates for replication ([\#3794](https://github.com/matrix-org/synapse/issues/3794))
-- Remove documentation regarding installation on Cygwin, the use of WSL is recommended instead. ([\#3873](https://github.com/matrix-org/synapse/issues/3873))
-- Fix typo in README, synaspse -> synapse ([\#3897](https://github.com/matrix-org/synapse/issues/3897))
-- Increase the timeout when filling missing events in federation requests ([\#3903](https://github.com/matrix-org/synapse/issues/3903))
-- Improve the logging when handling a federation transaction ([\#3904](https://github.com/matrix-org/synapse/issues/3904), [\#3966](https://github.com/matrix-org/synapse/issues/3966))
-- Improve logging of outbound federation requests ([\#3906](https://github.com/matrix-org/synapse/issues/3906), [\#3909](https://github.com/matrix-org/synapse/issues/3909))
-- Fix the docker image building on python 3 ([\#3911](https://github.com/matrix-org/synapse/issues/3911))
-- Add a regression test for logging failed HTTP requests on Python 3. ([\#3912](https://github.com/matrix-org/synapse/issues/3912))
-- Comments and interface cleanup for on_receive_pdu ([\#3924](https://github.com/matrix-org/synapse/issues/3924))
-- Fix spurious exceptions when remote http client closes conncetion ([\#3925](https://github.com/matrix-org/synapse/issues/3925))
-- Log exceptions thrown by background tasks ([\#3927](https://github.com/matrix-org/synapse/issues/3927))
-- Add a cache to get_destination_retry_timings ([\#3933](https://github.com/matrix-org/synapse/issues/3933), [\#3991](https://github.com/matrix-org/synapse/issues/3991))
-- Automate pushes to docker hub ([\#3946](https://github.com/matrix-org/synapse/issues/3946))
-- Require attrs 16.0.0 or later ([\#3947](https://github.com/matrix-org/synapse/issues/3947))
-- Fix incompatibility with python3 on alpine ([\#3948](https://github.com/matrix-org/synapse/issues/3948))
-- Run the test suite on the oldest supported versions of our dependencies in CI. ([\#3952](https://github.com/matrix-org/synapse/issues/3952))
-- CircleCI now only runs merged jobs on PRs, and commit jobs on develop, master, and release branches. ([\#3957](https://github.com/matrix-org/synapse/issues/3957))
-- Fix docstrings and add tests for state store methods ([\#3958](https://github.com/matrix-org/synapse/issues/3958))
-- fix docstring for FederationClient.get_state_for_room ([\#3963](https://github.com/matrix-org/synapse/issues/3963))
-- Run notify_app_services as a bg process ([\#3965](https://github.com/matrix-org/synapse/issues/3965))
-- Clarifications in FederationHandler ([\#3967](https://github.com/matrix-org/synapse/issues/3967))
-- Further reduce the docker image size ([\#3972](https://github.com/matrix-org/synapse/issues/3972))
-- Build py3 docker images for docker hub too ([\#3976](https://github.com/matrix-org/synapse/issues/3976))
-- Updated the installation instructions to point to the matrix-synapse package on PyPI. ([\#3985](https://github.com/matrix-org/synapse/issues/3985))
-- Disable USE_FROZEN_DICTS for unittests by default. ([\#3987](https://github.com/matrix-org/synapse/issues/3987))
-- Remove unused Jenkins and development related files from the repo. ([\#3988](https://github.com/matrix-org/synapse/issues/3988))
-- Improve stacktraces in certain exceptions in the logs ([\#3989](https://github.com/matrix-org/synapse/issues/3989))
-
-
 Synapse 0.33.5.1 (2018-09-25)
 =============================
 
 Internal Changes
 ----------------
 
-- Fix incompatibility with older Twisted version in tests. Thanks @OlegGirko! ([\#3940](https://github.com/matrix-org/synapse/issues/3940))
+- Fix incompatibility with older Twisted version in tests. Thanks 
+  @OlegGirko! ([\#3940](https://github.com/matrix-org/synapse/issues/3940))
 
 
 Synapse 0.33.5 (2018-09-24)

MANIFEST.in

@@ -12,20 +12,24 @@ recursive-include synapse/storage/schema *.sql
 recursive-include synapse/storage/schema *.py
 
 recursive-include docs *
+recursive-include res *
 recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
 recursive-include tests *.py
 
-recursive-include synapse/res *
 recursive-include synapse/static *.css
 recursive-include synapse/static *.gif
 recursive-include synapse/static *.html
 recursive-include synapse/static *.js
 
+exclude jenkins.sh
+exclude jenkins*.sh
+exclude jenkins*
 exclude Dockerfile
 exclude .dockerignore
 exclude test_postgresql.sh
+recursive-exclude jenkins *.sh
 
 include pyproject.toml
 recursive-include changelog.d *
@@ -34,7 +38,3 @@ prune .github
 prune demo/etc
 prune docker
 prune .circleci
-prune .coveragerc
-
-exclude jenkins*
-recursive-exclude jenkins *.sh

MAP.rst

@@ -0,0 +1,35 @@
+Directory Structure
+===================
+
+Warning: this may be a bit stale...
+
+::
+
+    .
+    ├── cmdclient           Basic CLI python Matrix client
+    ├── demo                Scripts for running standalone Matrix demos
+    ├── docs                All doc, including the draft Matrix API spec
+    │   ├── client-server       The client-server Matrix API spec
+    │   ├── model               Domain-specific elements of the Matrix API spec
+    │   ├── server-server       The server-server model of the Matrix API spec
+    │   └── sphinx              The internal API doc of the Synapse homeserver
+    ├── experiments         Early experiments of using Synapse's internal APIs
+    ├── graph               Visualisation of Matrix's distributed message store 
+    ├── synapse             The reference Matrix homeserver implementation
+    │   ├── api                 Common building blocks for the APIs
+    │   │   ├── events              Definition of state representation Events 
+    │   │   └── streams             Definition of streamable Event objects
+    │   ├── app                 The __main__ entry point for the homeserver
+    │   ├── crypto              The PKI client/server used for secure federation
+    │   │   └── resource            PKI helper objects (e.g. keys)
+    │   ├── federation          Server-server state replication logic
+    │   ├── handlers            The main business logic of the homeserver
+    │   ├── http                Wrappers around Twisted's HTTP server & client
+    │   ├── rest                Servlet-style RESTful API
+    │   ├── storage             Persistence subsystem (currently only sqlite3)
+    │   │   └── schema              sqlite persistence schema
+    │   └── util                Synapse-specific utilities
+    ├── tests               Unit tests for the Synapse homeserver
+    └── webclient           Basic AngularJS Matrix web client
+
+

README.rst

@@ -81,7 +81,7 @@ Thanks for using Matrix!
 Synapse Installation
 ====================
 
-Synapse is the reference Python/Twisted Matrix homeserver implementation.
+Synapse is the reference python/twisted Matrix homeserver implementation.
 
 System requirements:
 
@@ -91,13 +91,12 @@ System requirements:
 
 Installing from source
 ----------------------
-
 (Prebuilt packages are available for some platforms - see `Platform-Specific
 Instructions`_.)
 
-Synapse is written in Python but some of the libraries it uses are written in
-C. So before we can install Synapse itself we need a working C compiler and the
-header files for Python C extensions.
+Synapse is written in python but some of the libraries it uses are written in
+C. So before we can install synapse itself we need a working C compiler and the
+header files for python C extensions.
 
 Installing prerequisites on Ubuntu or Debian::
 
@@ -142,26 +141,20 @@ Installing prerequisites on openSUSE::
 Installing prerequisites on OpenBSD::
 
     doas pkg_add python libffi py-pip py-setuptools sqlite3 py-virtualenv \
-                 libxslt jpeg
+                 libxslt
 
-To install the Synapse homeserver run::
+To install the synapse homeserver run::
 
     virtualenv -p python2.7 ~/.synapse
     source ~/.synapse/bin/activate
     pip install --upgrade pip
     pip install --upgrade setuptools
-    pip install matrix-synapse
+    pip install https://github.com/matrix-org/synapse/tarball/master
 
-This installs Synapse, along with the libraries it uses, into a virtual
+This installs synapse, along with the libraries it uses, into a virtual
 environment under ``~/.synapse``.  Feel free to pick a different directory
 if you prefer.
 
-This Synapse installation can then be later upgraded by using pip again with the
-update flag::
-
-    source ~/.synapse/bin/activate
-    pip install -U matrix-synapse
-
 In case of problems, please see the _`Troubleshooting` section below.
 
 There is an offical synapse image available at
@@ -174,13 +167,7 @@ Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
 Dockerfile to automate a synapse server in a single Docker image, at
 https://hub.docker.com/r/avhost/docker-matrix/tags/
 
-Slavi Pantaleev has created an Ansible playbook,
-which installs the offical Docker image of Matrix Synapse
-along with many other Matrix-related services (Postgres database, riot-web, coturn, mxisd, SSL support, etc.).
-For more details, see
-https://github.com/spantaleev/matrix-docker-ansible-deploy
-
-Configuring Synapse
+Configuring synapse
 -------------------
 
 Before you can start Synapse, you will need to generate a configuration
@@ -262,6 +249,26 @@ Setting up a TURN server
 For reliable VoIP calls to be routed via this homeserver, you MUST configure
 a TURN server.  See `<docs/turn-howto.rst>`_ for details.
 
+IPv6
+----
+
+As of Synapse 0.19 we finally support IPv6, many thanks to @kyrias and @glyph
+for providing PR #1696.
+
+However, for federation to work on hosts with IPv6 DNS servers you **must**
+be running Twisted 17.1.0 or later - see https://github.com/matrix-org/synapse/issues/1002
+for details.  We can't make Synapse depend on Twisted 17.1 by default
+yet as it will break most older distributions (see https://github.com/matrix-org/synapse/pull/1909)
+so if you are using operating system dependencies you'll have to install your
+own Twisted 17.1 package via pip or backports etc.
+
+If you're running in a virtualenv then pip should have installed the newest
+Twisted automatically, but if your virtualenv is old you will need to manually
+upgrade to a newer Twisted dependency via:
+
+    pip install Twisted>=17.1.0
+
+
 Running Synapse
 ===============
 
@@ -437,7 +444,8 @@ settings require a slightly more difficult installation process.
    using the ``.`` command, rather than ``bash``'s ``source``.
 5) Optionally, use ``pip`` to install ``lxml``, which Synapse needs to parse
    webpages for their titles.
-6) Use ``pip`` to install this repository: ``pip install matrix-synapse``
+6) Use ``pip`` to install this repository: ``pip install
+   https://github.com/matrix-org/synapse/tarball/master``
 7) Optionally, change ``_synapse``'s shell to ``/bin/false`` to reduce the
    chance of a compromised Synapse server being used to take over your box.
 
@@ -465,7 +473,7 @@ Troubleshooting
 Troubleshooting Installation
 ----------------------------
 
-Synapse requires pip 8 or later, so if your OS provides too old a version you
+Synapse requires pip 1.7 or later, so if your OS provides too old a version you
 may need to manually upgrade it::
 
     sudo pip install --upgrade pip
@@ -500,6 +508,28 @@ failing, e.g.::
 
     pip install twisted
 
+On OS X, if you encounter clang: error: unknown argument: '-mno-fused-madd' you
+will need to export CFLAGS=-Qunused-arguments.
+
+Troubleshooting Running
+-----------------------
+
+If synapse fails with ``missing "sodium.h"`` crypto errors, you may need
+to manually upgrade PyNaCL, as synapse uses NaCl (https://nacl.cr.yp.to/) for
+encryption and digital signatures.
+Unfortunately PyNACL currently has a few issues
+(https://github.com/pyca/pynacl/issues/53) and
+(https://github.com/pyca/pynacl/issues/79) that mean it may not install
+correctly, causing all tests to fail with errors about missing "sodium.h". To
+fix try re-installing from PyPI or directly from
+(https://github.com/pyca/pynacl)::
+
+    # Install from PyPI
+    pip install --user --upgrade --force pynacl
+
+    # Install from github
+    pip install --user https://github.com/pyca/pynacl/tarball/master
+
 Running out of File Handles
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -657,8 +687,7 @@ Using a reverse proxy with Synapse
 
 It is recommended to put a reverse proxy such as
 `nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
-`Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_,
-`Caddy <https://caddyserver.com/docs/proxy>`_ or
+`Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_ or
 `HAProxy <https://www.haproxy.org/>`_ in front of Synapse. One advantage of
 doing so is that it means that you can expose the default https port (443) to
 Matrix clients without needing to run Synapse with root privileges.
@@ -689,15 +718,7 @@ so an example nginx configuration might look like::
       }
   }
 
-an example Caddy configuration might look like::
-
-    matrix.example.com {
-      proxy /_matrix http://localhost:8008 {
-        transparent
-      }
-    }
-
-and an example Apache configuration might look like::
+and an example apache configuration may look like::
 
     <VirtualHost *:443>
         SSLEngine on
@@ -729,10 +750,9 @@ port:
 
   .. __: `key_management`_
 
-* Until v0.33.3, Synapse did not support SNI on the federation port
-  (`bug #1491 <https://github.com/matrix-org/synapse/issues/1491>`_). This bug
-  is now fixed, but means that federating with older servers can be unreliable 
-  when using name-based virtual hosting.
+* Synapse does not currently support SNI on the federation protocol
+  (`bug #1491 <https://github.com/matrix-org/synapse/issues/1491>`_), which
+  means that using name-based virtual hosting is unreliable.
 
 Furthermore, a number of the normal reasons for using a reverse-proxy do not
 apply:

UPGRADE.rst

@@ -18,7 +18,7 @@ instructions that may be required are listed later in this document.
 
    .. code:: bash
 
-       pip install --upgrade --process-dependency-links matrix-synapse
+       pip install --upgrade --process-dependency-links https://github.com/matrix-org/synapse/tarball/master
 
        # restart synapse
        synctl restart
@@ -48,24 +48,11 @@ returned by the Client-Server API:
     # configured on port 443.
     curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"
 
-Upgrading to v0.33.7
-====================
-
-This release removes the example email notification templates from
-``res/templates`` (they are now internal to the python package). This should
-only affect you if you (a) deploy your Synapse instance from a git checkout or
-a github snapshot URL, and (b) have email notifications enabled.
-
-If you have email notifications enabled, you should ensure that
-``email.template_dir`` is either configured to point at a directory where you
-have installed customised templates, or leave it unset to use the default
-templates.
-
-Upgrading to v0.27.3
+Upgrading to $NEXT_VERSION
 ====================
 
 This release expands the anonymous usage stats sent if the opt-in
-``report_stats`` configuration is set to ``true``. We now capture RSS memory
+``report_stats`` configuration is set to ``true``. We now capture RSS memory 
 and cpu use at a very coarse level. This requires administrators to install
 the optional ``psutil`` python module.
 

changelog.d/3578.bugfix

@@ -0,0 +1 @@
+Fix problem when playing media from Chrome using direct URL (thanks @remjey!)

changelog.d/3699.misc

@@ -0,0 +1,2 @@
+Unit tests can now be run under PostgreSQL in Docker using 
+``test_postgresql.sh``.

changelog.d/3794.misc

@@ -0,0 +1 @@
+Speed up calculation of typing updates for replication

changelog.d/3830.feature

@@ -1 +0,0 @@
-Add option to track MAU stats (but not limit people)

changelog.d/3836.bugfix

@@ -0,0 +1 @@
+support registering regular users non-interactively with register_new_matrix_user script

changelog.d/3868.bugfix

@@ -0,0 +1 @@
+Fix broken invite email links for self hosted riots 

changelog.d/3873.misc

@@ -0,0 +1,2 @@
+Remove documentation regarding installation on Cygwin, the use of WSL is 
+recommended instead.

changelog.d/3879.bugfix

@@ -0,0 +1 @@
+Don't ratelimit autojoins

changelog.d/3883.feature

@@ -0,0 +1 @@
+Adding the ability to change MAX_UPLOAD_SIZE for the docker container variables.

changelog.d/3889.bugfix

@@ -0,0 +1 @@
+Fix 500 error when deleting unknown room alias

changelog.d/3892.bugfix

@@ -0,0 +1 @@
+Fix some b'abcd' noise in logs and metrics

changelog.d/3894.feature

@@ -0,0 +1 @@
+Report "python_version" in the phone home stats

changelog.d/3895.bugfix

@@ -0,0 +1 @@
+Fix some b'abcd' noise in logs and metrics

changelog.d/3897.misc

@@ -0,0 +1 @@
+Fix typo in README, synaspse -> synapse

changelog.d/3899.bugfix

@@ -0,0 +1 @@
+When we join a room, always try the server we used for the alias lookup first, to avoid unresponsive and out-of-date servers.

changelog.d/3903.misc

@@ -0,0 +1 @@
+Increase the timeout when filling missing events in federation requests

changelog.d/3904.misc

@@ -0,0 +1 @@
+Improve the logging when handling a federation transaction

changelog.d/3906.misc

@@ -0,0 +1 @@
+Improve logging of outbound federation requests

changelog.d/3907.bugfix

@@ -0,0 +1 @@
+Fix incorrect server-name indication for outgoing federation requests

changelog.d/3908.bugfix

@@ -0,0 +1 @@
+Fix adding client IPs to the database failing on Python 3.

changelog.d/3909.misc

@@ -0,0 +1 @@
+Improve logging of outbound federation requests

changelog.d/3910.bugfix

@@ -0,0 +1 @@
+Fix bug where things occaisonally were not being timed out correctly.

changelog.d/3911.misc

@@ -0,0 +1 @@
+Fix the docker image building on python 3

changelog.d/3912.misc

@@ -0,0 +1 @@
+Add a regression test for logging failed HTTP requests on Python 3.

changelog.d/3914.bugfix

@@ -0,0 +1 @@
+Fix bug where outbound federation would stop talking to some servers when using workers

changelog.d/3916.feature

@@ -0,0 +1 @@
+Always LL ourselves if we're in a room

changelog.d/3924.misc

@@ -0,0 +1 @@
+Comments and interface cleanup for on_receive_pdu

changelog.d/3925.misc

@@ -0,0 +1 @@
+Fix spurious exceptions when remote http client closes conncetion

changelog.d/3927.misc

@@ -0,0 +1 @@
+Log exceptions thrown by background tasks

changelog.d/3932.bugfix

@@ -0,0 +1 @@
+Fix some instances of ExpiringCache not expiring cache items

changelog.d/3936.bugfix

@@ -0,0 +1 @@
+Fix out-of-bounds error when LLing yourself

changelog.d/3946.misc

@@ -0,0 +1 @@
+Automate pushes to docker hub

changelog.d/3947.misc

@@ -0,0 +1 @@
+Require attrs 16.0.0 or later

changelog.d/3948.misc

@@ -0,0 +1 @@
+Fix incompatibility with python3 on alpine

changelog.d/3952.misc

@@ -0,0 +1 @@
+Run the test suite on the oldest supported versions of our dependencies in CI.

changelog.d/3956.bugfix

@@ -0,0 +1 @@
+Fix exceptions from metrics handler

changelog.d/3957.misc

@@ -0,0 +1 @@
+CircleCI now only runs merged jobs on PRs, and commit jobs on develop, master, and release branches.

changelog.d/3958.misc

@@ -0,0 +1 @@
+Fix docstrings and add tests for state store methods

changelog.d/3959.feature

@@ -0,0 +1 @@
+Include eventid in log lines when processing incoming federation transactions

changelog.d/3961.bugfix

@@ -0,0 +1 @@
+Fix errors due to concurrent monthly_active_user upserts 

changelog.d/3963.misc

@@ -0,0 +1 @@
+fix docstring for FederationClient.get_state_for_room

changelog.d/3965.misc

@@ -0,0 +1 @@
+Run notify_app_services as a bg process

changelog.d/3966.misc

@@ -0,0 +1 @@
+Improve the logging when handling a federation transaction

changelog.d/3967.misc

@@ -0,0 +1 @@
+Clarifications in FederationHandler

changelog.d/3970.bugfix

@@ -0,0 +1 @@
+Replaced all occurences of e.message with str(e). Contributed by Schnuffle

changelog.d/4176.bugfix

@@ -1 +0,0 @@
-The media repository now no longer fails to decode UTF-8 filenames when downloading remote media.

changelog.d/4180.misc

@@ -1 +0,0 @@
-A coveragerc file, as well as the py36-coverage tox target, have been added.

changelog.d/4182.misc

@@ -1 +0,0 @@
-Add a GitHub pull request template and add multiple issue templates

changelog.d/4183.bugfix

@@ -1 +0,0 @@
-URL previews now correctly decode non-UTF-8 text if the header contains a `<meta http-equiv="Content-Type"` header.

changelog.d/4188.misc

@@ -1 +0,0 @@
-Update README to reflect the fact that #1491 is fixed

changelog.d/4190.misc

@@ -1 +0,0 @@
-Add some diagnostics to the tests to detect logcontext problems

changelog.d/4192.bugfix

@@ -1 +0,0 @@
-Fix an issue where public consent URLs had two slashes.

changelog.d/4193.misc

@@ -1 +0,0 @@
-Add missing `jpeg` package prerequisite for OpenBSD in README.

changelog.d/4197.bugfix

@@ -1 +0,0 @@
-Fallback auth now accepts the session parameter on Python 3.

changelog.d/4200.misc

@@ -1 +0,0 @@
-Add a note saying you need to manually reclaim disk space after using the Purge History API

changelog.d/4204.misc

@@ -1 +0,0 @@
-Fix logcontext leaks in EmailPusher and in tests

changelog.d/4207.bugfix

@@ -1 +0,0 @@
-Remove riot.im from the list of trusted Identity Servers in the default configuration

changelog.d/4209.misc

@@ -1 +0,0 @@
-Fix logcontext leaks in EmailPusher and in tests

changelog.d/4214.misc

@@ -1 +0,0 @@
-Ignore __pycache__ directories in the database schema folder

changelog.d/4220.feature

@@ -1 +0,0 @@
-Rename login type m.login.cas to m.login.sso

changelog.d/4227.misc

@@ -1 +0,0 @@
-Garbage-collect after each unit test to fix logcontext leaks

contrib/docker/docker-compose.yml

@@ -6,11 +6,9 @@ version: '3'
 services:
 
   synapse:
-    build:
-        context: ../..
-        dockerfile: docker/Dockerfile
+    build: ../..
     image: docker.io/matrixdotorg/synapse:latest
-    # Since synapse does not retry to connect to the database, restart upon
+    # Since snyapse does not retry to connect to the database, restart upon
     # failure
     restart: unless-stopped
     # See the readme for a full documentation of the environment settings
@@ -49,4 +47,4 @@ services:
       # You may store the database tables in a local folder..
       - ./schemas:/var/lib/postgresql/data
       # .. or store them on some high performance storage for better results
-      # - /path/to/ssd/storage:/var/lib/postgresql/data
+      # - /path/to/ssd/storage:/var/lib/postfesql/data

contrib/purge_api/README.md

@@ -1,16 +0,0 @@
-Purge history API examples
-==========================
-
-# `purge_history.sh`
-
-A bash file, that uses the [purge history API](/docs/admin_api/README.rst) to 
-purge all messages in a list of rooms up to a certain event. You can select a 
-timeframe or a number of messages that you want to keep in the room.
-
-Just configure the variables DOMAIN, ADMIN, ROOMS_ARRAY and TIME at the top of
-the script.
-
-# `purge_remote_media.sh`
-
-A bash file, that uses the [purge history API](/docs/admin_api/README.rst) to 
-purge all old cached remote media.

contrib/purge_api/purge_history.sh

@@ -1,141 +0,0 @@
-#!/bin/bash
-
-# this script will use the api:
-#    https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst
-# 
-# It will purge all messages in a list of rooms up to a cetrain event
-
-###################################################################################################
-# define your domain and admin user
-###################################################################################################
-# add this user as admin in your home server:
-DOMAIN=yourserver.tld
-# add this user as admin in your home server:
-ADMIN="@you_admin_username:$DOMAIN"
-
-API_URL="$DOMAIN:8008/_matrix/client/r0"
-
-###################################################################################################
-#choose the rooms to prune old messages from (add a free comment at the end)
-###################################################################################################
-# the room_id's you can get e.g. from your Riot clients "View Source" button on each message
-ROOMS_ARRAY=(
-'!DgvjtOljKujDBrxyHk:matrix.org#riot:matrix.org'
-'!QtykxKocfZaZOUrTwp:matrix.org#Matrix HQ'
-)
-
-# ALTERNATIVELY:
-# you can select all the rooms that are not encrypted and loop over the result:
-# SELECT room_id FROM rooms WHERE room_id NOT IN (SELECT DISTINCT room_id FROM events WHERE type ='m.room.encrypted')
-# or
-# select all rooms with at least 100 members:
-# SELECT q.room_id FROM (select count(*) as numberofusers, room_id FROM current_state_events WHERE type ='m.room.member'
-#   GROUP BY room_id) AS q LEFT JOIN room_aliases a ON q.room_id=a.room_id WHERE q.numberofusers > 100 ORDER BY numberofusers desc
-
-###################################################################################################
-# evaluate the EVENT_ID before which should be pruned
-###################################################################################################
-# choose a time before which the messages should be pruned:
-TIME='12 months ago'
-# ALTERNATIVELY:
-# a certain time:
-# TIME='2016-08-31 23:59:59'
-
-# creates a timestamp from the given time string:
-UNIX_TIMESTAMP=$(date +%s%3N --date='TZ="UTC+2" '"$TIME")
-
-# ALTERNATIVELY:
-# prune all messages that are older than 1000 messages ago:
-# LAST_MESSAGES=1000
-# SQL_GET_EVENT="SELECT event_id from events WHERE type='m.room.message' AND room_id ='$ROOM' ORDER BY received_ts DESC LIMIT 1 offset $(($LAST_MESSAGES - 1))"
-
-# ALTERNATIVELY:
-# select the EVENT_ID manually:
-#EVENT_ID='$1471814088343495zpPNI:matrix.org' # an example event from 21st of Aug 2016 by Matthew
-
-###################################################################################################
-# make the admin user a server admin in the database with
-###################################################################################################
-# psql -A -t --dbname=synapse -c "UPDATE users SET admin=1 WHERE name LIKE '$ADMIN'"
-
-###################################################################################################
-# database function
-###################################################################################################
-sql (){
-  # for sqlite3:
-  #sqlite3 homeserver.db "pragma busy_timeout=20000;$1" | awk '{print $2}'
-  # for postgres:
-  psql -A -t --dbname=synapse -c "$1" | grep -v 'Pager'
-}
-
-###################################################################################################
-# get an access token
-###################################################################################################
-# for example externally by watching Riot in your browser's network inspector
-# or internally on the server locally, use this:
-TOKEN=$(sql "SELECT token FROM access_tokens WHERE user_id='$ADMIN' ORDER BY id DESC LIMIT 1")
-AUTH="Authorization: Bearer $TOKEN"
-
-###################################################################################################
-# check, if your TOKEN works. For example this works: 
-###################################################################################################
-# $ curl --header "$AUTH" "$API_URL/rooms/$ROOM/state/m.room.power_levels" 
-
-###################################################################################################
-# finally start pruning the room:
-###################################################################################################
-POSTDATA='{"delete_local_events":"true"}' # this will really delete local events, so the messages in the room really disappear unless they are restored by remote federation
-
-for ROOM in "${ROOMS_ARRAY[@]}"; do
-    echo "########################################### $(date) ################# "
-    echo "pruning room: $ROOM ..."
-    ROOM=${ROOM%#*}
-    #set -x
-    echo "check for alias in db..."
-    # for postgres:
-    sql "SELECT * FROM room_aliases WHERE room_id='$ROOM'"
-    echo "get event..."
-    # for postgres:
-    EVENT_ID=$(sql "SELECT event_id FROM events WHERE type='m.room.message' AND received_ts<'$UNIX_TIMESTAMP' AND room_id='$ROOM' ORDER BY received_ts DESC LIMIT 1;")
-    if [ "$EVENT_ID" == "" ]; then
-      echo "no event $TIME"
-    else
-      echo "event: $EVENT_ID"
-      SLEEP=2
-      set -x
-      # call purge
-      OUT=$(curl --header "$AUTH" -s -d $POSTDATA POST "$API_URL/admin/purge_history/$ROOM/$EVENT_ID")
-      PURGE_ID=$(echo "$OUT" |grep purge_id|cut -d'"' -f4 )
-      if [ "$PURGE_ID" == "" ]; then
-        # probably the history purge is already in progress for $ROOM
-        : "continuing with next room"
-      else
-        while : ; do
-          # get status of purge and sleep longer each time if still active
-          sleep $SLEEP
-          STATUS=$(curl --header "$AUTH" -s GET "$API_URL/admin/purge_history_status/$PURGE_ID" |grep status|cut -d'"' -f4)
-          : "$ROOM --> Status: $STATUS"
-          [[ "$STATUS" == "active" ]] || break 
-          SLEEP=$((SLEEP + 1))
-        done 
-      fi
-      set +x
-      sleep 1
-    fi  
-done
-
-
-###################################################################################################
-# additionally
-###################################################################################################
-# to benefit from pruning large amounts of data, you need to call VACUUM to free the unused space.
-# This can take a very long time (hours) and the client have to be stopped while you do so:
-# $ synctl stop
-# $ sqlite3 -line homeserver.db "vacuum;"
-# $ synctl start
-
-# This could be set, so you don't need to prune every time after deleting some rows:
-# $ sqlite3 homeserver.db "PRAGMA auto_vacuum = FULL;"
-# be cautious, it could make the database somewhat slow if there are a lot of deletions
-
-exit

contrib/purge_api/purge_remote_media.sh

@@ -1,54 +0,0 @@
-#!/bin/bash
-
-DOMAIN=yourserver.tld
-# add this user as admin in your home server:
-ADMIN="@you_admin_username:$DOMAIN"
-
-API_URL="$DOMAIN:8008/_matrix/client/r0"
-
-# choose a time before which the messages should be pruned:
-# TIME='2016-08-31 23:59:59'
-TIME='12 months ago'
-
-# creates a timestamp from the given time string:
-UNIX_TIMESTAMP=$(date +%s%3N --date='TZ="UTC+2" '"$TIME")
-
-
-###################################################################################################
-# database function
-###################################################################################################
-sql (){
-  # for sqlite3:
-  #sqlite3 homeserver.db "pragma busy_timeout=20000;$1" | awk '{print $2}'
-  # for postgres:
-  psql -A -t --dbname=synapse -c "$1" | grep -v 'Pager'
-}
-
-###############################################################################
-# make the admin user a server admin in the database with
-###############################################################################
-# sql "UPDATE users SET admin=1 WHERE name LIKE '$ADMIN'"
-
-###############################################################################
-# get an access token
-###############################################################################
-# for example externally by watching Riot in your browser's network inspector
-# or internally on the server locally, use this:
-TOKEN=$(sql "SELECT token FROM access_tokens WHERE user_id='$ADMIN' ORDER BY id DESC LIMIT 1")
-
-###############################################################################
-# check, if your TOKEN works. For example this works: 
-###############################################################################
-# curl --header "Authorization: Bearer $TOKEN" "$API_URL/rooms/$ROOM/state/m.room.power_levels"
-
-###############################################################################
-# optional check size before
-###############################################################################
-# echo calculate used storage before ...
-# du -shc ../.synapse/media_store/*
-
-###############################################################################
-# finally start pruning media:
-###############################################################################
-set -x # for debugging the generated string
-curl --header "Authorization: Bearer $TOKEN" -v POST "$API_URL/admin/purge_media_cache/?before_ts=$UNIX_TIMESTAMP"

docker/Dockerfile

@@ -1,13 +1,9 @@
 ARG PYTHON_VERSION=2
+FROM docker.io/python:${PYTHON_VERSION}-alpine3.8
 
-###
-### Stage 0: builder
-###
-FROM docker.io/python:${PYTHON_VERSION}-alpine3.8 as builder
+COPY . /synapse
 
-# install the OS build deps
-
-RUN apk add \
+RUN apk add --no-cache --virtual .build_deps \
         build-base \
         libffi-dev \
         libjpeg-turbo-dev \
@@ -15,47 +11,30 @@ RUN apk add \
         libxslt-dev \
         linux-headers \
         postgresql-dev \
-        zlib-dev
-
-# build things which have slow build steps, before we copy synapse, so that
-# the layer can be cached.
-#
-# (we really just care about caching a wheel here, as the "pip install" below
-# will install them again.)
-
-RUN pip install --prefix="/install" --no-warn-script-location \
-        cryptography \
-        msgpack-python \
-        pillow \
-        pynacl
-
-# now install synapse and all of the python deps to /install.
-
-COPY . /synapse
-RUN pip install --prefix="/install" --no-warn-script-location \
-        lxml \
-        psycopg2 \
-        /synapse
-
-###
-### Stage 1: runtime
-###
-
-FROM docker.io/python:${PYTHON_VERSION}-alpine3.8
-
-RUN apk add --no-cache --virtual .runtime_deps \
-        libffi \
+        zlib-dev \
+ && cd /synapse \
+ && apk add --no-cache --virtual .runtime_deps \
+ 	libffi \
         libjpeg-turbo \
-        libressl \
-        libxslt \
-        libpq \
-        zlib \
-        su-exec
-
-COPY --from=builder /install /usr/local
-COPY ./docker/start.py /start.py
-COPY ./docker/conf /conf
-
+	libressl \
+	libxslt \
+	libpq \
+	zlib \
+	su-exec \
+ && pip install --upgrade \
+        lxml \
+        pip \
+        psycopg2 \
+        setuptools \
+ && mkdir -p /synapse/cache \
+ && pip install -f /synapse/cache --upgrade --process-dependency-links . \
+ && mv /synapse/docker/start.py /synapse/docker/conf / \
+ && rm -rf \
+        setup.cfg \
+        setup.py \
+        synapse \
+ && apk del .build_deps
+ 
 VOLUME ["/data"]
 
 EXPOSE 8008/tcp 8448/tcp

docker/conf/homeserver.yaml

@@ -21,7 +21,7 @@ listeners:
   {% if not SYNAPSE_NO_TLS %}
   -
     port: 8448
-    bind_addresses: ['::']
+    bind_addresses: ['0.0.0.0']
     type: http
     tls: true
     x_forwarded: false
@@ -34,7 +34,7 @@ listeners:
 
   - port: 8008
     tls: false
-    bind_addresses: ['::']
+    bind_addresses: ['0.0.0.0']
     type: http
     x_forwarded: false
 
@@ -150,12 +150,10 @@ enable_group_creation: true
 
 # The list of identity servers trusted to verify third party
 # identifiers by this server.
-#
-# Also defines the ID server which will be called when an account is
-# deactivated (one will be picked arbitrarily).
 trusted_third_party_id_servers:
     - matrix.org
     - vector.im
+    - riot.im
 
 ## Metrics ###
 
@@ -213,9 +211,7 @@ email:
    require_transport_security: False
    notif_from: "{{ SYNAPSE_SMTP_FROM or "hostmaster@" + SYNAPSE_SERVER_NAME }}"
    app_name: Matrix
-   # if template_dir is unset, uses the example templates that are part of
-   # the Synapse distribution.
-   #template_dir: res/templates
+   template_dir: res/templates
    notif_template_html: notif_mail.html
    notif_template_text: notif_mail.txt
    notif_for_new_users: True

docs/admin_api/purge_history_api.rst

@@ -61,11 +61,3 @@ the following:
     }
 
 The status will be one of ``active``, ``complete``, or ``failed``.
-
-Reclaim disk space (Postgres)
------------------------------
-
-To reclaim the disk space and return it to the operating system, you need to run
-`VACUUM FULL;` on the database.
-
-https://www.postgresql.org/docs/current/sql-vacuum.html

docs/consent_tracking.md

@@ -31,7 +31,7 @@ Note that the templates must be stored under a name giving the language of the
 template - currently this must always be `en` (for "English");
 internationalisation support is intended for the future.
 
-The template for the policy itself should be versioned and named according to
+The template for the policy itself should be versioned and named according to 
 the version: for example `1.0.html`. The version of the policy which the user
 has agreed to is stored in the database.
 
@@ -85,37 +85,6 @@ Once this is complete, and the server has been restarted, try visiting
 an error "Missing string query parameter 'u'". It is now possible to manually
 construct URIs where users can give their consent.
 
-### Enabling consent tracking at registration
-
-1. Add the following to your configuration:
-
-   ```yaml
-   user_consent:
-     require_at_registration: true
-     policy_name: "Privacy Policy" # or whatever you'd like to call the policy
-   ```
-
-2. In your consent templates, make use of the `public_version` variable to
-   see if an unauthenticated user is viewing the page. This is typically
-   wrapped around the form that would be used to actually agree to the document:
-
-   ```
-   {% if not public_version %}
-     <!-- The variables used here are only provided when the 'u' param is given to the homeserver -->
-     <form method="post" action="consent">
-       <input type="hidden" name="v" value="{{version}}"/>
-       <input type="hidden" name="u" value="{{user}}"/>
-       <input type="hidden" name="h" value="{{userhmac}}"/>
-       <input type="submit" value="Sure thing!"/>
-     </form>
-   {% endif %}
-   ```
-
-3. Restart Synapse to apply the changes.
-
-Visiting `https://<server>/_matrix/consent` should now give you a view of the privacy
-document. This is what users will be able to see when registering for accounts.
-
 ### Constructing the consent URI
 
 It may be useful to manually construct the "consent URI" for a given user - for
@@ -137,12 +106,6 @@ query parameters:
    `https://<server>/_matrix/consent?u=<user>&h=68a152465a4d...`.
 
 
-Note that not providing a `u` parameter will be interpreted as wanting to view
-the document from an unauthenticated perspective, such as prior to registration.
-Therefore, the `h` parameter is not required in this scenario. To enable this
-behaviour, set `require_at_registration` to `true` in your `user_consent` config.
-
-
 Sending users a server notice asking them to agree to the policy
 ----------------------------------------------------------------
 

docs/log_contexts.rst

@@ -163,7 +163,7 @@ the logcontext was set, this will make things work out ok: provided
 It's all too easy to forget to ``yield``: for instance if we forgot that
 ``do_some_stuff`` returned a deferred, we might plough on regardless. This
 leads to a mess; it will probably work itself out eventually, but not before
-a load of stuff has been logged against the wrong context. (Normally, other
+a load of stuff has been logged against the wrong content. (Normally, other
 things will break, more obviously, if you forget to ``yield``, so this tends
 not to be a major problem in practice.)
 
@@ -440,59 +440,3 @@ To conclude: I think this scheme would have worked equally well, with less
 danger of messing it up, and probably made some more esoteric code easier to
 write. But again — changing the conventions of the entire Synapse codebase is
 not a sensible option for the marginal improvement offered.
-
-
-A note on garbage-collection of Deferred chains
------------------------------------------------
-
-It turns out that our logcontext rules do not play nicely with Deferred
-chains which get orphaned and garbage-collected.
-
-Imagine we have some code that looks like this:
-
-.. code:: python
-
-    listener_queue = []
-
-    def on_something_interesting():
-        for d in listener_queue:
-            d.callback("foo")
-
-    @defer.inlineCallbacks
-    def await_something_interesting():
-        new_deferred = defer.Deferred()
-        listener_queue.append(new_deferred)
-
-        with PreserveLoggingContext():
-            yield new_deferred
-
-Obviously, the idea here is that we have a bunch of things which are waiting
-for an event. (It's just an example of the problem here, but a relatively
-common one.)
-
-Now let's imagine two further things happen. First of all, whatever was
-waiting for the interesting thing goes away. (Perhaps the request times out,
-or something *even more* interesting happens.)
-
-Secondly, let's suppose that we decide that the interesting thing is never
-going to happen, and we reset the listener queue:
-
-.. code:: python
-
-    def reset_listener_queue():
-        listener_queue.clear()
-
-So, both ends of the deferred chain have now dropped their references, and the
-deferred chain is now orphaned, and will be garbage-collected at some point.
-Note that ``await_something_interesting`` is a generator function, and when
-Python garbage-collects generator functions, it gives them a chance to clean
-up by making the ``yield`` raise a ``GeneratorExit`` exception. In our case,
-that means that the ``__exit__`` handler of ``PreserveLoggingContext`` will
-carefully restore the request context, but there is now nothing waiting for
-its return, so the request context is never cleared.
-
-To reiterate, this problem only arises when *both* ends of a deferred chain
-are dropped. Dropping the the reference to a deferred you're supposed to be
-calling is probably bad practice, so this doesn't actually happen too much.
-Unfortunately, when it does happen, it will lead to leaked logcontexts which
-are incredibly hard to track down.

docs/privacy_policy_templates/en/1.0.html

@@ -12,15 +12,12 @@
     <p>
       All your base are belong to us.
     </p>
-    {% if not public_version %}
-      <!-- The variables used here are only provided when the 'u' param is given to the homeserver -->
-      <form method="post" action="consent">
-        <input type="hidden" name="v" value="{{version}}"/>
-        <input type="hidden" name="u" value="{{user}}"/>
-        <input type="hidden" name="h" value="{{userhmac}}"/>
-        <input type="submit" value="Sure thing!"/>
-      </form>
-    {% endif %}
+    <form method="post" action="consent">
+      <input type="hidden" name="v" value="{{version}}"/>
+      <input type="hidden" name="u" value="{{user}}"/>
+      <input type="hidden" name="h" value="{{userhmac}}"/>
+      <input type="submit" value="Sure thing!"/>
+    </form>
   {% endif %}
   </body>
 </html>

jenkins-dendron-haproxy-postgres.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -eux
+
+: ${WORKSPACE:="$(pwd)"}
+
+export WORKSPACE
+export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
+
+export HAPROXY_BIN=/home/haproxy/haproxy-1.6.11/haproxy
+
+./jenkins/prepare_synapse.sh
+./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git
+./jenkins/clone.sh dendron https://github.com/matrix-org/dendron.git
+./dendron/jenkins/build_dendron.sh
+./sytest/jenkins/prep_sytest_for_postgres.sh
+
+./sytest/jenkins/install_and_run.sh \
+    --python $WORKSPACE/.tox/py27/bin/python \
+    --synapse-directory $WORKSPACE \
+    --dendron $WORKSPACE/dendron/bin/dendron \
+    --haproxy \

jenkins-dendron-postgres.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -eux
+
+: ${WORKSPACE:="$(pwd)"}
+
+export WORKSPACE
+export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
+
+./jenkins/prepare_synapse.sh
+./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git
+./jenkins/clone.sh dendron https://github.com/matrix-org/dendron.git
+./dendron/jenkins/build_dendron.sh
+./sytest/jenkins/prep_sytest_for_postgres.sh
+
+./sytest/jenkins/install_and_run.sh \
+    --python $WORKSPACE/.tox/py27/bin/python \
+    --synapse-directory $WORKSPACE \
+    --dendron $WORKSPACE/dendron/bin/dendron \

jenkins-flake8.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -eux
+
+: ${WORKSPACE:="$(pwd)"}
+
+export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
+
+# Output test results as junit xml
+export TRIAL_FLAGS="--reporter=subunit"
+export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml"
+# Write coverage reports to a separate file for each process
+export COVERAGE_OPTS="-p"
+export DUMP_COVERAGE_COMMAND="coverage help"
+
+# Output flake8 violations to violations.flake8.log
+export PEP8SUFFIX="--output-file=violations.flake8.log"
+
+rm .coverage* || echo "No coverage files to remove"
+
+tox -e packaging -e pep8

jenkins-postgres.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -eux
+
+: ${WORKSPACE:="$(pwd)"}
+
+export WORKSPACE
+export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
+
+./jenkins/prepare_synapse.sh
+./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git
+
+./sytest/jenkins/prep_sytest_for_postgres.sh
+
+./sytest/jenkins/install_and_run.sh \
+    --python $WORKSPACE/.tox/py27/bin/python \
+    --synapse-directory $WORKSPACE \

jenkins-sqlite.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -eux
+
+: ${WORKSPACE:="$(pwd)"}
+
+export WORKSPACE
+export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
+
+./jenkins/prepare_synapse.sh
+./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git
+
+./sytest/jenkins/install_and_run.sh \
+    --python $WORKSPACE/.tox/py27/bin/python \
+    --synapse-directory $WORKSPACE \

jenkins-unittests.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -eux
+
+: ${WORKSPACE:="$(pwd)"}
+
+export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
+
+# Output test results as junit xml
+export TRIAL_FLAGS="--reporter=subunit"
+export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml"
+# Write coverage reports to a separate file for each process
+export COVERAGE_OPTS="-p"
+export DUMP_COVERAGE_COMMAND="coverage help"
+
+# Output flake8 violations to violations.flake8.log
+# Don't exit with non-0 status code on Jenkins,
+# so that the build steps continue and a later step can decided whether to
+# UNSTABLE or FAILURE this build.
+export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?"
+
+rm .coverage* || echo "No coverage files to remove"
+
+tox --notest -e py27
+TOX_BIN=$WORKSPACE/.tox/py27/bin
+python synapse/python_dependencies.py | xargs -n1 $TOX_BIN/pip install
+$TOX_BIN/pip install lxml
+
+tox -e py27

jenkins/clone.sh

@@ -0,0 +1,44 @@
+#! /bin/bash
+
+# This clones a project from github into a named subdirectory
+# If the project has a branch with the same name as this branch
+# then it will checkout that branch after cloning.
+# Otherwise it will checkout "origin/develop."
+# The first argument is the name of the directory to checkout
+# the branch into.
+# The second argument is the URL of the remote repository to checkout.
+# Usually something like https://github.com/matrix-org/sytest.git
+
+set -eux
+
+NAME=$1
+PROJECT=$2
+BASE=".$NAME-base"
+
+# Update our mirror.
+if [ ! -d ".$NAME-base" ]; then
+  # Create a local mirror of the source repository.
+  # This saves us from having to download the entire repository
+  # when this script is next run.
+  git clone "$PROJECT" "$BASE" --mirror
+else
+  # Fetch any updates from the source repository.
+  (cd "$BASE"; git fetch -p)
+fi
+
+# Remove the existing repository so that we have a clean copy
+rm -rf "$NAME"
+# Cloning with --shared means that we will share portions of the
+# .git directory with our local mirror.
+git clone "$BASE" "$NAME" --shared
+
+# Jenkins may have supplied us with the name of the branch in the
+# environment. Otherwise we will have to guess based on the current
+# commit.
+: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"}
+cd "$NAME"
+# check out the relevant branch
+git checkout "${GIT_BRANCH}" || (
+    echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop"
+    git checkout "origin/develop"
+)

jenkins/prepare_synapse.sh

@@ -14,3 +14,22 @@ fi
 
 # set up the virtualenv
 tox -e py27 --notest -v
+
+TOX_BIN=$TOX_DIR/py27/bin
+
+# cryptography 2.2 requires setuptools >= 18.5.
+#
+# older versions of virtualenv (?) give us a virtualenv with the same version
+# of setuptools as is installed on the system python (and tox runs virtualenv
+# under python3, so we get the version of setuptools that is installed on that).
+#
+# anyway, make sure that we have a recent enough setuptools.
+$TOX_BIN/pip install 'setuptools>=18.5'
+
+# we also need a semi-recent version of pip, because old ones fail to install
+# the "enum34" dependency of cryptography.
+$TOX_BIN/pip install 'pip>=10'
+
+{ python synapse/python_dependencies.py
+  echo lxml
+} | xargs $TOX_BIN/pip install