Merge branch 'develop' into bwindels/registerasregularuser

.buildkite/.env

@@ -1,13 +0,0 @@
-CI
-BUILDKITE
-BUILDKITE_BUILD_NUMBER
-BUILDKITE_BRANCH
-BUILDKITE_BUILD_NUMBER
-BUILDKITE_JOB_ID
-BUILDKITE_BUILD_URL
-BUILDKITE_PROJECT_SLUG
-BUILDKITE_COMMIT
-BUILDKITE_PULL_REQUEST
-BUILDKITE_TAG
-CODECOV_TOKEN
-TRIAL_FLAGS

.buildkite/docker-compose.py27.pg94.yaml

@@ -1,21 +0,0 @@
-version: '3.1'
-
-services:
-
-  postgres:
-    image: postgres:9.4
-    environment:
-      POSTGRES_PASSWORD: postgres
-
-  testenv:
-    image: python:2.7
-    depends_on:
-      - postgres
-    env_file: .env
-    environment:
-      SYNAPSE_POSTGRES_HOST: postgres
-      SYNAPSE_POSTGRES_USER: postgres
-      SYNAPSE_POSTGRES_PASSWORD: postgres
-    working_dir: /app
-    volumes:
-      - ..:/app

.buildkite/docker-compose.py27.pg95.yaml

@@ -1,21 +0,0 @@
-version: '3.1'
-
-services:
-
-  postgres:
-    image: postgres:9.5
-    environment:
-      POSTGRES_PASSWORD: postgres
-
-  testenv:
-    image: python:2.7
-    depends_on:
-      - postgres
-    env_file: .env
-    environment:
-      SYNAPSE_POSTGRES_HOST: postgres
-      SYNAPSE_POSTGRES_USER: postgres
-      SYNAPSE_POSTGRES_PASSWORD: postgres
-    working_dir: /app
-    volumes:
-      - ..:/app

.buildkite/docker-compose.py35.pg94.yaml

@@ -1,21 +0,0 @@
-version: '3.1'
-
-services:
-
-  postgres:
-    image: postgres:9.4
-    environment:
-      POSTGRES_PASSWORD: postgres
-
-  testenv:
-    image: python:3.5
-    depends_on:
-      - postgres
-    env_file: .env
-    environment:
-      SYNAPSE_POSTGRES_HOST: postgres
-      SYNAPSE_POSTGRES_USER: postgres
-      SYNAPSE_POSTGRES_PASSWORD: postgres
-    working_dir: /app
-    volumes:
-      - ..:/app

.buildkite/docker-compose.py35.pg95.yaml

@@ -1,21 +0,0 @@
-version: '3.1'
-
-services:
-
-  postgres:
-    image: postgres:9.5
-    environment:
-      POSTGRES_PASSWORD: postgres
-
-  testenv:
-    image: python:3.5
-    depends_on:
-      - postgres
-    env_file: .env
-    environment:
-      SYNAPSE_POSTGRES_HOST: postgres
-      SYNAPSE_POSTGRES_USER: postgres
-      SYNAPSE_POSTGRES_PASSWORD: postgres
-    working_dir: /app
-    volumes:
-      - ..:/app

.buildkite/docker-compose.py37.pg11.yaml

@@ -1,21 +0,0 @@
-version: '3.1'
-
-services:
-
-  postgres:
-    image: postgres:11
-    environment:
-      POSTGRES_PASSWORD: postgres
-
-  testenv:
-    image: python:3.7
-    depends_on:
-      - postgres
-    env_file: .env
-    environment:
-      SYNAPSE_POSTGRES_HOST: postgres
-      SYNAPSE_POSTGRES_USER: postgres
-      SYNAPSE_POSTGRES_PASSWORD: postgres
-    working_dir: /app
-    volumes:
-      - ..:/app

.buildkite/docker-compose.py37.pg95.yaml

@@ -1,21 +0,0 @@
-version: '3.1'
-
-services:
-
-  postgres:
-    image: postgres:9.5
-    environment:
-      POSTGRES_PASSWORD: postgres
-
-  testenv:
-    image: python:3.7
-    depends_on:
-      - postgres
-    env_file: .env
-    environment:
-      SYNAPSE_POSTGRES_HOST: postgres
-      SYNAPSE_POSTGRES_USER: postgres
-      SYNAPSE_POSTGRES_PASSWORD: postgres
-    working_dir: /app
-    volumes:
-      - ..:/app

.buildkite/pipeline.yml

@@ -1,234 +0,0 @@
-env:
-  CODECOV_TOKEN: "2dd7eb9b-0eda-45fe-a47c-9b5ac040045f"
-
-steps:
-  - command:
-      - "python -m pip install tox"
-      - "tox -e pep8"
-    label: "\U0001F9F9 PEP-8"
-    plugins:
-      - docker#v3.0.1:
-          image: "python:3.6"
-
-  - command:
-      - "python -m pip install tox"
-      - "tox -e packaging"
-    label: "\U0001F9F9 packaging"
-    plugins:
-      - docker#v3.0.1:
-          image: "python:3.6"
-
-  - command:
-      - "python -m pip install tox"
-      - "tox -e check_isort"
-    label: "\U0001F9F9 isort"
-    plugins:
-      - docker#v3.0.1:
-          image: "python:3.6"
-
-  - command:
-      - "python -m pip install tox"
-      - "scripts-dev/check-newsfragment"
-    label: ":newspaper: Newsfile"
-    branches: "!master !develop !release-*"
-    plugins:
-      - docker#v3.0.1:
-          image: "python:3.6"
-          propagate-environment: true
-
-  - command:
-      - "python -m pip install tox"
-      - "tox -e check-sampleconfig"
-    label: "\U0001F9F9 check-sample-config"
-    plugins:
-      - docker#v3.0.1:
-          image: "python:3.6"
-
-  - wait
-
-  - command:
-      - "python -m pip install tox"
-      - "tox -e py27,codecov"
-    label: ":python: 2.7 / SQLite"
-    env:
-      TRIAL_FLAGS: "-j 2"
-    plugins:
-      - docker#v3.0.1:
-          image: "python:2.7"
-          propagate-environment: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
-
-  - command:
-      - "python -m pip install tox"
-      - "tox -e py35,codecov"
-    label: ":python: 3.5 / SQLite"
-    env:
-      TRIAL_FLAGS: "-j 2"
-    plugins:
-      - docker#v3.0.1:
-          image: "python:3.5"
-          propagate-environment: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
-
-  - command:
-      - "python -m pip install tox"
-      - "tox -e py36,codecov"
-    label: ":python: 3.6 / SQLite"
-    env:
-      TRIAL_FLAGS: "-j 2"
-    plugins:
-      - docker#v3.0.1:
-          image: "python:3.6"
-          propagate-environment: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
-
-  - command:
-      - "python -m pip install tox"
-      - "tox -e py37,codecov"
-    label: ":python: 3.7 / SQLite"
-    env:
-      TRIAL_FLAGS: "-j 2"
-    plugins:
-      - docker#v3.0.1:
-          image: "python:3.7"
-          propagate-environment: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
-
-  - command:
-      - "python -m pip install tox"
-      - "tox -e py27-old,codecov"
-    label: ":python: 2.7 / SQLite / Old Deps"
-    env:
-      TRIAL_FLAGS: "-j 2"
-    plugins:
-      - docker#v3.0.1:
-          image: "python:2.7"
-          propagate-environment: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
-
-  - label: ":python: 2.7 / :postgres: 9.4"
-    env:
-      TRIAL_FLAGS: "-j 4"
-    command:
-      - "bash -c 'python -m pip install tox && python -m tox -e py27-postgres,codecov'"
-    plugins:
-      - docker-compose#v2.1.0:
-          run: testenv
-          config:
-            - .buildkite/docker-compose.py27.pg94.yaml
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
-
-  - label: ":python: 2.7 / :postgres: 9.5"
-    env:
-      TRIAL_FLAGS: "-j 4"
-    command:
-      - "bash -c 'python -m pip install tox && python -m tox -e py27-postgres,codecov'"
-    plugins:
-      - docker-compose#v2.1.0:
-          run: testenv
-          config:
-            - .buildkite/docker-compose.py27.pg95.yaml
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
-
-  - label: ":python: 3.5 / :postgres: 9.4"
-    env:
-      TRIAL_FLAGS: "-j 4"
-    command:
-      - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'"
-    plugins:
-      - docker-compose#v2.1.0:
-          run: testenv
-          config:
-            - .buildkite/docker-compose.py35.pg94.yaml
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
-
-  - label: ":python: 3.5 / :postgres: 9.5"
-    env:
-      TRIAL_FLAGS: "-j 4"
-    command:
-      - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'"
-    plugins:
-      - docker-compose#v2.1.0:
-          run: testenv
-          config:
-            - .buildkite/docker-compose.py35.pg95.yaml
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
-
-  - label: ":python: 3.7 / :postgres: 9.5"
-    env:
-      TRIAL_FLAGS: "-j 4"
-    command:
-      - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
-    plugins:
-      - docker-compose#v2.1.0:
-          run: testenv
-          config:
-            - .buildkite/docker-compose.py37.pg95.yaml
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
-
-  - label: ":python: 3.7 / :postgres: 11"
-    env:
-      TRIAL_FLAGS: "-j 4"
-    command:
-      - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
-    plugins:
-      - docker-compose#v2.1.0:
-          run: testenv
-          config:
-            - .buildkite/docker-compose.py37.pg11.yaml
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2

.circleci/config.yml

@@ -1,24 +1,159 @@
 version: 2
 jobs:
-  sytestpy2postgresmerged:
-    docker:
-      - image: matrixdotorg/sytest-synapse:dinsic
-    working_directory: /src
+  dockerhubuploadrelease:
+    machine: true
+    steps:
+      - checkout
+      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:$CIRCLE_TAG .
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      - run: docker push matrixdotorg/synapse:$CIRCLE_TAG
+  dockerhubuploadlatest:
+    machine: true
+    steps:
+      - checkout
+      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:$CIRCLE_SHA1 .
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      - run: docker tag matrixdotorg/synapse:$CIRCLE_SHA1 matrixdotorg/synapse:latest
+      - run: docker push matrixdotorg/synapse:$CIRCLE_SHA1
+      - run: docker push matrixdotorg/synapse:latest
+  sytestpy2:
+    machine: true
+    steps:
+      - checkout
+      - run: docker pull matrixdotorg/sytest-synapsepy2
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs
+  sytestpy2postgres:
+    machine: true
+    steps:
+      - checkout
+      - run: docker pull matrixdotorg/sytest-synapsepy2
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs
+  sytestpy2merged:
+    machine: true
     steps:
       - checkout
       - run: bash .circleci/merge_base_branch.sh
-      - run: POSTGRES=1 /synapse_sytest.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy2
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2
       - store_artifacts:
-          path: /logs
+          path: ~/project/logs
           destination: logs
       - store_test_results:
-          path: /logs
+          path: logs
+
+  sytestpy2postgresmerged:
+    machine: true
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy2
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs
+
+  sytestpy3:
+    machine: true
+    steps:
+      - checkout
+      - run: docker pull matrixdotorg/sytest-synapsepy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs
+  sytestpy3postgres:
+    machine: true
+    steps:
+      - checkout
+      - run: docker pull matrixdotorg/sytest-synapsepy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs
+  sytestpy3merged:
+    machine: true
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs
+  sytestpy3postgresmerged:
+    machine: true
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs
 
 workflows:
   version: 2
   build:
     jobs:
+      - sytestpy2:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy2postgres:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy3:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy3postgres:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy2merged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
       - sytestpy2postgresmerged:
           filters:
             branches:
-              ignore: /develop|master|release-.*/
+              ignore: /develop|master|release-.*/
+      - sytestpy3merged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
+      - sytestpy3postgresmerged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
+      - dockerhubuploadrelease:
+          filters:
+            tags:
+              only: /^v[0-9].[0-9]+.[0-9]+(.[0-9]+)?/
+            branches:
+              ignore: /.*/
+      - dockerhubuploadlatest:
+          filters:
+            branches:
+              only: master

.circleci/merge_base_branch.sh

@@ -9,19 +9,15 @@ source $BASH_ENV
 
 if [[ -z "${CIRCLE_PR_NUMBER}" ]]
 then
-    echo "Can't figure out what the PR number is! Assuming merge target is dinsic."
-
-    # It probably hasn't had a PR opened yet. Since all PRs for dinsic land on
-    # dinsic, we can probably assume it's based on it and will be merged into
-    # it.
-    GITBASE="dinsic"
-else
-    # Get the reference, using the GitHub API
-    GITBASE=`wget -O- https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
+    echo "Can't figure out what the PR number is!"
+    exit 1
 fi
 
+# Get the reference, using the GitHub API
+GITBASE=`curl -q https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
+
 # Show what we are before
-git --no-pager show -s
+git show -s
 
 # Set up username so it can do a merge
 git config --global user.email bot@matrix.org
@@ -32,4 +28,4 @@ git fetch -u origin $GITBASE
 git merge --no-edit origin/$GITBASE
 
 # Show what we are after.
-git --no-pager show -s
+git show -s

.codecov.yml

@@ -1,15 +0,0 @@
-comment:
-  layout: "diff"
-
-coverage:
-  status:
-    project:
-      default:
-        target: 0  # Target % coverage, can be auto. Turned off for now
-        threshold: null
-        base: auto
-    patch:
-      default:
-        target: 0
-        threshold: null
-        base: auto

.coveragerc

@@ -1,7 +0,0 @@
-[run]
-branch = True
-parallel = True
-include = synapse/*
-
-[report]
-precision = 2

.dockerignore

@@ -5,5 +5,3 @@ demo/etc
 tox.ini
 .git/*
 .tox/*
-debian/matrix-synapse/
-debian/matrix-synapse-*/

.editorconfig

@@ -1,9 +0,0 @@
-# EditorConfig https://EditorConfig.org
-
-# top-most EditorConfig file
-root = true
-
-# 4 space indentation
-[*.py]
-indent_style = space
-indent_size = 4

.github/ISSUE_TEMPLATE.md

@@ -1,12 +1,6 @@
----
-name: Bug report
-about: Create a report to help us improve
+<!-- 
 
----
-
-<!--
-
-**IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**:
+**IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**: 
 You will likely get better support more quickly if you ask in ** #matrix:matrix.org ** ;)
 
 
@@ -23,44 +17,32 @@ Text between <!-- and --​> marks will be invisible in the report.
 
 ### Description
 
-<!-- Describe here the problem that you are experiencing -->
+Describe here the problem that you are experiencing, or the feature you are requesting.
 
 ### Steps to reproduce
 
-- list the steps
+- For bugs, list the steps
 - that reproduce the bug
 - using hyphens as bullet points
 
-<!--
 Describe how what happens differs from what you expected.
 
-If you can identify any relevant log snippets from _homeserver.log_, please include
+<!-- If you can identify any relevant log snippets from _homeserver.log_, please include
 those (please be careful to remove any personal or private data). Please surround them with
-``` (three backticks, on a line on their own), so that they are formatted legibly.
--->
+``` (three backticks, on a line on their own), so that they are formatted legibly. -->
 
 ### Version information
 
 <!-- IMPORTANT: please answer the following questions, to help us narrow down the problem -->
 
-<!-- Was this issue identified on matrix.org or another homeserver? -->
-- **Homeserver**: 
+- **Homeserver**: Was this issue identified on matrix.org or another homeserver?
 
 If not matrix.org:
-
-<!--
-What version of Synapse is running?
+- **Version**:        What version of Synapse is running? <!-- 
 You can find the Synapse version by inspecting the server headers (replace matrix.org with
 your own homeserver domain):
 $ curl -v https://matrix.org/_matrix/client/versions 2>&1 | grep "Server:"
 -->
-- **Version**: 
-
-- **Install method**: 
-<!-- examples: package manager/git clone/pip  -->
-
-- **Platform**: 
-<!--
-Tell us about the environment in which your homeserver is operating
-distro, hardware, if it's running in a vm/container, etc.
--->
+- **Install method**: package manager/git clone/pip      
+- **Platform**:       Tell us about the environment in which your homeserver is operating
+                      - distro, hardware, if it's running in a vm/container, etc.

.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md

@@ -1,9 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-
----
-
-**Description:**
-
-<!-- Describe here the feature you are requesting. -->

.github/ISSUE_TEMPLATE/SUPPORT_REQUEST.md

@@ -1,9 +0,0 @@
----
-name: Support request
-about: I need support for Synapse
-
----
-
-# Please ask for support in [**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org)
-
-## Don't file an issue as a support request.

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +0,0 @@
-### Pull Request Checklist
-
-<!-- Please read CONTRIBUTING.rst before submitting your pull request -->
-
-* [ ] Pull request is based on the develop branch
-* [ ] Pull request includes a [changelog file](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst#changelog)
-* [ ] Pull request includes a [sign off](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst#sign-off)

.github/SUPPORT.md

@@ -1,3 +0,0 @@
-[**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org) is the official support room for Matrix, and can be accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html 
-
-It can also be access via IRC bridge at irc://irc.freenode.net/matrix or on the web here: https://webchat.freenode.net/?channels=matrix

.gitignore

@@ -1,40 +1,59 @@
-# filename patterns
-*~
-.*.swp
-.#*
-*.deb
-*.egg
-*.egg-info
-*.lock
 *.pyc
-*.tac
+.*.swp
+*~
+*.lock
+
+.DS_Store
 _trial_temp/
 _trial_temp*/
+logs/
+dbs/
+*.egg
+dist/
+docs/build/
+*.egg-info
 
-# stuff that is likely to exist when you run a server locally
-/*.db
-/*.log
-/*.log.config
-/*.pid
-/*.signing.key
-/env/
-/homeserver*.yaml
-/media_store/
-/uploads
+cmdclient_config.json
+homeserver*.db
+homeserver*.log
+homeserver*.log.*
+homeserver*.pid
+homeserver*.yaml
 
-# IDEs
-/.idea/
-/.ropeproject/
-/.vscode/
+*.signing.key
+*.tls.crt
+*.tls.dh
+*.tls.key
 
-# build products
-/.coverage*
-!/.coveragerc
-/.tox
-/build/
-/coverage.*
-/dist/
-/docs/build/
-/htmlcov
-/pip-wheel-metadata/
+.coverage
+htmlcov
 
+demo/*/*.db
+demo/*/*.log
+demo/*/*.log.*
+demo/*/*.pid
+demo/media_store.*
+demo/etc
+
+uploads
+cache
+
+.idea/
+media_store/
+
+*.tac
+
+build/
+venv/
+venv*/
+*venv/
+
+localhost-800*/
+static/client/register/register_config.js
+.tox
+
+env/
+*.config
+
+.vscode/
+.ropeproject/

.travis.yml

@@ -0,0 +1,52 @@
+sudo: false
+language: python
+
+# tell travis to cache ~/.cache/pip
+cache: pip
+
+before_script:
+  - git remote set-branches --add origin develop
+  - git fetch origin develop
+
+matrix:
+  fast_finish: true
+  include:
+  - python: 2.7
+    env: TOX_ENV=packaging
+
+  - python: 2.7
+    env: TOX_ENV=pep8
+
+  - python: 2.7
+    env: TOX_ENV=py27
+
+  - python: 2.7
+    env: TOX_ENV=py27-old
+
+  - python: 2.7
+    env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
+    services:
+      - postgresql
+
+  - python: 3.5
+    env: TOX_ENV=py35
+
+  - python: 3.6
+    env: TOX_ENV=py36
+
+  - python: 3.6
+    env: TOX_ENV=py36-postgres TRIAL_FLAGS="-j 4"
+    services:
+      - postgresql
+
+  - python: 3.6
+    env: TOX_ENV=check_isort
+
+  - python: 3.6
+    env: TOX_ENV=check-newsfragment
+
+install:
+  - pip install tox
+
+script:
+  - tox -e $TOX_ENV

AUTHORS.rst

@@ -65,10 +65,4 @@ Pierre Jaury <pierre at jaury.eu>
 * Docker packaging
 
 Serban Constantin <serban.constantin at gmail dot com>
- * Small bug fix
-
-Jason Robinson <jasonr at matrix.org>
- * Minor fixes
-
-Joseph Weston <joseph at weston.cloud>
- + Add admin API for querying HS version
+ * Small bug fix

CHANGES.md

@@ -1,994 +1,11 @@
-Synapse 1.0.0 (2019-06-11)
-==========================
-
-Bugfixes
---------
-
-- Fix bug where attempting to send transactions with large number of EDUs can fail. ([\#5418](https://github.com/matrix-org/synapse/issues/5418))
-
-
-Improved Documentation
-----------------------
-
-- Expand the federation guide to include relevant content from the MSC1711 FAQ ([\#5419](https://github.com/matrix-org/synapse/issues/5419))
-
-
-Internal Changes
-----------------
-
-- Move password reset links to /_matrix/client/unstable namespace. ([\#5424](https://github.com/matrix-org/synapse/issues/5424))
-
-
-Synapse 1.0.0rc3 (2019-06-10)
-=============================
-
-Security: Fix authentication bug introduced in 1.0.0rc1. Please upgrade to rc3 immediately
-
-
-Synapse 1.0.0rc2 (2019-06-10)
-=============================
-
-Bugfixes
---------
-
-- Remove redundant warning about key server response validation. ([\#5392](https://github.com/matrix-org/synapse/issues/5392))
-- Fix bug where old keys stored in the database with a null valid until timestamp caused all verification requests for that key to fail. ([\#5415](https://github.com/matrix-org/synapse/issues/5415))
-- Fix excessive memory using with default `federation_verify_certificates: true` configuration. ([\#5417](https://github.com/matrix-org/synapse/issues/5417))
-
-
-Synapse 1.0.0rc1 (2019-06-07)
-=============================
-
-Features
---------
-
-- Synapse now more efficiently collates room statistics. ([\#4338](https://github.com/matrix-org/synapse/issues/4338), [\#5260](https://github.com/matrix-org/synapse/issues/5260), [\#5324](https://github.com/matrix-org/synapse/issues/5324))
-- Add experimental support for relations (aka reactions and edits). ([\#5220](https://github.com/matrix-org/synapse/issues/5220))
-- Ability to configure default room version. ([\#5223](https://github.com/matrix-org/synapse/issues/5223), [\#5249](https://github.com/matrix-org/synapse/issues/5249))
-- Allow configuring a range for the account validity startup job. ([\#5276](https://github.com/matrix-org/synapse/issues/5276))
-- CAS login will now hit the r0 API, not the deprecated v1 one. ([\#5286](https://github.com/matrix-org/synapse/issues/5286))
-- Validate federation server TLS certificates by default (implements [MSC1711](https://github.com/matrix-org/matrix-doc/blob/master/proposals/1711-x509-for-federation.md)). ([\#5359](https://github.com/matrix-org/synapse/issues/5359))
-- Update /_matrix/client/versions to reference support for r0.5.0. ([\#5360](https://github.com/matrix-org/synapse/issues/5360))
-- Add a script to generate new signing-key files. ([\#5361](https://github.com/matrix-org/synapse/issues/5361))
-- Update upgrade and installation guides ahead of 1.0. ([\#5371](https://github.com/matrix-org/synapse/issues/5371))
-- Replace the `perspectives` configuration section with `trusted_key_servers`, and make validating the signatures on responses optional (since TLS will do this job for us). ([\#5374](https://github.com/matrix-org/synapse/issues/5374))
-- Add ability to perform password reset via email without trusting the identity server. ([\#5377](https://github.com/matrix-org/synapse/issues/5377))
-- Set default room version to v4. ([\#5379](https://github.com/matrix-org/synapse/issues/5379))
-
-
-Bugfixes
---------
-
-- Fixes client-server API not sending "m.heroes" to lazy-load /sync requests when a rooms name or its canonical alias are empty. Thanks to @dnaf for this work! ([\#5089](https://github.com/matrix-org/synapse/issues/5089))
-- Prevent federation device list updates breaking when processing multiple updates at once. ([\#5156](https://github.com/matrix-org/synapse/issues/5156))
-- Fix worker registration bug caused by ClientReaderSlavedStore being unable to see get_profileinfo. ([\#5200](https://github.com/matrix-org/synapse/issues/5200))
-- Fix race when backfilling in rooms with worker mode. ([\#5221](https://github.com/matrix-org/synapse/issues/5221))
-- Fix appservice timestamp massaging. ([\#5233](https://github.com/matrix-org/synapse/issues/5233))
-- Ensure that server_keys fetched via a notary server are correctly signed. ([\#5251](https://github.com/matrix-org/synapse/issues/5251))
-- Show the correct error when logging out and access token is missing. ([\#5256](https://github.com/matrix-org/synapse/issues/5256))
-- Fix error code when there is an invalid parameter on /_matrix/client/r0/publicRooms ([\#5257](https://github.com/matrix-org/synapse/issues/5257))
-- Fix error when downloading thumbnail with missing width/height parameter. ([\#5258](https://github.com/matrix-org/synapse/issues/5258))
-- Fix schema update for account validity. ([\#5268](https://github.com/matrix-org/synapse/issues/5268))
-- Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. ([\#5274](https://github.com/matrix-org/synapse/issues/5274), [\#5278](https://github.com/matrix-org/synapse/issues/5278), [\#5291](https://github.com/matrix-org/synapse/issues/5291))
-- Fix "db txn 'update_presence' from sentinel context" log messages. ([\#5275](https://github.com/matrix-org/synapse/issues/5275))
-- Fix dropped logcontexts during high outbound traffic. ([\#5277](https://github.com/matrix-org/synapse/issues/5277))
-- Fix a bug where it is not possible to get events in the federation format with the request `GET /_matrix/client/r0/rooms/{roomId}/messages`. ([\#5293](https://github.com/matrix-org/synapse/issues/5293))
-- Fix performance problems with the rooms stats background update. ([\#5294](https://github.com/matrix-org/synapse/issues/5294))
-- Fix noisy 'no key for server' logs. ([\#5300](https://github.com/matrix-org/synapse/issues/5300))
-- Fix bug where a notary server would sometimes forget old keys. ([\#5307](https://github.com/matrix-org/synapse/issues/5307))
-- Prevent users from setting huge displaynames and avatar URLs. ([\#5309](https://github.com/matrix-org/synapse/issues/5309))
-- Fix handling of failures when processing incoming events where calling `/event_auth` on remote server fails. ([\#5317](https://github.com/matrix-org/synapse/issues/5317))
-- Ensure that we have an up-to-date copy of the signing key when validating incoming federation requests. ([\#5321](https://github.com/matrix-org/synapse/issues/5321))
-- Fix various problems which made the signing-key notary server time out for some requests. ([\#5333](https://github.com/matrix-org/synapse/issues/5333))
-- Fix bug which would make certain operations (such as room joins) block for 20 minutes while attemoting to fetch verification keys. ([\#5334](https://github.com/matrix-org/synapse/issues/5334))
-- Fix a bug where we could rapidly mark a server as unreachable even though it was only down for a few minutes. ([\#5335](https://github.com/matrix-org/synapse/issues/5335), [\#5340](https://github.com/matrix-org/synapse/issues/5340))
-- Fix a bug where account validity renewal emails could only be sent when email notifs were enabled. ([\#5341](https://github.com/matrix-org/synapse/issues/5341))
-- Fix failure when fetching batches of events during backfill, etc. ([\#5342](https://github.com/matrix-org/synapse/issues/5342))
-- Add a new room version where the timestamps on events are checked against the validity periods on signing keys. ([\#5348](https://github.com/matrix-org/synapse/issues/5348), [\#5354](https://github.com/matrix-org/synapse/issues/5354))
-- Fix room stats and presence background updates to correctly handle missing events. ([\#5352](https://github.com/matrix-org/synapse/issues/5352))
-- Include left members in room summaries' heroes. ([\#5355](https://github.com/matrix-org/synapse/issues/5355))
-- Fix `federation_custom_ca_list` configuration option. ([\#5362](https://github.com/matrix-org/synapse/issues/5362))
-- Fix missing logcontext warnings on shutdown. ([\#5369](https://github.com/matrix-org/synapse/issues/5369))
-
-
-Improved Documentation
-----------------------
-
-- Fix docs on resetting the user directory. ([\#5282](https://github.com/matrix-org/synapse/issues/5282))
-- Fix notes about ACME in the MSC1711 faq. ([\#5357](https://github.com/matrix-org/synapse/issues/5357))
-
-
-Internal Changes
-----------------
-
-- Synapse will now serve the experimental "room complexity" API endpoint. ([\#5216](https://github.com/matrix-org/synapse/issues/5216))
-- The base classes for the v1 and v2_alpha REST APIs have been unified. ([\#5226](https://github.com/matrix-org/synapse/issues/5226), [\#5328](https://github.com/matrix-org/synapse/issues/5328))
-- Simplifications and comments in do_auth. ([\#5227](https://github.com/matrix-org/synapse/issues/5227))
-- Remove urllib3 pin as requests 2.22.0 has been released supporting urllib3 1.25.2. ([\#5230](https://github.com/matrix-org/synapse/issues/5230))
-- Preparatory work for key-validity features. ([\#5232](https://github.com/matrix-org/synapse/issues/5232), [\#5234](https://github.com/matrix-org/synapse/issues/5234), [\#5235](https://github.com/matrix-org/synapse/issues/5235), [\#5236](https://github.com/matrix-org/synapse/issues/5236), [\#5237](https://github.com/matrix-org/synapse/issues/5237), [\#5244](https://github.com/matrix-org/synapse/issues/5244), [\#5250](https://github.com/matrix-org/synapse/issues/5250), [\#5296](https://github.com/matrix-org/synapse/issues/5296), [\#5299](https://github.com/matrix-org/synapse/issues/5299), [\#5343](https://github.com/matrix-org/synapse/issues/5343), [\#5347](https://github.com/matrix-org/synapse/issues/5347), [\#5356](https://github.com/matrix-org/synapse/issues/5356))
-- Specify the type of reCAPTCHA key to use. ([\#5283](https://github.com/matrix-org/synapse/issues/5283))
-- Improve sample config for monthly active user blocking. ([\#5284](https://github.com/matrix-org/synapse/issues/5284))
-- Remove spurious debug from MatrixFederationHttpClient.get_json. ([\#5287](https://github.com/matrix-org/synapse/issues/5287))
-- Improve logging for logcontext leaks. ([\#5288](https://github.com/matrix-org/synapse/issues/5288))
-- Clarify that the admin change password API logs the user out. ([\#5303](https://github.com/matrix-org/synapse/issues/5303))
-- New installs will now use the v54 full schema, rather than the full schema v14 and applying incremental updates to v54. ([\#5320](https://github.com/matrix-org/synapse/issues/5320))
-- Improve docstrings on MatrixFederationClient. ([\#5332](https://github.com/matrix-org/synapse/issues/5332))
-- Clean up FederationClient.get_events for clarity. ([\#5344](https://github.com/matrix-org/synapse/issues/5344))
-- Various improvements to debug logging. ([\#5353](https://github.com/matrix-org/synapse/issues/5353))
-- Don't run CI build checks until sample config check has passed. ([\#5370](https://github.com/matrix-org/synapse/issues/5370))
-- Automatically retry buildkite builds (max twice) when an agent is lost. ([\#5380](https://github.com/matrix-org/synapse/issues/5380))
-
-
-Synapse 0.99.5.2 (2019-05-30)
-=============================
-
-Bugfixes
---------
-
-- Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. ([\#5274](https://github.com/matrix-org/synapse/issues/5274), [\#5278](https://github.com/matrix-org/synapse/issues/5278), [\#5291](https://github.com/matrix-org/synapse/issues/5291))
-
-
-Synapse 0.99.5.1 (2019-05-22)
-=============================
-
-0.99.5.1 supersedes 0.99.5 due to malformed debian changelog - no functional changes.
-
-Synapse 0.99.5 (2019-05-22)
-===========================
-
-No significant changes.
-
-
-Synapse 0.99.5rc1 (2019-05-21)
-==============================
-
-Features
---------
-
-- Add ability to blacklist IP ranges for the federation client. ([\#5043](https://github.com/matrix-org/synapse/issues/5043))
-- Ratelimiting configuration for clients sending messages and the federation server has been altered to match login ratelimiting. The old configuration names will continue working. Check the sample config for details of the new names. ([\#5181](https://github.com/matrix-org/synapse/issues/5181))
-- Drop support for the undocumented /_matrix/client/v2_alpha API prefix. ([\#5190](https://github.com/matrix-org/synapse/issues/5190))
-- Add an option to disable per-room profiles. ([\#5196](https://github.com/matrix-org/synapse/issues/5196))
-- Stick an expiration date to any registered user missing one at startup if account validity is enabled. ([\#5204](https://github.com/matrix-org/synapse/issues/5204))
-- Add experimental support for relations (aka reactions and edits). ([\#5209](https://github.com/matrix-org/synapse/issues/5209), [\#5211](https://github.com/matrix-org/synapse/issues/5211), [\#5203](https://github.com/matrix-org/synapse/issues/5203), [\#5212](https://github.com/matrix-org/synapse/issues/5212))
-- Add a room version 4 which uses a new event ID format, as per [MSC2002](https://github.com/matrix-org/matrix-doc/pull/2002). ([\#5210](https://github.com/matrix-org/synapse/issues/5210), [\#5217](https://github.com/matrix-org/synapse/issues/5217))
-
-
-Bugfixes
---------
-
-- Fix image orientation when generating thumbnails (needs pillow>=4.3.0). Contributed by Pau Rodriguez-Estivill. ([\#5039](https://github.com/matrix-org/synapse/issues/5039))
-- Exclude soft-failed events from forward-extremity candidates: fixes "No forward extremities left!" error. ([\#5146](https://github.com/matrix-org/synapse/issues/5146))
-- Re-order stages in registration flows such that msisdn and email verification are done last. ([\#5174](https://github.com/matrix-org/synapse/issues/5174))
-- Fix 3pid guest invites. ([\#5177](https://github.com/matrix-org/synapse/issues/5177))
-- Fix a bug where the register endpoint would fail with M_THREEPID_IN_USE instead of returning an account previously registered in the same session. ([\#5187](https://github.com/matrix-org/synapse/issues/5187))
-- Prevent registration for user ids that are too long to fit into a state key. Contributed by Reid Anderson. ([\#5198](https://github.com/matrix-org/synapse/issues/5198))
-- Fix incompatibility between ACME support and Python 3.5.2. ([\#5218](https://github.com/matrix-org/synapse/issues/5218))
-- Fix error handling for rooms whose versions are unknown. ([\#5219](https://github.com/matrix-org/synapse/issues/5219))
-
-
-Internal Changes
-----------------
-
-- Make /sync attempt to return device updates for both joined and invited users. Note that this doesn't currently work correctly due to other bugs. ([\#3484](https://github.com/matrix-org/synapse/issues/3484))
-- Update tests to consistently be configured via the same code that is used when loading from configuration files. ([\#5171](https://github.com/matrix-org/synapse/issues/5171), [\#5185](https://github.com/matrix-org/synapse/issues/5185))
-- Allow client event serialization to be async. ([\#5183](https://github.com/matrix-org/synapse/issues/5183))
-- Expose DataStore._get_events as get_events_as_list. ([\#5184](https://github.com/matrix-org/synapse/issues/5184))
-- Make generating SQL bounds for pagination generic. ([\#5191](https://github.com/matrix-org/synapse/issues/5191))
-- Stop telling people to install the optional dependencies by default. ([\#5197](https://github.com/matrix-org/synapse/issues/5197))
-
-
-Synapse 0.99.4 (2019-05-15)
-===========================
-
-No significant changes.
-
-
-Synapse 0.99.4rc1 (2019-05-13)
-==============================
-
-Features
---------
-
-- Add systemd-python to the optional dependencies to enable logging to the systemd journal. Install with `pip install matrix-synapse[systemd]`. ([\#4339](https://github.com/matrix-org/synapse/issues/4339))
-- Add a default .m.rule.tombstone push rule. ([\#4867](https://github.com/matrix-org/synapse/issues/4867))
-- Add ability for password provider modules to bind email addresses to users upon registration. ([\#4947](https://github.com/matrix-org/synapse/issues/4947))
-- Implementation of [MSC1711](https://github.com/matrix-org/matrix-doc/pull/1711) including config options for requiring valid TLS certificates for federation traffic, the ability to disable TLS validation for specific domains, and the ability to specify your own list of CA certificates. ([\#4967](https://github.com/matrix-org/synapse/issues/4967))
-- Remove presence list support as per MSC 1819. ([\#4989](https://github.com/matrix-org/synapse/issues/4989))
-- Reduce CPU usage starting pushers during start up. ([\#4991](https://github.com/matrix-org/synapse/issues/4991))
-- Add a delete group admin API. ([\#5002](https://github.com/matrix-org/synapse/issues/5002))
-- Add config option to block users from looking up 3PIDs. ([\#5010](https://github.com/matrix-org/synapse/issues/5010))
-- Add context to phonehome stats. ([\#5020](https://github.com/matrix-org/synapse/issues/5020))
-- Configure the example systemd units to have a log identifier of `matrix-synapse`
-  instead of the executable name, `python`.
-  Contributed by Christoph Müller. ([\#5023](https://github.com/matrix-org/synapse/issues/5023))
-- Add time-based account expiration. ([\#5027](https://github.com/matrix-org/synapse/issues/5027), [\#5047](https://github.com/matrix-org/synapse/issues/5047), [\#5073](https://github.com/matrix-org/synapse/issues/5073), [\#5116](https://github.com/matrix-org/synapse/issues/5116))
-- Add support for handling `/versions`, `/voip` and `/push_rules` client endpoints to client_reader worker. ([\#5063](https://github.com/matrix-org/synapse/issues/5063), [\#5065](https://github.com/matrix-org/synapse/issues/5065), [\#5070](https://github.com/matrix-org/synapse/issues/5070))
-- Add a configuration option to require authentication on /publicRooms and /profile endpoints. ([\#5083](https://github.com/matrix-org/synapse/issues/5083))
-- Move admin APIs to `/_synapse/admin/v1`. (The old paths are retained for backwards-compatibility, for now). ([\#5119](https://github.com/matrix-org/synapse/issues/5119))
-- Implement an admin API for sending server notices. Many thanks to @krombel who provided a foundation for this work. ([\#5121](https://github.com/matrix-org/synapse/issues/5121), [\#5142](https://github.com/matrix-org/synapse/issues/5142))
-
-
-Bugfixes
---------
-
-- Avoid redundant URL encoding of redirect URL for SSO login in the fallback login page. Fixes a regression introduced in [#4220](https://github.com/matrix-org/synapse/pull/4220). Contributed by Marcel Fabian Krüger ("[zaugin](https://github.com/zauguin)"). ([\#4555](https://github.com/matrix-org/synapse/issues/4555))
-- Fix bug where presence updates were sent to all servers in a room when a new server joined, rather than to just the new server. ([\#4942](https://github.com/matrix-org/synapse/issues/4942), [\#5103](https://github.com/matrix-org/synapse/issues/5103))
-- Fix sync bug which made accepting invites unreliable in worker-mode synapses. ([\#4955](https://github.com/matrix-org/synapse/issues/4955), [\#4956](https://github.com/matrix-org/synapse/issues/4956))
-- start.sh: Fix the --no-rate-limit option for messages and make it bypass rate limit on registration and login too. ([\#4981](https://github.com/matrix-org/synapse/issues/4981))
-- Transfer related groups on room upgrade. ([\#4990](https://github.com/matrix-org/synapse/issues/4990))
-- Prevent the ability to kick users from a room they aren't in. ([\#4999](https://github.com/matrix-org/synapse/issues/4999))
-- Fix issue #4596 so synapse_port_db script works with --curses option on Python 3. Contributed by Anders Jensen-Waud <anders@jensenwaud.com>. ([\#5003](https://github.com/matrix-org/synapse/issues/5003))
-- Clients timing out/disappearing while downloading from the media repository will now no longer log a spurious "Producer was not unregistered" message. ([\#5009](https://github.com/matrix-org/synapse/issues/5009))
-- Fix "cannot import name execute_batch" error with postgres. ([\#5032](https://github.com/matrix-org/synapse/issues/5032))
-- Fix disappearing exceptions in manhole. ([\#5035](https://github.com/matrix-org/synapse/issues/5035))
-- Workaround bug in twisted where attempting too many concurrent DNS requests could cause it to hang due to running out of file descriptors. ([\#5037](https://github.com/matrix-org/synapse/issues/5037))
-- Make sure we're not registering the same 3pid twice on registration. ([\#5071](https://github.com/matrix-org/synapse/issues/5071))
-- Don't crash on lack of expiry templates. ([\#5077](https://github.com/matrix-org/synapse/issues/5077))
-- Fix the ratelimiting on third party invites. ([\#5104](https://github.com/matrix-org/synapse/issues/5104))
-- Add some missing limitations to room alias creation. ([\#5124](https://github.com/matrix-org/synapse/issues/5124), [\#5128](https://github.com/matrix-org/synapse/issues/5128))
-- Limit the number of EDUs in transactions to 100 as expected by synapse. Thanks to @superboum for this work! ([\#5138](https://github.com/matrix-org/synapse/issues/5138))
-
-Internal Changes
-----------------
-
-- Add test to verify threepid auth check added in #4435. ([\#4474](https://github.com/matrix-org/synapse/issues/4474))
-- Fix/improve some docstrings in the replication code. ([\#4949](https://github.com/matrix-org/synapse/issues/4949))
-- Split synapse.replication.tcp.streams into smaller files. ([\#4953](https://github.com/matrix-org/synapse/issues/4953))
-- Refactor replication row generation/parsing. ([\#4954](https://github.com/matrix-org/synapse/issues/4954))
-- Run `black` to clean up formatting on `synapse/storage/roommember.py` and `synapse/storage/events.py`. ([\#4959](https://github.com/matrix-org/synapse/issues/4959))
-- Remove log line for password via the admin API. ([\#4965](https://github.com/matrix-org/synapse/issues/4965))
-- Fix typo in TLS filenames in docker/README.md. Also add the '-p' commandline option to the 'docker run' example. Contributed by Jurrie Overgoor. ([\#4968](https://github.com/matrix-org/synapse/issues/4968))
-- Refactor room version definitions. ([\#4969](https://github.com/matrix-org/synapse/issues/4969))
-- Reduce log level of .well-known/matrix/client responses. ([\#4972](https://github.com/matrix-org/synapse/issues/4972))
-- Add `config.signing_key_path` that can be read by `synapse.config` utility. ([\#4974](https://github.com/matrix-org/synapse/issues/4974))
-- Track which identity server is used when binding a threepid and use that for unbinding, as per MSC1915. ([\#4982](https://github.com/matrix-org/synapse/issues/4982))
-- Rewrite KeyringTestCase as a HomeserverTestCase. ([\#4985](https://github.com/matrix-org/synapse/issues/4985))
-- README updates: Corrected the default POSTGRES_USER. Added port forwarding hint in TLS section. ([\#4987](https://github.com/matrix-org/synapse/issues/4987))
-- Remove a number of unused tables from the database schema. ([\#4992](https://github.com/matrix-org/synapse/issues/4992), [\#5028](https://github.com/matrix-org/synapse/issues/5028), [\#5033](https://github.com/matrix-org/synapse/issues/5033))
-- Run `black` on the remainder of `synapse/storage/`. ([\#4996](https://github.com/matrix-org/synapse/issues/4996))
-- Fix grammar in get_current_users_in_room and give it a docstring. ([\#4998](https://github.com/matrix-org/synapse/issues/4998))
-- Clean up some code in the server-key Keyring. ([\#5001](https://github.com/matrix-org/synapse/issues/5001))
-- Convert SYNAPSE_NO_TLS Docker variable to boolean for user friendliness. Contributed by Gabriel Eckerson. ([\#5005](https://github.com/matrix-org/synapse/issues/5005))
-- Refactor synapse.storage._base._simple_select_list_paginate. ([\#5007](https://github.com/matrix-org/synapse/issues/5007))
-- Store the notary server name correctly in server_keys_json. ([\#5024](https://github.com/matrix-org/synapse/issues/5024))
-- Rewrite Datastore.get_server_verify_keys to reduce the number of database transactions. ([\#5030](https://github.com/matrix-org/synapse/issues/5030))
-- Remove extraneous period from copyright headers. ([\#5046](https://github.com/matrix-org/synapse/issues/5046))
-- Update documentation for where to get Synapse packages. ([\#5067](https://github.com/matrix-org/synapse/issues/5067))
-- Add workarounds for pep-517 install errors. ([\#5098](https://github.com/matrix-org/synapse/issues/5098))
-- Improve logging when event-signature checks fail. ([\#5100](https://github.com/matrix-org/synapse/issues/5100))
-- Factor out an "assert_requester_is_admin" function. ([\#5120](https://github.com/matrix-org/synapse/issues/5120))
-- Remove the requirement to authenticate for /admin/server_version. ([\#5122](https://github.com/matrix-org/synapse/issues/5122))
-- Prevent an exception from being raised in a IResolutionReceiver and use a more generic error message for blacklisted URL previews. ([\#5155](https://github.com/matrix-org/synapse/issues/5155))
-- Run `black` on the tests directory. ([\#5170](https://github.com/matrix-org/synapse/issues/5170))
-- Fix CI after new release of isort. ([\#5179](https://github.com/matrix-org/synapse/issues/5179))
-- Fix bogus imports in unit tests. ([\#5154](https://github.com/matrix-org/synapse/issues/5154))
-
-
-Synapse 0.99.3.2 (2019-05-03)
-=============================
-
-Internal Changes
-----------------
-
-- Ensure that we have `urllib3` <1.25, to resolve incompatibility with `requests`. ([\#5135](https://github.com/matrix-org/synapse/issues/5135))
-
-
-Synapse 0.99.3.1 (2019-05-03)
-=============================
-
-Security update
----------------
-
-This release includes two security fixes:
-
-- Switch to using a cryptographically-secure random number generator for token strings, ensuring they cannot be predicted by an attacker. Thanks to @opnsec for identifying and responsibly disclosing this issue! ([\#5133](https://github.com/matrix-org/synapse/issues/5133))
-- Blacklist 0.0.0.0 and :: by default for URL previews. Thanks to @opnsec for identifying and responsibly disclosing this issue too! ([\#5134](https://github.com/matrix-org/synapse/issues/5134))
-
-Synapse 0.99.3 (2019-04-01)
-===========================
-
-No significant changes.
-
-
-Synapse 0.99.3rc1 (2019-03-27)
-==============================
-
-Features
---------
-
-- The user directory has been rewritten to make it faster, with less chance of falling behind on a large server. ([\#4537](https://github.com/matrix-org/synapse/issues/4537), [\#4846](https://github.com/matrix-org/synapse/issues/4846), [\#4864](https://github.com/matrix-org/synapse/issues/4864), [\#4887](https://github.com/matrix-org/synapse/issues/4887), [\#4900](https://github.com/matrix-org/synapse/issues/4900), [\#4944](https://github.com/matrix-org/synapse/issues/4944))
-- Add configurable rate limiting to the /register endpoint. ([\#4735](https://github.com/matrix-org/synapse/issues/4735), [\#4804](https://github.com/matrix-org/synapse/issues/4804))
-- Move server key queries to federation reader. ([\#4757](https://github.com/matrix-org/synapse/issues/4757))
-- Add support for /account/3pid REST endpoint to client_reader worker. ([\#4759](https://github.com/matrix-org/synapse/issues/4759))
-- Add an endpoint to the admin API for querying the server version. Contributed by Joseph Weston. ([\#4772](https://github.com/matrix-org/synapse/issues/4772))
-- Include a default configuration file in the 'docs' directory. ([\#4791](https://github.com/matrix-org/synapse/issues/4791), [\#4801](https://github.com/matrix-org/synapse/issues/4801))
-- Synapse is now permissive about trailing slashes on some of its federation endpoints, allowing zero or more to be present. ([\#4793](https://github.com/matrix-org/synapse/issues/4793))
-- Add support for /keys/query and /keys/changes REST endpoints to client_reader worker. ([\#4796](https://github.com/matrix-org/synapse/issues/4796))
-- Add checks to incoming events over federation for events evading auth (aka "soft fail"). ([\#4814](https://github.com/matrix-org/synapse/issues/4814))
-- Add configurable rate limiting to the /login endpoint. ([\#4821](https://github.com/matrix-org/synapse/issues/4821), [\#4865](https://github.com/matrix-org/synapse/issues/4865))
-- Remove trailing slashes from certain outbound federation requests. Retry if receiving a 404. Context: #3622. ([\#4840](https://github.com/matrix-org/synapse/issues/4840))
-- Allow passing --daemonize flags to workers in the same way as with master. ([\#4853](https://github.com/matrix-org/synapse/issues/4853))
-- Batch up outgoing read-receipts to reduce federation traffic. ([\#4890](https://github.com/matrix-org/synapse/issues/4890), [\#4927](https://github.com/matrix-org/synapse/issues/4927))
-- Add option to disable searching the user directory. ([\#4895](https://github.com/matrix-org/synapse/issues/4895))
-- Add option to disable searching of local and remote public room lists. ([\#4896](https://github.com/matrix-org/synapse/issues/4896))
-- Add ability for password providers to login/register a user via 3PID (email, phone). ([\#4931](https://github.com/matrix-org/synapse/issues/4931))
-
-
-Bugfixes
---------
-
-- Fix a bug where media with spaces in the name would get a corrupted name. ([\#2090](https://github.com/matrix-org/synapse/issues/2090))
-- Fix attempting to paginate in rooms where server cannot see any events, to avoid unnecessarily pulling in lots of redacted events. ([\#4699](https://github.com/matrix-org/synapse/issues/4699))
-- 'event_id' is now a required parameter in federated state requests, as per the matrix spec. ([\#4740](https://github.com/matrix-org/synapse/issues/4740))
-- Fix tightloop over connecting to replication server. ([\#4749](https://github.com/matrix-org/synapse/issues/4749))
-- Fix parsing of Content-Disposition headers on remote media requests and URL previews. ([\#4763](https://github.com/matrix-org/synapse/issues/4763))
-- Fix incorrect log about not persisting duplicate state event. ([\#4776](https://github.com/matrix-org/synapse/issues/4776))
-- Fix v4v6 option in HAProxy example config. Contributed by Flakebi. ([\#4790](https://github.com/matrix-org/synapse/issues/4790))
-- Handle batch updates in worker replication protocol. ([\#4792](https://github.com/matrix-org/synapse/issues/4792))
-- Fix bug where we didn't correctly throttle sending of USER_IP commands over replication. ([\#4818](https://github.com/matrix-org/synapse/issues/4818))
-- Fix potential race in handling missing updates in device list updates. ([\#4829](https://github.com/matrix-org/synapse/issues/4829))
-- Fix bug where synapse expected an un-specced `prev_state` field on state events. ([\#4837](https://github.com/matrix-org/synapse/issues/4837))
-- Transfer a user's notification settings (push rules) on room upgrade. ([\#4838](https://github.com/matrix-org/synapse/issues/4838))
-- fix test_auto_create_auto_join_where_no_consent. ([\#4886](https://github.com/matrix-org/synapse/issues/4886))
-- Fix a bug where hs_disabled_message was sometimes not correctly enforced. ([\#4888](https://github.com/matrix-org/synapse/issues/4888))
-- Fix bug in shutdown room admin API where it would fail if a user in the room hadn't consented to the privacy policy. ([\#4904](https://github.com/matrix-org/synapse/issues/4904))
-- Fix bug where blocked world-readable rooms were still peekable. ([\#4908](https://github.com/matrix-org/synapse/issues/4908))
-
-
-Internal Changes
-----------------
-
-- Add a systemd setup that supports synapse workers. Contributed by Luca Corbatto. ([\#4662](https://github.com/matrix-org/synapse/issues/4662))
-- Change from TravisCI to Buildkite for CI. ([\#4752](https://github.com/matrix-org/synapse/issues/4752))
-- When presence is disabled don't send over replication. ([\#4757](https://github.com/matrix-org/synapse/issues/4757))
-- Minor docstring fixes for MatrixFederationAgent. ([\#4765](https://github.com/matrix-org/synapse/issues/4765))
-- Optimise EDU transmission for the federation_sender worker. ([\#4770](https://github.com/matrix-org/synapse/issues/4770))
-- Update test_typing to use HomeserverTestCase. ([\#4771](https://github.com/matrix-org/synapse/issues/4771))
-- Update URLs for riot.im icons and logos in the default notification templates. ([\#4779](https://github.com/matrix-org/synapse/issues/4779))
-- Removed unnecessary $ from some federation endpoint path regexes. ([\#4794](https://github.com/matrix-org/synapse/issues/4794))
-- Remove link to deleted title in README. ([\#4795](https://github.com/matrix-org/synapse/issues/4795))
-- Clean up read-receipt handling. ([\#4797](https://github.com/matrix-org/synapse/issues/4797))
-- Add some debug about processing read receipts. ([\#4798](https://github.com/matrix-org/synapse/issues/4798))
-- Clean up some replication code. ([\#4799](https://github.com/matrix-org/synapse/issues/4799))
-- Add some docstrings. ([\#4815](https://github.com/matrix-org/synapse/issues/4815))
-- Add debug logger to try and track down #4422. ([\#4816](https://github.com/matrix-org/synapse/issues/4816))
-- Make shutdown API send explanation message to room after users have been forced joined. ([\#4817](https://github.com/matrix-org/synapse/issues/4817))
-- Update example_log_config.yaml. ([\#4820](https://github.com/matrix-org/synapse/issues/4820))
-- Document the `generate` option for the docker image. ([\#4824](https://github.com/matrix-org/synapse/issues/4824))
-- Fix check-newsfragment for debian-only changes. ([\#4825](https://github.com/matrix-org/synapse/issues/4825))
-- Add some debug logging for device list updates to help with #4828. ([\#4828](https://github.com/matrix-org/synapse/issues/4828))
-- Improve federation documentation, specifically .well-known support. Many thanks to @vaab. ([\#4832](https://github.com/matrix-org/synapse/issues/4832))
-- Disable captcha registration by default in unit tests. ([\#4839](https://github.com/matrix-org/synapse/issues/4839))
-- Add stuff back to the .gitignore. ([\#4843](https://github.com/matrix-org/synapse/issues/4843))
-- Clarify what registration_shared_secret allows for. ([\#4844](https://github.com/matrix-org/synapse/issues/4844))
-- Correctly log expected errors when fetching server keys. ([\#4847](https://github.com/matrix-org/synapse/issues/4847))
-- Update install docs to explicitly state a full-chain (not just the top-level) TLS certificate must be provided to Synapse. This caused some people's Synapse ports to appear correct in a browser but still (rightfully so) upset the federation tester. ([\#4849](https://github.com/matrix-org/synapse/issues/4849))
-- Move client read-receipt processing to federation sender worker. ([\#4852](https://github.com/matrix-org/synapse/issues/4852))
-- Refactor federation TransactionQueue. ([\#4855](https://github.com/matrix-org/synapse/issues/4855))
-- Comment out most options in the generated config. ([\#4863](https://github.com/matrix-org/synapse/issues/4863))
-- Fix yaml library warnings by using safe_load. ([\#4869](https://github.com/matrix-org/synapse/issues/4869))
-- Update Apache setup to remove location syntax. Thanks to @cwmke! ([\#4870](https://github.com/matrix-org/synapse/issues/4870))
-- Reinstate test case that runs unit tests against oldest supported dependencies. ([\#4879](https://github.com/matrix-org/synapse/issues/4879))
-- Update link to federation docs. ([\#4881](https://github.com/matrix-org/synapse/issues/4881))
-- fix test_auto_create_auto_join_where_no_consent. ([\#4886](https://github.com/matrix-org/synapse/issues/4886))
-- Use a regular HomeServerConfig object for unit tests rater than a Mock. ([\#4889](https://github.com/matrix-org/synapse/issues/4889))
-- Add some notes about tuning postgres for larger deployments. ([\#4895](https://github.com/matrix-org/synapse/issues/4895))
-- Add a config option for torture-testing worker replication. ([\#4902](https://github.com/matrix-org/synapse/issues/4902))
-- Log requests which are simulated by the unit tests. ([\#4905](https://github.com/matrix-org/synapse/issues/4905))
-- Allow newsfragments to end with exclamation marks. Exciting! ([\#4912](https://github.com/matrix-org/synapse/issues/4912))
-- Refactor some more tests to use HomeserverTestCase. ([\#4913](https://github.com/matrix-org/synapse/issues/4913))
-- Refactor out the state deltas portion of the user directory store and handler. ([\#4917](https://github.com/matrix-org/synapse/issues/4917))
-- Fix nginx example in ACME doc. ([\#4923](https://github.com/matrix-org/synapse/issues/4923))
-- Use an explicit dbname for postgres connections in the tests. ([\#4928](https://github.com/matrix-org/synapse/issues/4928))
-- Fix `ClientReplicationStreamProtocol.__str__()`. ([\#4929](https://github.com/matrix-org/synapse/issues/4929))
-
-
-Synapse 0.99.2 (2019-03-01)
-===========================
-
-Features
---------
-
-- Added an HAProxy example in the reverse proxy documentation. Contributed by Benoît S. (“Benpro”). ([\#4541](https://github.com/matrix-org/synapse/issues/4541))
-- Add basic optional sentry integration. ([\#4632](https://github.com/matrix-org/synapse/issues/4632), [\#4694](https://github.com/matrix-org/synapse/issues/4694))
-- Transfer bans on room upgrade. ([\#4642](https://github.com/matrix-org/synapse/issues/4642))
-- Add configurable room list publishing rules. ([\#4647](https://github.com/matrix-org/synapse/issues/4647))
-- Support .well-known delegation when issuing certificates through ACME. ([\#4652](https://github.com/matrix-org/synapse/issues/4652))
-- Allow registration and login to be handled by a worker instance. ([\#4666](https://github.com/matrix-org/synapse/issues/4666), [\#4670](https://github.com/matrix-org/synapse/issues/4670), [\#4682](https://github.com/matrix-org/synapse/issues/4682))
-- Reduce the overhead of creating outbound federation connections over TLS by caching the TLS client options. ([\#4674](https://github.com/matrix-org/synapse/issues/4674))
-- Add prometheus metrics for number of outgoing EDUs, by type. ([\#4695](https://github.com/matrix-org/synapse/issues/4695))
-- Return correct error code when inviting a remote user to a room whose homeserver does not support the room version. ([\#4721](https://github.com/matrix-org/synapse/issues/4721))
-- Prevent showing rooms to other servers that were set to not federate. ([\#4746](https://github.com/matrix-org/synapse/issues/4746))
-
-
-Bugfixes
---------
-
-- Fix possible exception when paginating. ([\#4263](https://github.com/matrix-org/synapse/issues/4263))
-- The dependency checker now correctly reports a version mismatch for optional
-  dependencies, instead of reporting the dependency missing. ([\#4450](https://github.com/matrix-org/synapse/issues/4450))
-- Set CORS headers on .well-known requests. ([\#4651](https://github.com/matrix-org/synapse/issues/4651))
-- Fix kicking guest users on guest access revocation in worker mode. ([\#4667](https://github.com/matrix-org/synapse/issues/4667))
-- Fix an issue in the database migration script where the
-  `e2e_room_keys.is_verified` column wasn't considered as
-  a boolean. ([\#4680](https://github.com/matrix-org/synapse/issues/4680))
-- Fix TaskStopped exceptions in logs when outbound requests time out. ([\#4690](https://github.com/matrix-org/synapse/issues/4690))
-- Fix ACME config for python 2. ([\#4717](https://github.com/matrix-org/synapse/issues/4717))
-- Fix paginating over federation persisting incorrect state. ([\#4718](https://github.com/matrix-org/synapse/issues/4718))
-
-
-Internal Changes
-----------------
-
-- Run `black` to reformat user directory code. ([\#4635](https://github.com/matrix-org/synapse/issues/4635))
-- Reduce number of exceptions we log. ([\#4643](https://github.com/matrix-org/synapse/issues/4643), [\#4668](https://github.com/matrix-org/synapse/issues/4668))
-- Introduce upsert batching functionality in the database layer. ([\#4644](https://github.com/matrix-org/synapse/issues/4644))
-- Fix various spelling mistakes. ([\#4657](https://github.com/matrix-org/synapse/issues/4657))
-- Cleanup request exception logging. ([\#4669](https://github.com/matrix-org/synapse/issues/4669), [\#4737](https://github.com/matrix-org/synapse/issues/4737), [\#4738](https://github.com/matrix-org/synapse/issues/4738))
-- Improve replication performance by reducing cache invalidation traffic. ([\#4671](https://github.com/matrix-org/synapse/issues/4671), [\#4715](https://github.com/matrix-org/synapse/issues/4715), [\#4748](https://github.com/matrix-org/synapse/issues/4748))
-- Test against Postgres 9.5 as well as 9.4. ([\#4676](https://github.com/matrix-org/synapse/issues/4676))
-- Run unit tests against python 3.7. ([\#4677](https://github.com/matrix-org/synapse/issues/4677))
-- Attempt to clarify installation instructions/config. ([\#4681](https://github.com/matrix-org/synapse/issues/4681))
-- Clean up gitignores. ([\#4688](https://github.com/matrix-org/synapse/issues/4688))
-- Minor tweaks to acme docs. ([\#4689](https://github.com/matrix-org/synapse/issues/4689))
-- Improve the logging in the pusher process. ([\#4691](https://github.com/matrix-org/synapse/issues/4691))
-- Better checks on newsfragments. ([\#4698](https://github.com/matrix-org/synapse/issues/4698), [\#4750](https://github.com/matrix-org/synapse/issues/4750))
-- Avoid some redundant work when processing read receipts. ([\#4706](https://github.com/matrix-org/synapse/issues/4706))
-- Run `push_receipts_to_remotes` as background job. ([\#4707](https://github.com/matrix-org/synapse/issues/4707))
-- Add prometheus metrics for number of badge update pushes. ([\#4709](https://github.com/matrix-org/synapse/issues/4709))
-- Reduce pusher logging on startup ([\#4716](https://github.com/matrix-org/synapse/issues/4716))
-- Don't log exceptions when failing to fetch remote server keys. ([\#4722](https://github.com/matrix-org/synapse/issues/4722))
-- Correctly proxy exception in frontend_proxy worker. ([\#4723](https://github.com/matrix-org/synapse/issues/4723))
-- Add database version to phonehome stats. ([\#4753](https://github.com/matrix-org/synapse/issues/4753))
-
-
-Synapse 0.99.1.1 (2019-02-14)
-=============================
-
-Bugfixes
---------
-
-- Fix "TypeError: '>' not supported" when starting without an existing certificate.
-  Fix a bug where an existing certificate would be reprovisoned every day. ([\#4648](https://github.com/matrix-org/synapse/issues/4648))
-
-
-Synapse 0.99.1 (2019-02-14)
-===========================
-
-Features
---------
-
-- Include m.room.encryption on invites by default ([\#3902](https://github.com/matrix-org/synapse/issues/3902))
-- Federation OpenID listener resource can now be activated even if federation is disabled ([\#4420](https://github.com/matrix-org/synapse/issues/4420))
-- Synapse's ACME support will now correctly reprovision a certificate that approaches its expiry while Synapse is running. ([\#4522](https://github.com/matrix-org/synapse/issues/4522))
-- Add ability to update backup versions ([\#4580](https://github.com/matrix-org/synapse/issues/4580))
-- Allow the "unavailable" presence status for /sync.
-  This change makes Synapse compliant with r0.4.0 of the Client-Server specification. ([\#4592](https://github.com/matrix-org/synapse/issues/4592))
-- There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners ([\#4613](https://github.com/matrix-org/synapse/issues/4613), [\#4615](https://github.com/matrix-org/synapse/issues/4615), [\#4617](https://github.com/matrix-org/synapse/issues/4617), [\#4636](https://github.com/matrix-org/synapse/issues/4636))
-- The default configuration no longer requires TLS certificates. ([\#4614](https://github.com/matrix-org/synapse/issues/4614))
-
-
-Bugfixes
---------
-
-- Copy over room federation ability on room upgrade. ([\#4530](https://github.com/matrix-org/synapse/issues/4530))
-- Fix noisy "twisted.internet.task.TaskStopped" errors in logs ([\#4546](https://github.com/matrix-org/synapse/issues/4546))
-- Synapse is now tolerant of the `tls_fingerprints` option being None or not specified. ([\#4589](https://github.com/matrix-org/synapse/issues/4589))
-- Fix 'no unique or exclusion constraint' error ([\#4591](https://github.com/matrix-org/synapse/issues/4591))
-- Transfer Server ACLs on room upgrade. ([\#4608](https://github.com/matrix-org/synapse/issues/4608))
-- Fix failure to start when not TLS certificate was given even if TLS was disabled. ([\#4618](https://github.com/matrix-org/synapse/issues/4618))
-- Fix self-signed cert notice from generate-config. ([\#4625](https://github.com/matrix-org/synapse/issues/4625))
-- Fix performance of `user_ips` table deduplication background update ([\#4626](https://github.com/matrix-org/synapse/issues/4626), [\#4627](https://github.com/matrix-org/synapse/issues/4627))
-
-
-Internal Changes
-----------------
-
-- Change the user directory state query to use a filtered call to the db instead of a generic one. ([\#4462](https://github.com/matrix-org/synapse/issues/4462))
-- Reject federation transactions if they include more than 50 PDUs or 100 EDUs. ([\#4513](https://github.com/matrix-org/synapse/issues/4513))
-- Reduce duplication of ``synapse.app`` code. ([\#4567](https://github.com/matrix-org/synapse/issues/4567))
-- Fix docker upload job to push -py2 images. ([\#4576](https://github.com/matrix-org/synapse/issues/4576))
-- Add port configuration information to ACME instructions. ([\#4578](https://github.com/matrix-org/synapse/issues/4578))
-- Update MSC1711 FAQ to calrify .well-known usage ([\#4584](https://github.com/matrix-org/synapse/issues/4584))
-- Clean up default listener configuration ([\#4586](https://github.com/matrix-org/synapse/issues/4586))
-- Clarifications for reverse proxy docs ([\#4607](https://github.com/matrix-org/synapse/issues/4607))
-- Move ClientTLSOptionsFactory init out of `refresh_certificates` ([\#4611](https://github.com/matrix-org/synapse/issues/4611))
-- Fail cleanly if listener config lacks a 'port' ([\#4616](https://github.com/matrix-org/synapse/issues/4616))
-- Remove redundant entries from docker config ([\#4619](https://github.com/matrix-org/synapse/issues/4619))
-- README updates ([\#4621](https://github.com/matrix-org/synapse/issues/4621))
-
-
-Synapse 0.99.0 (2019-02-05)
-===========================
-
-Synapse v0.99.x is a precursor to the upcoming Synapse v1.0 release. It contains foundational changes to room architecture and the federation security model necessary to support the upcoming r0 release of the Server to Server API.
-
-Features
---------
-
-- Synapse's cipher string has been updated to require ECDH key exchange. Configuring and generating dh_params is no longer required, and they will be ignored. ([\#4229](https://github.com/matrix-org/synapse/issues/4229))
-- Synapse can now automatically provision TLS certificates via ACME (the protocol used by CAs like Let's Encrypt). ([\#4384](https://github.com/matrix-org/synapse/issues/4384), [\#4492](https://github.com/matrix-org/synapse/issues/4492), [\#4525](https://github.com/matrix-org/synapse/issues/4525), [\#4572](https://github.com/matrix-org/synapse/issues/4572), [\#4564](https://github.com/matrix-org/synapse/issues/4564), [\#4566](https://github.com/matrix-org/synapse/issues/4566), [\#4547](https://github.com/matrix-org/synapse/issues/4547), [\#4557](https://github.com/matrix-org/synapse/issues/4557))
-- Implement MSC1708 (.well-known routing for server-server federation) ([\#4408](https://github.com/matrix-org/synapse/issues/4408), [\#4409](https://github.com/matrix-org/synapse/issues/4409), [\#4426](https://github.com/matrix-org/synapse/issues/4426), [\#4427](https://github.com/matrix-org/synapse/issues/4427), [\#4428](https://github.com/matrix-org/synapse/issues/4428), [\#4464](https://github.com/matrix-org/synapse/issues/4464), [\#4468](https://github.com/matrix-org/synapse/issues/4468), [\#4487](https://github.com/matrix-org/synapse/issues/4487), [\#4488](https://github.com/matrix-org/synapse/issues/4488), [\#4489](https://github.com/matrix-org/synapse/issues/4489), [\#4497](https://github.com/matrix-org/synapse/issues/4497), [\#4511](https://github.com/matrix-org/synapse/issues/4511), [\#4516](https://github.com/matrix-org/synapse/issues/4516), [\#4520](https://github.com/matrix-org/synapse/issues/4520), [\#4521](https://github.com/matrix-org/synapse/issues/4521), [\#4539](https://github.com/matrix-org/synapse/issues/4539), [\#4542](https://github.com/matrix-org/synapse/issues/4542), [\#4544](https://github.com/matrix-org/synapse/issues/4544))
-- Search now includes results from predecessor rooms after a room upgrade. ([\#4415](https://github.com/matrix-org/synapse/issues/4415))
-- Config option to disable requesting MSISDN on registration. ([\#4423](https://github.com/matrix-org/synapse/issues/4423))
-- Add a metric for tracking event stream position of the user directory. ([\#4445](https://github.com/matrix-org/synapse/issues/4445))
-- Support exposing server capabilities in CS API (MSC1753, MSC1804) ([\#4472](https://github.com/matrix-org/synapse/issues/4472), [81b7e7eed](https://github.com/matrix-org/synapse/commit/81b7e7eed323f55d6550e7a270a9dc2c4c7b0fe0)))
-- Add support for room version 3 ([\#4483](https://github.com/matrix-org/synapse/issues/4483), [\#4499](https://github.com/matrix-org/synapse/issues/4499), [\#4515](https://github.com/matrix-org/synapse/issues/4515), [\#4523](https://github.com/matrix-org/synapse/issues/4523), [\#4535](https://github.com/matrix-org/synapse/issues/4535))
-- Synapse will now reload TLS certificates from disk upon SIGHUP. ([\#4495](https://github.com/matrix-org/synapse/issues/4495), [\#4524](https://github.com/matrix-org/synapse/issues/4524))
-- The matrixdotorg/synapse Docker images now use Python 3 by default. ([\#4558](https://github.com/matrix-org/synapse/issues/4558))
-
-Bugfixes
---------
-
-- Prevent users with access tokens predating the introduction of device IDs from creating spurious entries in the user_ips table. ([\#4369](https://github.com/matrix-org/synapse/issues/4369))
-- Fix typo in ALL_USER_TYPES definition to ensure type is a tuple ([\#4392](https://github.com/matrix-org/synapse/issues/4392))
-- Fix high CPU usage due to remote devicelist updates ([\#4397](https://github.com/matrix-org/synapse/issues/4397))
-- Fix potential bug where creating or joining a room could fail ([\#4404](https://github.com/matrix-org/synapse/issues/4404))
-- Fix bug when rejecting remote invites ([\#4405](https://github.com/matrix-org/synapse/issues/4405), [\#4527](https://github.com/matrix-org/synapse/issues/4527))
-- Fix incorrect logcontexts after a Deferred was cancelled ([\#4407](https://github.com/matrix-org/synapse/issues/4407))
-- Ensure encrypted room state is persisted across room upgrades. ([\#4411](https://github.com/matrix-org/synapse/issues/4411))
-- Copy over whether a room is a direct message and any associated room tags on room upgrade. ([\#4412](https://github.com/matrix-org/synapse/issues/4412))
-- Fix None guard in calling config.server.is_threepid_reserved ([\#4435](https://github.com/matrix-org/synapse/issues/4435))
-- Don't send IP addresses as SNI ([\#4452](https://github.com/matrix-org/synapse/issues/4452))
-- Fix UnboundLocalError in post_urlencoded_get_json ([\#4460](https://github.com/matrix-org/synapse/issues/4460))
-- Add a timeout to filtered room directory queries. ([\#4461](https://github.com/matrix-org/synapse/issues/4461))
-- Workaround for login error when using both LDAP and internal authentication. ([\#4486](https://github.com/matrix-org/synapse/issues/4486))
-- Fix a bug where setting a relative consent directory path would cause a crash. ([\#4512](https://github.com/matrix-org/synapse/issues/4512))
-
-
-Deprecations and Removals
--------------------------
-
-- Synapse no longer generates self-signed TLS certificates when generating a configuration file. ([\#4509](https://github.com/matrix-org/synapse/issues/4509))
-
-
-Improved Documentation
-----------------------
-
-- Update debian installation instructions ([\#4526](https://github.com/matrix-org/synapse/issues/4526))
-
-
-Internal Changes
-----------------
-
-- Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+. ([\#4306](https://github.com/matrix-org/synapse/issues/4306), [\#4459](https://github.com/matrix-org/synapse/issues/4459), [\#4466](https://github.com/matrix-org/synapse/issues/4466), [\#4471](https://github.com/matrix-org/synapse/issues/4471), [\#4477](https://github.com/matrix-org/synapse/issues/4477), [\#4505](https://github.com/matrix-org/synapse/issues/4505))
-- Update README to use the new virtualenv everywhere ([\#4342](https://github.com/matrix-org/synapse/issues/4342))
-- Add better logging for unexpected errors while sending transactions ([\#4368](https://github.com/matrix-org/synapse/issues/4368))
-- Apply a unique index to the user_ips table, preventing duplicates. ([\#4370](https://github.com/matrix-org/synapse/issues/4370), [\#4432](https://github.com/matrix-org/synapse/issues/4432), [\#4434](https://github.com/matrix-org/synapse/issues/4434))
-- Silence travis-ci build warnings by removing non-functional python3.6 ([\#4377](https://github.com/matrix-org/synapse/issues/4377))
-- Fix a comment in the generated config file ([\#4387](https://github.com/matrix-org/synapse/issues/4387))
-- Add ground work for implementing future federation API versions ([\#4390](https://github.com/matrix-org/synapse/issues/4390))
-- Update dependencies on msgpack and pymacaroons to use the up-to-date packages. ([\#4399](https://github.com/matrix-org/synapse/issues/4399))
-- Tweak codecov settings to make them less loud. ([\#4400](https://github.com/matrix-org/synapse/issues/4400))
-- Implement server support for MSC1794 - Federation v2 Invite API ([\#4402](https://github.com/matrix-org/synapse/issues/4402))
-- debian package: symlink to explicit python version ([\#4433](https://github.com/matrix-org/synapse/issues/4433))
-- Add infrastructure to support different event formats ([\#4437](https://github.com/matrix-org/synapse/issues/4437), [\#4447](https://github.com/matrix-org/synapse/issues/4447), [\#4448](https://github.com/matrix-org/synapse/issues/4448), [\#4470](https://github.com/matrix-org/synapse/issues/4470), [\#4481](https://github.com/matrix-org/synapse/issues/4481), [\#4482](https://github.com/matrix-org/synapse/issues/4482), [\#4493](https://github.com/matrix-org/synapse/issues/4493), [\#4494](https://github.com/matrix-org/synapse/issues/4494), [\#4496](https://github.com/matrix-org/synapse/issues/4496), [\#4510](https://github.com/matrix-org/synapse/issues/4510), [\#4514](https://github.com/matrix-org/synapse/issues/4514))
-- Generate the debian config during build ([\#4444](https://github.com/matrix-org/synapse/issues/4444))
-- Clarify documentation for the `public_baseurl` config param ([\#4458](https://github.com/matrix-org/synapse/issues/4458), [\#4498](https://github.com/matrix-org/synapse/issues/4498))
-- Fix quoting for allowed_local_3pids example config ([\#4476](https://github.com/matrix-org/synapse/issues/4476))
-- Remove deprecated --process-dependency-links option from UPGRADE.rst ([\#4485](https://github.com/matrix-org/synapse/issues/4485))
-- Make it possible to set the log level for tests via an environment variable ([\#4506](https://github.com/matrix-org/synapse/issues/4506))
-- Reduce the log level of linearizer lock acquirement to DEBUG. ([\#4507](https://github.com/matrix-org/synapse/issues/4507))
-- Fix code to comply with linting in PyFlakes 3.7.1. ([\#4519](https://github.com/matrix-org/synapse/issues/4519))
-- Add some debug for membership syncing issues ([\#4538](https://github.com/matrix-org/synapse/issues/4538))
-- Docker: only copy what we need to the build image ([\#4562](https://github.com/matrix-org/synapse/issues/4562))
-
-
-Synapse 0.34.1.1 (2019-01-11)
-=============================
-
-This release fixes CVE-2019-5885 and is recommended for all users of Synapse 0.34.1.
-
-This release is compatible with Python 2.7 and 3.5+. Python 3.7 is fully supported.
-
-Bugfixes
---------
-
-- Fix spontaneous logout on upgrade
-  ([\#4374](https://github.com/matrix-org/synapse/issues/4374))
-
-
-Synapse 0.34.1 (2019-01-09)
-===========================
-
-Internal Changes
-----------------
-
-- Add better logging for unexpected errors while sending transactions ([\#4361](https://github.com/matrix-org/synapse/issues/4361), [\#4362](https://github.com/matrix-org/synapse/issues/4362))
-
-
-Synapse 0.34.1rc1 (2019-01-08)
-==============================
-
-Features
---------
-
-- Special-case a support user for use in verifying behaviour of a given server. The support user does not appear in user directory or monthly active user counts. ([\#4141](https://github.com/matrix-org/synapse/issues/4141), [\#4344](https://github.com/matrix-org/synapse/issues/4344))
-- Support for serving .well-known files ([\#4262](https://github.com/matrix-org/synapse/issues/4262))
-- Rework SAML2 authentication ([\#4265](https://github.com/matrix-org/synapse/issues/4265), [\#4267](https://github.com/matrix-org/synapse/issues/4267))
-- SAML2 authentication: Initialise user display name from SAML2 data ([\#4272](https://github.com/matrix-org/synapse/issues/4272))
-- Synapse can now have its conditional/extra dependencies installed by pip. This functionality can be used by using `pip install matrix-synapse[feature]`, where feature is a comma separated list with the possible values `email.enable_notifs`, `matrix-synapse-ldap3`, `postgres`, `resources.consent`, `saml2`, `url_preview`, and `test`. If you want to install all optional dependencies, you can use "all" instead. ([\#4298](https://github.com/matrix-org/synapse/issues/4298), [\#4325](https://github.com/matrix-org/synapse/issues/4325), [\#4327](https://github.com/matrix-org/synapse/issues/4327))
-- Add routes for reading account data. ([\#4303](https://github.com/matrix-org/synapse/issues/4303))
-- Add opt-in support for v2 rooms ([\#4307](https://github.com/matrix-org/synapse/issues/4307))
-- Add a script to generate a clean config file ([\#4315](https://github.com/matrix-org/synapse/issues/4315))
-- Return server data in /login response ([\#4319](https://github.com/matrix-org/synapse/issues/4319))
-
-
-Bugfixes
---------
-
-- Fix contains_url check to be consistent with other instances in code-base and check that value is an instance of string. ([\#3405](https://github.com/matrix-org/synapse/issues/3405))
-- Fix CAS login when username is not valid in an MXID ([\#4264](https://github.com/matrix-org/synapse/issues/4264))
-- Send CORS headers for /media/config ([\#4279](https://github.com/matrix-org/synapse/issues/4279))
-- Add 'sandbox' to CSP for media reprository ([\#4284](https://github.com/matrix-org/synapse/issues/4284))
-- Make the new landing page prettier. ([\#4294](https://github.com/matrix-org/synapse/issues/4294))
-- Fix deleting E2E room keys when using old SQLite versions. ([\#4295](https://github.com/matrix-org/synapse/issues/4295))
-- The metric synapse_admin_mau:current previously did not update when config.mau_stats_only was set to True ([\#4305](https://github.com/matrix-org/synapse/issues/4305))
-- Fixed per-room account data filters ([\#4309](https://github.com/matrix-org/synapse/issues/4309))
-- Fix indentation in default config ([\#4313](https://github.com/matrix-org/synapse/issues/4313))
-- Fix synapse:latest docker upload ([\#4316](https://github.com/matrix-org/synapse/issues/4316))
-- Fix test_metric.py compatibility with prometheus_client 0.5. Contributed by Maarten de Vries <maarten@de-vri.es>. ([\#4317](https://github.com/matrix-org/synapse/issues/4317))
-- Avoid packaging _trial_temp directory in -py3 debian packages ([\#4326](https://github.com/matrix-org/synapse/issues/4326))
-- Check jinja version for consent resource ([\#4327](https://github.com/matrix-org/synapse/issues/4327))
-- fix NPE in /messages by checking if all events were filtered out ([\#4330](https://github.com/matrix-org/synapse/issues/4330))
-- Fix `python -m synapse.config` on Python 3. ([\#4356](https://github.com/matrix-org/synapse/issues/4356))
-
-
-Deprecations and Removals
--------------------------
-
-- Remove the deprecated v1/register API on Python 2. It was never ported to Python 3. ([\#4334](https://github.com/matrix-org/synapse/issues/4334))
-
-
-Internal Changes
-----------------
-
-- Getting URL previews of IP addresses no longer fails on Python 3. ([\#4215](https://github.com/matrix-org/synapse/issues/4215))
-- drop undocumented dependency on dateutil ([\#4266](https://github.com/matrix-org/synapse/issues/4266))
-- Update the example systemd config to use a virtualenv ([\#4273](https://github.com/matrix-org/synapse/issues/4273))
-- Update link to kernel DCO guide ([\#4274](https://github.com/matrix-org/synapse/issues/4274))
-- Make isort tox check print diff when it fails ([\#4283](https://github.com/matrix-org/synapse/issues/4283))
-- Log room_id in Unknown room errors ([\#4297](https://github.com/matrix-org/synapse/issues/4297))
-- Documentation improvements for coturn setup. Contributed by Krithin Sitaram. ([\#4333](https://github.com/matrix-org/synapse/issues/4333))
-- Update pull request template to use absolute links ([\#4341](https://github.com/matrix-org/synapse/issues/4341))
-- Update README to not lie about required restart when updating TLS certificates ([\#4343](https://github.com/matrix-org/synapse/issues/4343))
-- Update debian packaging for compatibility with transitional package ([\#4349](https://github.com/matrix-org/synapse/issues/4349))
-- Fix command hint to generate a config file when trying to start without a config file ([\#4353](https://github.com/matrix-org/synapse/issues/4353))
-- Add better logging for unexpected errors while sending transactions ([\#4358](https://github.com/matrix-org/synapse/issues/4358))
-
-
-Synapse 0.34.0 (2018-12-20)
-===========================
-
-Synapse 0.34.0 is the first release to fully support Python 3. Synapse will now
-run on Python versions 3.5 or 3.6 (as well as 2.7). Support for Python 3.7
-remains experimental.
-
-We recommend upgrading to Python 3, but make sure to read the [upgrade
-notes](UPGRADE.rst#upgrading-to-v0340) when doing so.
-
-Features
---------
-
-- Add 'sandbox' to CSP for media reprository ([\#4284](https://github.com/matrix-org/synapse/issues/4284))
-- Make the new landing page prettier. ([\#4294](https://github.com/matrix-org/synapse/issues/4294))
-- Fix deleting E2E room keys when using old SQLite versions. ([\#4295](https://github.com/matrix-org/synapse/issues/4295))
-- Add a welcome page for the client API port. Credit to @krombel! ([\#4289](https://github.com/matrix-org/synapse/issues/4289))
-- Remove Matrix console from the default distribution ([\#4290](https://github.com/matrix-org/synapse/issues/4290))
-- Add option to track MAU stats (but not limit people) ([\#3830](https://github.com/matrix-org/synapse/issues/3830))
-- Add an option to enable recording IPs for appservice users ([\#3831](https://github.com/matrix-org/synapse/issues/3831))
-- Rename login type `m.login.cas` to `m.login.sso` ([\#4220](https://github.com/matrix-org/synapse/issues/4220))
-- Add an option to disable search for homeservers that may not be interested in it. ([\#4230](https://github.com/matrix-org/synapse/issues/4230))
-
-
-Bugfixes
---------
-
-- Pushrules can now again be made with non-ASCII rule IDs. ([\#4165](https://github.com/matrix-org/synapse/issues/4165))
-- The media repository now no longer fails to decode UTF-8 filenames when downloading remote media. ([\#4176](https://github.com/matrix-org/synapse/issues/4176))
-- URL previews now correctly decode non-UTF-8 text if the header contains a `<meta http-equiv="Content-Type"` header. ([\#4183](https://github.com/matrix-org/synapse/issues/4183))
-- Fix an issue where public consent URLs had two slashes. ([\#4192](https://github.com/matrix-org/synapse/issues/4192))
-- Fallback auth now accepts the session parameter on Python 3. ([\#4197](https://github.com/matrix-org/synapse/issues/4197))
-- Remove riot.im from the list of trusted Identity Servers in the default configuration ([\#4207](https://github.com/matrix-org/synapse/issues/4207))
-- fix start up failure when mau_limit_reserved_threepids set and db is postgres ([\#4211](https://github.com/matrix-org/synapse/issues/4211))
-- Fix auto join failures for servers that require user consent ([\#4223](https://github.com/matrix-org/synapse/issues/4223))
-- Fix exception caused by non-ascii event IDs ([\#4241](https://github.com/matrix-org/synapse/issues/4241))
-- Pushers can now be unsubscribed from on Python 3. ([\#4250](https://github.com/matrix-org/synapse/issues/4250))
-- Fix UnicodeDecodeError when postgres is configured to give non-English errors ([\#4253](https://github.com/matrix-org/synapse/issues/4253))
-
-
-Internal Changes
-----------------
-
-- Debian packages utilising a virtualenv with bundled dependencies can now be built. ([\#4212](https://github.com/matrix-org/synapse/issues/4212))
-- Disable pager when running git-show in CI ([\#4291](https://github.com/matrix-org/synapse/issues/4291))
-- A coveragerc file has been added. ([\#4180](https://github.com/matrix-org/synapse/issues/4180))
-- Add a GitHub pull request template and add multiple issue templates ([\#4182](https://github.com/matrix-org/synapse/issues/4182))
-- Update README to reflect the fact that [\#1491](https://github.com/matrix-org/synapse/issues/1491) is fixed ([\#4188](https://github.com/matrix-org/synapse/issues/4188))
-- Run the AS senders as background processes to fix warnings ([\#4189](https://github.com/matrix-org/synapse/issues/4189))
-- Add some diagnostics to the tests to detect logcontext problems ([\#4190](https://github.com/matrix-org/synapse/issues/4190))
-- Add missing `jpeg` package prerequisite for OpenBSD in README. ([\#4193](https://github.com/matrix-org/synapse/issues/4193))
-- Add a note saying you need to manually reclaim disk space after using the Purge History API ([\#4200](https://github.com/matrix-org/synapse/issues/4200))
-- More logcontext checking in unittests ([\#4205](https://github.com/matrix-org/synapse/issues/4205))
-- Ignore `__pycache__` directories in the database schema folder ([\#4214](https://github.com/matrix-org/synapse/issues/4214))
-- Add note to UPGRADE.rst about removing riot.im from list of trusted identity servers ([\#4224](https://github.com/matrix-org/synapse/issues/4224))
-- Added automated coverage reporting to CI. ([\#4225](https://github.com/matrix-org/synapse/issues/4225))
-- Garbage-collect after each unit test to fix logcontext leaks ([\#4227](https://github.com/matrix-org/synapse/issues/4227))
-- add more detail to logging regarding "More than one row matched" error ([\#4234](https://github.com/matrix-org/synapse/issues/4234))
-- Drop sent_transactions table ([\#4244](https://github.com/matrix-org/synapse/issues/4244))
-- Add a basic .editorconfig ([\#4257](https://github.com/matrix-org/synapse/issues/4257))
-- Update README.rst and UPGRADE.rst for Python 3. ([\#4260](https://github.com/matrix-org/synapse/issues/4260))
-- Remove obsolete `verbose` and `log_file` settings from `homeserver.yaml` for Docker image. ([\#4261](https://github.com/matrix-org/synapse/issues/4261))
-
-
-Synapse 0.33.9 (2018-11-19)
-===========================
-
-No significant changes.
-
-
-Synapse 0.33.9rc1 (2018-11-14)
-==============================
-
-Features
---------
-
-- Include flags to optionally add `m.login.terms` to the registration flow when consent tracking is enabled. ([\#4004](https://github.com/matrix-org/synapse/issues/4004), [\#4133](https://github.com/matrix-org/synapse/issues/4133), [\#4142](https://github.com/matrix-org/synapse/issues/4142), [\#4184](https://github.com/matrix-org/synapse/issues/4184))
-- Support for replacing rooms with new ones ([\#4091](https://github.com/matrix-org/synapse/issues/4091), [\#4099](https://github.com/matrix-org/synapse/issues/4099), [\#4100](https://github.com/matrix-org/synapse/issues/4100), [\#4101](https://github.com/matrix-org/synapse/issues/4101))
-
-
-Bugfixes
---------
-
-- Fix exceptions when using the email mailer on Python 3. ([\#4095](https://github.com/matrix-org/synapse/issues/4095))
-- Fix e2e key backup with more than 9 backup versions ([\#4113](https://github.com/matrix-org/synapse/issues/4113))
-- Searches that request profile info now no longer fail with a 500. ([\#4122](https://github.com/matrix-org/synapse/issues/4122))
-- fix return code of empty key backups ([\#4123](https://github.com/matrix-org/synapse/issues/4123))
-- If the typing stream ID goes backwards (as on a worker when the master restarts), the worker's typing handler will no longer erroneously report rooms containing new typing events. ([\#4127](https://github.com/matrix-org/synapse/issues/4127))
-- Fix table lock of device_lists_remote_cache which could freeze the application ([\#4132](https://github.com/matrix-org/synapse/issues/4132))
-- Fix exception when using state res v2 algorithm ([\#4135](https://github.com/matrix-org/synapse/issues/4135))
-- Generating the user consent URI no longer fails on Python 3. ([\#4140](https://github.com/matrix-org/synapse/issues/4140), [\#4163](https://github.com/matrix-org/synapse/issues/4163))
-- Loading URL previews from the DB cache on Postgres will no longer cause Unicode type errors when responding to the request, and URL previews will no longer fail if the remote server returns a Content-Type header with the chartype in quotes. ([\#4157](https://github.com/matrix-org/synapse/issues/4157))
-- The hash_password script now works on Python 3. ([\#4161](https://github.com/matrix-org/synapse/issues/4161))
-- Fix noop checks when updating device keys, reducing spurious device list update notifications. ([\#4164](https://github.com/matrix-org/synapse/issues/4164))
-
-
-Deprecations and Removals
--------------------------
-
-- The disused and un-specced identicon generator has been removed. ([\#4106](https://github.com/matrix-org/synapse/issues/4106))
-- The obsolete and non-functional /pull federation endpoint has been removed. ([\#4118](https://github.com/matrix-org/synapse/issues/4118))
-- The deprecated v1 key exchange endpoints have been removed. ([\#4119](https://github.com/matrix-org/synapse/issues/4119))
-- Synapse will no longer fetch keys using the fallback deprecated v1 key exchange method and will now always use v2. ([\#4120](https://github.com/matrix-org/synapse/issues/4120))
-
-
-Internal Changes
-----------------
-
-- Fix build of Docker image with docker-compose ([\#3778](https://github.com/matrix-org/synapse/issues/3778))
-- Delete unreferenced state groups during history purge ([\#4006](https://github.com/matrix-org/synapse/issues/4006))
-- The "Received rdata" log messages on workers is now logged at DEBUG, not INFO. ([\#4108](https://github.com/matrix-org/synapse/issues/4108))
-- Reduce replication traffic for device lists ([\#4109](https://github.com/matrix-org/synapse/issues/4109))
-- Fix `synapse_replication_tcp_protocol_*_commands` metric label to be full command name, rather than just the first character ([\#4110](https://github.com/matrix-org/synapse/issues/4110))
-- Log some bits about room creation ([\#4121](https://github.com/matrix-org/synapse/issues/4121))
-- Fix `tox` failure on old systems ([\#4124](https://github.com/matrix-org/synapse/issues/4124))
-- Add STATE_V2_TEST room version ([\#4128](https://github.com/matrix-org/synapse/issues/4128))
-- Clean up event accesses and tests ([\#4137](https://github.com/matrix-org/synapse/issues/4137))
-- The default logging config will now set an explicit log file encoding of UTF-8. ([\#4138](https://github.com/matrix-org/synapse/issues/4138))
-- Add helpers functions for getting prev and auth events of an event ([\#4139](https://github.com/matrix-org/synapse/issues/4139))
-- Add some tests for the HTTP pusher. ([\#4149](https://github.com/matrix-org/synapse/issues/4149))
-- add purge_history.sh and purge_remote_media.sh scripts to contrib/ ([\#4155](https://github.com/matrix-org/synapse/issues/4155))
-- HTTP tests have been refactored to contain less boilerplate. ([\#4156](https://github.com/matrix-org/synapse/issues/4156))
-- Drop incoming events from federation for unknown rooms ([\#4165](https://github.com/matrix-org/synapse/issues/4165))
-
-
-Synapse 0.33.8 (2018-11-01)
-===========================
-
-No significant changes.
-
-
-Synapse 0.33.8rc2 (2018-10-31)
-==============================
-
-Bugfixes
---------
-
-- Searches that request profile info now no longer fail with a 500. Fixes
-  a regression in 0.33.8rc1. ([\#4122](https://github.com/matrix-org/synapse/issues/4122))
-
-
-Synapse 0.33.8rc1 (2018-10-29)
-==============================
-
-Features
---------
-
-- Servers with auto-join rooms will now automatically create those rooms when the first user registers ([\#3975](https://github.com/matrix-org/synapse/issues/3975))
-- Add config option to control alias creation ([\#4051](https://github.com/matrix-org/synapse/issues/4051))
-- The register_new_matrix_user script is now ported to Python 3. ([\#4085](https://github.com/matrix-org/synapse/issues/4085))
-- Configure Docker image to listen on both ipv4 and ipv6. ([\#4089](https://github.com/matrix-org/synapse/issues/4089))
-
-
-Bugfixes
---------
-
-- Fix HTTP error response codes for federated group requests. ([\#3969](https://github.com/matrix-org/synapse/issues/3969))
-- Fix issue where Python 3 users couldn't paginate /publicRooms ([\#4046](https://github.com/matrix-org/synapse/issues/4046))
-- Fix URL previewing to work in Python 3.7 ([\#4050](https://github.com/matrix-org/synapse/issues/4050))
-- synctl will use the right python executable to run worker processes ([\#4057](https://github.com/matrix-org/synapse/issues/4057))
-- Manhole now works again on Python 3, instead of failing with a "couldn't match all kex parts" when connecting. ([\#4060](https://github.com/matrix-org/synapse/issues/4060), [\#4067](https://github.com/matrix-org/synapse/issues/4067))
-- Fix some metrics being racy and causing exceptions when polled by Prometheus. ([\#4061](https://github.com/matrix-org/synapse/issues/4061))
-- Fix bug which prevented email notifications from being sent unless an absolute path was given for `email_templates`. ([\#4068](https://github.com/matrix-org/synapse/issues/4068))
-- Correctly account for cpu usage by background threads ([\#4074](https://github.com/matrix-org/synapse/issues/4074))
-- Fix race condition where config defined reserved users were not being added to
-  the monthly active user list prior to the homeserver reactor firing up ([\#4081](https://github.com/matrix-org/synapse/issues/4081))
-- Fix bug which prevented backslashes being used in event field filters ([\#4083](https://github.com/matrix-org/synapse/issues/4083))
-
-
-Internal Changes
-----------------
-
-- Add information about the [matrix-docker-ansible-deploy](https://github.com/spantaleev/matrix-docker-ansible-deploy) playbook ([\#3698](https://github.com/matrix-org/synapse/issues/3698))
-- Add initial implementation of new state resolution algorithm ([\#3786](https://github.com/matrix-org/synapse/issues/3786))
-- Reduce database load when fetching state groups ([\#4011](https://github.com/matrix-org/synapse/issues/4011))
-- Various cleanups in the federation client code ([\#4031](https://github.com/matrix-org/synapse/issues/4031))
-- Run the CircleCI builds in docker containers ([\#4041](https://github.com/matrix-org/synapse/issues/4041))
-- Only colourise synctl output when attached to tty ([\#4049](https://github.com/matrix-org/synapse/issues/4049))
-- Refactor room alias creation code ([\#4063](https://github.com/matrix-org/synapse/issues/4063))
-- Make the Python scripts in the top-level scripts folders meet pep8 and pass flake8. ([\#4068](https://github.com/matrix-org/synapse/issues/4068))
-- The README now contains example for the Caddy web server. Contributed by steamp0rt. ([\#4072](https://github.com/matrix-org/synapse/issues/4072))
-- Add psutil as an explicit dependency ([\#4073](https://github.com/matrix-org/synapse/issues/4073))
-- Clean up threading and logcontexts in pushers ([\#4075](https://github.com/matrix-org/synapse/issues/4075))
-- Correctly manage logcontexts during startup to fix some "Unexpected logging context" warnings ([\#4076](https://github.com/matrix-org/synapse/issues/4076))
-- Give some more things logcontexts ([\#4077](https://github.com/matrix-org/synapse/issues/4077))
-- Clean up some bits of code which were flagged by the linter ([\#4082](https://github.com/matrix-org/synapse/issues/4082))
-
-
-Synapse 0.33.7 (2018-10-18)
-===========================
-
-**Warning**: This release removes the example email notification templates from
-`res/templates` (they are now internal to the python package). This should only
-affect you if you (a) deploy your Synapse instance from a git checkout or a
-github snapshot URL, and (b) have email notifications enabled.
-
-If you have email notifications enabled, you should ensure that
-`email.template_dir` is either configured to point at a directory where you
-have installed customised templates, or leave it unset to use the default
-templates.
-
-Synapse 0.33.7rc2 (2018-10-17)
-==============================
-
-Features
---------
-
-- Ship the example email templates as part of the package ([\#4052](https://github.com/matrix-org/synapse/issues/4052))
-
-Bugfixes
---------
-
-- Fix bug which made get_missing_events return too few events ([\#4045](https://github.com/matrix-org/synapse/issues/4045))
-
-
-Synapse 0.33.7rc1 (2018-10-15)
-==============================
-
-Features
---------
-
-- Add support for end-to-end key backup (MSC1687) ([\#4019](https://github.com/matrix-org/synapse/issues/4019))
-
-
-Bugfixes
---------
-
-- Fix bug in event persistence logic which caused 'NoneType is not iterable' ([\#3995](https://github.com/matrix-org/synapse/issues/3995))
-- Fix exception in background metrics collection ([\#3996](https://github.com/matrix-org/synapse/issues/3996))
-- Fix exception handling in fetching remote profiles ([\#3997](https://github.com/matrix-org/synapse/issues/3997))
-- Fix handling of rejected threepid invites ([\#3999](https://github.com/matrix-org/synapse/issues/3999))
-- Workers now start on Python 3. ([\#4027](https://github.com/matrix-org/synapse/issues/4027))
-- Synapse now starts on Python 3.7. ([\#4033](https://github.com/matrix-org/synapse/issues/4033))
-
-
-Internal Changes
-----------------
-
-- Log exceptions in looping calls ([\#4008](https://github.com/matrix-org/synapse/issues/4008))
-- Optimisation for serving federation requests ([\#4017](https://github.com/matrix-org/synapse/issues/4017))
-- Add metric to count number of non-empty sync responses ([\#4022](https://github.com/matrix-org/synapse/issues/4022))
-
-
-Synapse 0.33.6 (2018-10-04)
-===========================
-
-Internal Changes
-----------------
-
-- Pin to prometheus_client<0.4 to avoid renaming all of our metrics ([\#4002](https://github.com/matrix-org/synapse/issues/4002))
-
-
-Synapse 0.33.6rc1 (2018-10-03)
-==============================
-
-Features
---------
-
-- Adding the ability to change MAX_UPLOAD_SIZE for the docker container variables. ([\#3883](https://github.com/matrix-org/synapse/issues/3883))
-- Report "python_version" in the phone home stats ([\#3894](https://github.com/matrix-org/synapse/issues/3894))
-- Always LL ourselves if we're in a room ([\#3916](https://github.com/matrix-org/synapse/issues/3916))
-- Include eventid in log lines when processing incoming federation transactions ([\#3959](https://github.com/matrix-org/synapse/issues/3959))
-- Remove spurious check which made 'localhost' servers not work ([\#3964](https://github.com/matrix-org/synapse/issues/3964))
-
-
-Bugfixes
---------
-
-- Fix problem when playing media from Chrome using direct URL (thanks @remjey!) ([\#3578](https://github.com/matrix-org/synapse/issues/3578))
-- support registering regular users non-interactively with register_new_matrix_user script ([\#3836](https://github.com/matrix-org/synapse/issues/3836))
-- Fix broken invite email links for self hosted riots ([\#3868](https://github.com/matrix-org/synapse/issues/3868))
-- Don't ratelimit autojoins ([\#3879](https://github.com/matrix-org/synapse/issues/3879))
-- Fix 500 error when deleting unknown room alias ([\#3889](https://github.com/matrix-org/synapse/issues/3889))
-- Fix some b'abcd' noise in logs and metrics ([\#3892](https://github.com/matrix-org/synapse/issues/3892), [\#3895](https://github.com/matrix-org/synapse/issues/3895))
-- When we join a room, always try the server we used for the alias lookup first, to avoid unresponsive and out-of-date servers. ([\#3899](https://github.com/matrix-org/synapse/issues/3899))
-- Fix incorrect server-name indication for outgoing federation requests ([\#3907](https://github.com/matrix-org/synapse/issues/3907))
-- Fix adding client IPs to the database failing on Python 3. ([\#3908](https://github.com/matrix-org/synapse/issues/3908))
-- Fix bug where things occaisonally were not being timed out correctly. ([\#3910](https://github.com/matrix-org/synapse/issues/3910))
-- Fix bug where outbound federation would stop talking to some servers when using workers ([\#3914](https://github.com/matrix-org/synapse/issues/3914))
-- Fix some instances of ExpiringCache not expiring cache items ([\#3932](https://github.com/matrix-org/synapse/issues/3932), [\#3980](https://github.com/matrix-org/synapse/issues/3980))
-- Fix out-of-bounds error when LLing yourself ([\#3936](https://github.com/matrix-org/synapse/issues/3936))
-- Sending server notices regarding user consent now works on Python 3. ([\#3938](https://github.com/matrix-org/synapse/issues/3938))
-- Fix exceptions from metrics handler ([\#3956](https://github.com/matrix-org/synapse/issues/3956))
-- Fix error message for events with m.room.create missing from auth_events ([\#3960](https://github.com/matrix-org/synapse/issues/3960))
-- Fix errors due to concurrent monthly_active_user upserts ([\#3961](https://github.com/matrix-org/synapse/issues/3961))
-- Fix exceptions when processing incoming events over federation ([\#3968](https://github.com/matrix-org/synapse/issues/3968))
-- Replaced all occurences of e.message with str(e). Contributed by Schnuffle ([\#3970](https://github.com/matrix-org/synapse/issues/3970))
-- Fix lazy loaded sync in the presence of rejected state events ([\#3986](https://github.com/matrix-org/synapse/issues/3986))
-- Fix error when logging incomplete HTTP requests ([\#3990](https://github.com/matrix-org/synapse/issues/3990))
-
-
-Internal Changes
-----------------
-
-- Unit tests can now be run under PostgreSQL in Docker using ``test_postgresql.sh``. ([\#3699](https://github.com/matrix-org/synapse/issues/3699))
-- Speed up calculation of typing updates for replication ([\#3794](https://github.com/matrix-org/synapse/issues/3794))
-- Remove documentation regarding installation on Cygwin, the use of WSL is recommended instead. ([\#3873](https://github.com/matrix-org/synapse/issues/3873))
-- Fix typo in README, synaspse -> synapse ([\#3897](https://github.com/matrix-org/synapse/issues/3897))
-- Increase the timeout when filling missing events in federation requests ([\#3903](https://github.com/matrix-org/synapse/issues/3903))
-- Improve the logging when handling a federation transaction ([\#3904](https://github.com/matrix-org/synapse/issues/3904), [\#3966](https://github.com/matrix-org/synapse/issues/3966))
-- Improve logging of outbound federation requests ([\#3906](https://github.com/matrix-org/synapse/issues/3906), [\#3909](https://github.com/matrix-org/synapse/issues/3909))
-- Fix the docker image building on python 3 ([\#3911](https://github.com/matrix-org/synapse/issues/3911))
-- Add a regression test for logging failed HTTP requests on Python 3. ([\#3912](https://github.com/matrix-org/synapse/issues/3912))
-- Comments and interface cleanup for on_receive_pdu ([\#3924](https://github.com/matrix-org/synapse/issues/3924))
-- Fix spurious exceptions when remote http client closes conncetion ([\#3925](https://github.com/matrix-org/synapse/issues/3925))
-- Log exceptions thrown by background tasks ([\#3927](https://github.com/matrix-org/synapse/issues/3927))
-- Add a cache to get_destination_retry_timings ([\#3933](https://github.com/matrix-org/synapse/issues/3933), [\#3991](https://github.com/matrix-org/synapse/issues/3991))
-- Automate pushes to docker hub ([\#3946](https://github.com/matrix-org/synapse/issues/3946))
-- Require attrs 16.0.0 or later ([\#3947](https://github.com/matrix-org/synapse/issues/3947))
-- Fix incompatibility with python3 on alpine ([\#3948](https://github.com/matrix-org/synapse/issues/3948))
-- Run the test suite on the oldest supported versions of our dependencies in CI. ([\#3952](https://github.com/matrix-org/synapse/issues/3952))
-- CircleCI now only runs merged jobs on PRs, and commit jobs on develop, master, and release branches. ([\#3957](https://github.com/matrix-org/synapse/issues/3957))
-- Fix docstrings and add tests for state store methods ([\#3958](https://github.com/matrix-org/synapse/issues/3958))
-- fix docstring for FederationClient.get_state_for_room ([\#3963](https://github.com/matrix-org/synapse/issues/3963))
-- Run notify_app_services as a bg process ([\#3965](https://github.com/matrix-org/synapse/issues/3965))
-- Clarifications in FederationHandler ([\#3967](https://github.com/matrix-org/synapse/issues/3967))
-- Further reduce the docker image size ([\#3972](https://github.com/matrix-org/synapse/issues/3972))
-- Build py3 docker images for docker hub too ([\#3976](https://github.com/matrix-org/synapse/issues/3976))
-- Updated the installation instructions to point to the matrix-synapse package on PyPI. ([\#3985](https://github.com/matrix-org/synapse/issues/3985))
-- Disable USE_FROZEN_DICTS for unittests by default. ([\#3987](https://github.com/matrix-org/synapse/issues/3987))
-- Remove unused Jenkins and development related files from the repo. ([\#3988](https://github.com/matrix-org/synapse/issues/3988))
-- Improve stacktraces in certain exceptions in the logs ([\#3989](https://github.com/matrix-org/synapse/issues/3989))
-
-
 Synapse 0.33.5.1 (2018-09-25)
 =============================
 
 Internal Changes
 ----------------
 
-- Fix incompatibility with older Twisted version in tests. Thanks @OlegGirko! ([\#3940](https://github.com/matrix-org/synapse/issues/3940))
+- Fix incompatibility with older Twisted version in tests. Thanks 
+  @OlegGirko! ([\#3940](https://github.com/matrix-org/synapse/issues/3940))
 
 
 Synapse 0.33.5 (2018-09-24)

CONTRIBUTING.rst

@@ -30,7 +30,7 @@ use github's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Travis CI
+We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Travis CI 
 <https://travis-ci.org/matrix-org/synapse>`_ for continuous integration. All
 pull requests to synapse get automatically tested by Travis and CircleCI.
 If your change breaks the build, this will be shown in GitHub, so please
@@ -74,39 +74,16 @@ entry. These are managed by Towncrier
 To create a changelog entry, make a new file in the ``changelog.d``
 file named in the format of ``PRnumber.type``. The type can be
 one of ``feature``, ``bugfix``, ``removal`` (also used for
-deprecations), or ``misc`` (for internal-only changes).
-
-The content of the file is your changelog entry, which can contain Markdown
-formatting. The entry should end with a full stop ('.') for consistency.
-
-Adding credits to the changelog is encouraged, we value your
-contributions and would like to have you shouted out in the release notes!
+deprecations), or ``misc`` (for internal-only changes). The content of
+the file is your changelog entry, which can contain Markdown
+formatting. Adding credits to the changelog is encouraged, we value
+your contributions and would like to have you shouted out in the
+release notes!
 
 For example, a fix in PR #1234 would have its changelog entry in
 ``changelog.d/1234.bugfix``, and contain content like "The security levels of
 Florbs are now validated when recieved over federation. Contributed by Jane
-Matrix.".
-
-Debian changelog
-----------------
-
-Changes which affect the debian packaging files (in ``debian``) are an
-exception.
-
-In this case, you will need to add an entry to the debian changelog for the
-next release. For this, run the following command::
-
-  dch
-
-This will make up a new version number (if there isn't already an unreleased
-version in flight), and open an editor where you can add a new changelog entry.
-(Our release process will ensure that the version number and maintainer name is
-corrected for the release.)
-
-If your change affects both the debian packaging *and* files outside the debian
-directory, you will need both a regular newsfragment *and* an entry in the
-debian changelog. (Though typically such changes should be submitted as two
-separate pull requests.)
+Matrix".
 
 Attribution
 ~~~~~~~~~~~
@@ -125,7 +102,7 @@ Sign off
 In order to have a concrete record that your contribution is intentional
 and you agree to license it under the same terms as the project's license, we've adopted the
 same lightweight approach that the Linux Kernel
-`submitting patches process <https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin>`_, Docker
+(https://www.kernel.org/doc/Documentation/SubmittingPatches), Docker
 (https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other
 projects use: the DCO (Developer Certificate of Origin:
 http://developercertificate.org/). This is a simple declaration that you wrote

INSTALL.md

@@ -1,460 +0,0 @@
-- [Installing Synapse](#installing-synapse)
-  - [Installing from source](#installing-from-source)
-    - [Platform-Specific Instructions](#platform-specific-instructions)
-    - [Troubleshooting Installation](#troubleshooting-installation)
-  - [Prebuilt packages](#prebuilt-packages)
-- [Setting up Synapse](#setting-up-synapse)
-  - [TLS certificates](#tls-certificates)
-  - [Email](#email)
-  - [Registering a user](#registering-a-user)
-  - [Setting up a TURN server](#setting-up-a-turn-server)
-  - [URL previews](#url-previews)
-
-# Installing Synapse
-
-## Installing from source
-
-(Prebuilt packages are available for some platforms - see [Prebuilt packages](#prebuilt-packages).)
-
-System requirements:
-
-- POSIX-compliant system (tested on Linux & OS X)
-- Python 3.5, 3.6, 3.7, or 2.7
-- At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
-
-Synapse is written in Python but some of the libraries it uses are written in
-C. So before we can install Synapse itself we need a working C compiler and the
-header files for Python C extensions. See [Platform-Specific
-Instructions](#platform-specific-instructions) for information on installing
-these on various platforms.
-
-To install the Synapse homeserver run:
-
-```
-mkdir -p ~/synapse
-virtualenv -p python3 ~/synapse/env
-source ~/synapse/env/bin/activate
-pip install --upgrade pip
-pip install --upgrade setuptools
-pip install matrix-synapse
-```
-
-This will download Synapse from [PyPI](https://pypi.org/project/matrix-synapse)
-and install it, along with the python libraries it uses, into a virtual environment
-under `~/synapse/env`.  Feel free to pick a different directory if you
-prefer.
-
-This Synapse installation can then be later upgraded by using pip again with the
-update flag:
-
-```
-source ~/synapse/env/bin/activate
-pip install -U matrix-synapse
-```
-
-Before you can start Synapse, you will need to generate a configuration
-file. To do this, run (in your virtualenv, as before)::
-
-```
-cd ~/synapse
-python -m synapse.app.homeserver \
-    --server-name my.domain.name \
-    --config-path homeserver.yaml \
-    --generate-config \
-    --report-stats=[yes|no]
-```
-
-... substituting an appropriate value for `--server-name`. The server name
-determines the "domain" part of user-ids for users on your server: these will
-all be of the format `@user:my.domain.name`. It also determines how other
-matrix servers will reach yours for Federation. For a test configuration,
-set this to the hostname of your server. For a more production-ready setup, you
-will probably want to specify your domain (`example.com`) rather than a
-matrix-specific hostname here (in the same way that your email address is
-probably `user@example.com` rather than `user@email.example.com`) - but
-doing so may require more advanced setup: see [Setting up Federation](docs/federate.md).
-Beware that the server name cannot be changed later.
-
-This command will generate you a config file that you can then customise, but it will
-also generate a set of keys for you. These keys will allow your Home Server to
-identify itself to other Home Servers, so don't lose or delete them. It would be
-wise to back them up somewhere safe. (If, for whatever reason, you do need to
-change your Home Server's keys, you may find that other Home Servers have the
-old key cached. If you update the signing key, you should change the name of the
-key in the `<server name>.signing.key` file (the second word) to something
-different. See the
-[spec](https://matrix.org/docs/spec/server_server/latest.html#retrieving-server-keys)
-for more information on key management.)
-
-You will need to give Synapse a TLS certficate before it will start - see [TLS
-certificates](#tls-certificates).
-
-To actually run your new homeserver, pick a working directory for Synapse to
-run (e.g. `~/synapse`), and::
-
-    cd ~/synapse
-    source env/bin/activate
-    synctl start
-
-### Platform-Specific Instructions
-
-#### Debian/Ubuntu/Raspbian
-
-Installing prerequisites on Ubuntu or Debian:
-
-```
-sudo apt-get install build-essential python3-dev libffi-dev \
-                     python-pip python-setuptools sqlite3 \
-                     libssl-dev python-virtualenv libjpeg-dev libxslt1-dev
-```
-
-#### ArchLinux
-
-Installing prerequisites on ArchLinux:
-
-```
-sudo pacman -S base-devel python python-pip \
-               python-setuptools python-virtualenv sqlite3
-```
-
-#### CentOS/Fedora
-
-Installing prerequisites on CentOS 7 or Fedora 25:
-
-```
-sudo yum install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
-                 lcms2-devel libwebp-devel tcl-devel tk-devel redhat-rpm-config \
-                 python-virtualenv libffi-devel openssl-devel
-sudo yum groupinstall "Development Tools"
-```
-
-#### Mac OS X
-
-Installing prerequisites on Mac OS X:
-
-```
-xcode-select --install
-sudo easy_install pip
-sudo pip install virtualenv
-brew install pkg-config libffi
-```
-
-#### OpenSUSE
-
-Installing prerequisites on openSUSE:
-
-```
-sudo zypper in -t pattern devel_basis
-sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \
-               python-devel libffi-devel libopenssl-devel libjpeg62-devel
-```
-
-#### OpenBSD
-
-Installing prerequisites on OpenBSD:
-
-```
-doas pkg_add python libffi py-pip py-setuptools sqlite3 py-virtualenv \
-              libxslt jpeg
-```
-
-There is currently no port for OpenBSD. Additionally, OpenBSD's security
-settings require a slightly more difficult installation process.
-
-XXX: I suspect this is out of date.
-
-1. Create a new directory in `/usr/local` called `_synapse`. Also, create a
-   new user called `_synapse` and set that directory as the new user's home.
-   This is required because, by default, OpenBSD only allows binaries which need
-   write and execute permissions on the same memory space to be run from
-   `/usr/local`.
-2. `su` to the new `_synapse` user and change to their home directory.
-3. Create a new virtualenv: `virtualenv -p python2.7 ~/.synapse`
-4. Source the virtualenv configuration located at
-   `/usr/local/_synapse/.synapse/bin/activate`. This is done in `ksh` by
-   using the `.` command, rather than `bash`'s `source`.
-5. Optionally, use `pip` to install `lxml`, which Synapse needs to parse
-   webpages for their titles.
-6. Use `pip` to install this repository: `pip install matrix-synapse`
-7. Optionally, change `_synapse`'s shell to `/bin/false` to reduce the
-   chance of a compromised Synapse server being used to take over your box.
-
-After this, you may proceed with the rest of the install directions.
-
-#### Windows
-
-If you wish to run or develop Synapse on Windows, the Windows Subsystem For
-Linux provides a Linux environment on Windows 10 which is capable of using the
-Debian, Fedora, or source installation methods. More information about WSL can
-be found at https://docs.microsoft.com/en-us/windows/wsl/install-win10 for
-Windows 10 and https://docs.microsoft.com/en-us/windows/wsl/install-on-server
-for Windows Server.
-
-### Troubleshooting Installation
-
-XXX a bunch of this is no longer relevant.
-
-Synapse requires pip 8 or later, so if your OS provides too old a version you
-may need to manually upgrade it::
-
-    sudo pip install --upgrade pip
-
-Installing may fail with `Could not find any downloads that satisfy the requirement pymacaroons-pynacl (from matrix-synapse==0.12.0)`.
-You can fix this by manually upgrading pip and virtualenv::
-
-    sudo pip install --upgrade virtualenv
-
-You can next rerun `virtualenv -p python3 synapse` to update the virtual env.
-
-Installing may fail during installing virtualenv with `InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.`
-You can fix this  by manually installing ndg-httpsclient::
-
-    pip install --upgrade ndg-httpsclient
-
-Installing may fail with `mock requires setuptools>=17.1. Aborting installation`.
-You can fix this by upgrading setuptools::
-
-    pip install --upgrade setuptools
-
-If pip crashes mid-installation for reason (e.g. lost terminal), pip may
-refuse to run until you remove the temporary installation directory it
-created. To reset the installation::
-
-    rm -rf /tmp/pip_install_matrix
-
-pip seems to leak *lots* of memory during installation.  For instance, a Linux
-host with 512MB of RAM may run out of memory whilst installing Twisted.  If this
-happens, you will have to individually install the dependencies which are
-failing, e.g.::
-
-    pip install twisted
-
-## Prebuilt packages
-
-As an alternative to installing from source, prebuilt packages are available
-for a number of platforms.
-
-### Docker images and Ansible playbooks
-
-There is an offical synapse image available at
-https://hub.docker.com/r/matrixdotorg/synapse which can be used with
-the docker-compose file available at [contrib/docker](contrib/docker). Further information on
-this including configuration options is available in the README on
-hub.docker.com.
-
-Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
-Dockerfile to automate a synapse server in a single Docker image, at
-https://hub.docker.com/r/avhost/docker-matrix/tags/
-
-Slavi Pantaleev has created an Ansible playbook,
-which installs the offical Docker image of Matrix Synapse
-along with many other Matrix-related services (Postgres database, riot-web, coturn, mxisd, SSL support, etc.).
-For more details, see
-https://github.com/spantaleev/matrix-docker-ansible-deploy
-
-
-### Debian/Ubuntu
-
-#### Matrix.org packages
-
-Matrix.org provides Debian/Ubuntu packages of the latest stable version of
-Synapse via https://packages.matrix.org/debian/. They are available for Debian
-9 (Stretch), Ubuntu 16.04 (Xenial), and later. To use them:
-
-```
-sudo apt install -y lsb-release wget apt-transport-https
-sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
-echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" |
-    sudo tee /etc/apt/sources.list.d/matrix-org.list
-sudo apt update
-sudo apt install matrix-synapse-py3
-```
-
-**Note**: if you followed a previous version of these instructions which
-recommended using `apt-key add` to add an old key from
-`https://matrix.org/packages/debian/`, you should note that this key has been
-revoked. You should remove the old key with `sudo apt-key remove
-C35EB17E1EAE708E6603A9B3AD0592FE47F0DF61`, and follow the above instructions to
-update your configuration.
-
-The fingerprint of the repository signing key (as shown by `gpg
-/usr/share/keyrings/matrix-org-archive-keyring.gpg`) is
-`AAF9AE843A7584B5A3E4CD2BCF45A512DE2DA058`.
-
-#### Downstream Debian/Ubuntu packages
-
-For `buster` and `sid`, Synapse is available in the Debian repositories and
-it should be possible to install it with simply:
-
-```
-    sudo apt install matrix-synapse
-```
-
-There is also a version of `matrix-synapse` in `stretch-backports`. Please see
-the [Debian documentation on
-backports](https://backports.debian.org/Instructions/) for information on how
-to use them.
-
-We do not recommend using the packages in downstream Ubuntu at this time, as
-they are old and suffer from known security vulnerabilities.
-
-### Fedora
-
-Synapse is in the Fedora repositories as `matrix-synapse`:
-
-```
-sudo dnf install matrix-synapse
-```
-
-Oleg Girko provides Fedora RPMs at
-https://obs.infoserver.lv/project/monitor/matrix-synapse
-
-### OpenSUSE
-
-Synapse is in the OpenSUSE repositories as `matrix-synapse`:
-
-```
-sudo zypper install matrix-synapse
-```
-
-### SUSE Linux Enterprise Server
-
-Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 repository at
-https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/
-
-### ArchLinux
-
-The quickest way to get up and running with ArchLinux is probably with the community package
-https://www.archlinux.org/packages/community/any/matrix-synapse/, which should pull in most of
-the necessary dependencies.
-
-pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):
-
-```
-sudo pip install --upgrade pip
-```
-
-If you encounter an error with lib bcrypt causing an Wrong ELF Class:
-ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly
-compile it under the right architecture. (This should not be needed if
-installing under virtualenv):
-
-```
-sudo pip uninstall py-bcrypt
-sudo pip install py-bcrypt
-```
-
-### FreeBSD
-
-Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from:
-
- - Ports: `cd /usr/ports/net-im/py-matrix-synapse && make install clean`
- - Packages: `pkg install py27-matrix-synapse`
-
-
-### NixOS
-
-Robin Lambertz has packaged Synapse for NixOS at:
-https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-synapse.nix
-
-# Setting up Synapse
-
-Once you have installed synapse as above, you will need to configure it.
-
-## TLS certificates
-
-The default configuration exposes a single HTTP port: http://localhost:8008. It
-is suitable for local testing, but for any practical use, you will either need
-to enable a reverse proxy, or configure Synapse to expose an HTTPS port.
-
-For information on using a reverse proxy, see
-[docs/reverse_proxy.rst](docs/reverse_proxy.rst).
-
-To configure Synapse to expose an HTTPS port, you will need to edit
-`homeserver.yaml`, as follows:
-
-* First, under the `listeners` section, uncomment the configuration for the
-  TLS-enabled listener. (Remove the hash sign (`#`) at the start of
-  each line). The relevant lines are like this:
-
-  ```
-    - port: 8448
-      type: http
-      tls: true
-      resources:
-        - names: [client, federation]
-  ```
-* You will also need to uncomment the `tls_certificate_path` and
-  `tls_private_key_path` lines under the `TLS` section. You can either
-  point these settings at an existing certificate and key, or you can
-  enable Synapse's built-in ACME (Let's Encrypt) support. Instructions
-  for having Synapse automatically provision and renew federation
-  certificates through ACME can be found at [ACME.md](docs/ACME.md). If you
-  are using your own certificate, be sure to use a `.pem` file that includes
-  the full certificate chain including any intermediate certificates (for
-  instance, if using certbot, use `fullchain.pem` as your certificate, not
-  `cert.pem`).
-
-For a more detailed guide to configuring your server for federation, see
-[federate.md](docs/federate.md)
-
-
-## Email
-
-It is desirable for Synapse to have the capability to send email. For example,
-this is required to support the 'password reset' feature.
-
-To configure an SMTP server for Synapse, modify the configuration section
-headed ``email``, and be sure to have at least the ``smtp_host``, ``smtp_port``
-and ``notif_from`` fields filled out. You may also need to set ``smtp_user``,
-``smtp_pass``, and ``require_transport_security``.
-
-If Synapse is not configured with an SMTP server, password reset via email will
- be disabled by default.
-
-## Registering a user
-
-You will need at least one user on your server in order to use a Matrix
-client. Users can be registered either via a Matrix client, or via a
-commandline script.
-
-To get started, it is easiest to use the command line to register new
-users. This can be done as follows:
-
-```
-$ source ~/synapse/env/bin/activate
-$ synctl start # if not already running
-$ register_new_matrix_user -c homeserver.yaml http://localhost:8008
-New user localpart: erikj
-Password:
-Confirm password:
-Make admin [no]:
-Success!
-```
-
-This process uses a setting `registration_shared_secret` in
-`homeserver.yaml`, which is shared between Synapse itself and the
-`register_new_matrix_user` script. It doesn't matter what it is (a random
-value is generated by `--generate-config`), but it should be kept secret, as
-anyone with knowledge of it can register users, including admin accounts,
-on your server even if `enable_registration` is `false`.
-
-## Setting up a TURN server
-
-For reliable VoIP calls to be routed via this homeserver, you MUST configure
-a TURN server.  See [docs/turn-howto.rst](docs/turn-howto.rst) for details.
-
-## URL previews
-
-Synapse includes support for previewing URLs, which is disabled by default.  To
-turn it on you must enable the `url_preview_enabled: True` config parameter
-and explicitly specify the IP ranges that Synapse is not allowed to spider for
-previewing in the `url_preview_ip_range_blacklist` configuration parameter.
-This is critical from a security perspective to stop arbitrary Matrix users
-spidering 'internal' URLs on your network.  At the very least we recommend that
-your loopback and RFC1918 IP addresses are blacklisted.
-
-This also requires the optional lxml and netaddr python dependencies to be
-installed.  This in turn requires the libxml2 library to be available - on
-Debian/Ubuntu this means `apt-get install libxml2-dev`, or equivalent for
-your OS.

MANIFEST.in

@@ -9,30 +9,27 @@ include demo/*.py
 include demo/*.sh
 
 recursive-include synapse/storage/schema *.sql
-recursive-include synapse/storage/schema *.sql.postgres
-recursive-include synapse/storage/schema *.sql.sqlite
 recursive-include synapse/storage/schema *.py
-recursive-include synapse/storage/schema *.txt
 
 recursive-include docs *
+recursive-include res *
 recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
 recursive-include tests *.py
-include tests/http/ca.crt
-include tests/http/ca.key
-include tests/http/server.key
 
-recursive-include synapse/res *
 recursive-include synapse/static *.css
 recursive-include synapse/static *.gif
 recursive-include synapse/static *.html
 recursive-include synapse/static *.js
 
+exclude jenkins.sh
+exclude jenkins*.sh
+exclude jenkins*
 exclude Dockerfile
 exclude .dockerignore
 exclude test_postgresql.sh
-exclude .editorconfig
+recursive-exclude jenkins *.sh
 
 include pyproject.toml
 recursive-include changelog.d *
@@ -41,15 +38,3 @@ prune .github
 prune demo/etc
 prune docker
 prune .circleci
-prune .coveragerc
-prune debian
-prune .codecov.yml
-prune .buildkite
-
-exclude jenkins*
-recursive-exclude jenkins *.sh
-
-# FIXME: we shouldn't have these templates here
-recursive-include res/templates-dinsic *.css
-recursive-include res/templates-dinsic *.html
-recursive-include res/templates-dinsic *.txt

MAP.rst

@@ -0,0 +1,35 @@
+Directory Structure
+===================
+
+Warning: this may be a bit stale...
+
+::
+
+    .
+    ├── cmdclient           Basic CLI python Matrix client
+    ├── demo                Scripts for running standalone Matrix demos
+    ├── docs                All doc, including the draft Matrix API spec
+    │   ├── client-server       The client-server Matrix API spec
+    │   ├── model               Domain-specific elements of the Matrix API spec
+    │   ├── server-server       The server-server model of the Matrix API spec
+    │   └── sphinx              The internal API doc of the Synapse homeserver
+    ├── experiments         Early experiments of using Synapse's internal APIs
+    ├── graph               Visualisation of Matrix's distributed message store 
+    ├── synapse             The reference Matrix homeserver implementation
+    │   ├── api                 Common building blocks for the APIs
+    │   │   ├── events              Definition of state representation Events 
+    │   │   └── streams             Definition of streamable Event objects
+    │   ├── app                 The __main__ entry point for the homeserver
+    │   ├── crypto              The PKI client/server used for secure federation
+    │   │   └── resource            PKI helper objects (e.g. keys)
+    │   ├── federation          Server-server state replication logic
+    │   ├── handlers            The main business logic of the homeserver
+    │   ├── http                Wrappers around Twisted's HTTP server & client
+    │   ├── rest                Servlet-style RESTful API
+    │   ├── storage             Persistence subsystem (currently only sqlite3)
+    │   │   └── schema              sqlite persistence schema
+    │   └── util                Synapse-specific utilities
+    ├── tests               Unit tests for the Synapse homeserver
+    └── webclient           Basic AngularJS Matrix web client
+
+

README.rst

@@ -26,6 +26,7 @@ via IRC bridge at irc://irc.freenode.net/matrix.
 Synapse is currently in rapid development, but as of version 0.5 we believe it
 is sufficiently stable to be run as an internet-facing service for real usage!
 
+
 About Matrix
 ============
 
@@ -80,34 +81,228 @@ Thanks for using Matrix!
 Synapse Installation
 ====================
 
-.. _federation:
+Synapse is the reference python/twisted Matrix homeserver implementation.
 
-* For details on how to install synapse, see `<INSTALL.md>`_.
-* For specific details on how to configure Synapse for federation see `docs/federate.md <docs/federate.md>`_
+System requirements:
+
+- POSIX-compliant system (tested on Linux & OS X)
+- Python 2.7
+- At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
+
+Installing from source
+----------------------
+(Prebuilt packages are available for some platforms - see `Platform-Specific
+Instructions`_.)
+
+Synapse is written in python but some of the libraries it uses are written in
+C. So before we can install synapse itself we need a working C compiler and the
+header files for python C extensions.
+
+Installing prerequisites on Ubuntu or Debian::
+
+    sudo apt-get install build-essential python2.7-dev libffi-dev \
+                         python-pip python-setuptools sqlite3 \
+                         libssl-dev python-virtualenv libjpeg-dev libxslt1-dev
+
+Installing prerequisites on ArchLinux::
+
+    sudo pacman -S base-devel python2 python-pip \
+                   python-setuptools python-virtualenv sqlite3
+
+Installing prerequisites on CentOS 7 or Fedora 25::
+
+    sudo yum install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
+                     lcms2-devel libwebp-devel tcl-devel tk-devel redhat-rpm-config \
+                     python-virtualenv libffi-devel openssl-devel
+    sudo yum groupinstall "Development Tools"
+
+Installing prerequisites on Mac OS X::
+
+    xcode-select --install
+    sudo easy_install pip
+    sudo pip install virtualenv
+    brew install pkg-config libffi
+
+Installing prerequisites on Raspbian::
+
+    sudo apt-get install build-essential python2.7-dev libffi-dev \
+                         python-pip python-setuptools sqlite3 \
+                         libssl-dev python-virtualenv libjpeg-dev
+    sudo pip install --upgrade pip
+    sudo pip install --upgrade ndg-httpsclient
+    sudo pip install --upgrade virtualenv
+
+Installing prerequisites on openSUSE::
+
+    sudo zypper in -t pattern devel_basis
+    sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \
+                   python-devel libffi-devel libopenssl-devel libjpeg62-devel
+
+Installing prerequisites on OpenBSD::
+
+    doas pkg_add python libffi py-pip py-setuptools sqlite3 py-virtualenv \
+                 libxslt
+
+To install the synapse homeserver run::
+
+    virtualenv -p python2.7 ~/.synapse
+    source ~/.synapse/bin/activate
+    pip install --upgrade pip
+    pip install --upgrade setuptools
+    pip install https://github.com/matrix-org/synapse/tarball/master
+
+This installs synapse, along with the libraries it uses, into a virtual
+environment under ``~/.synapse``.  Feel free to pick a different directory
+if you prefer.
+
+In case of problems, please see the _`Troubleshooting` section below.
+
+There is an offical synapse image available at
+https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with
+the docker-compose file available at `contrib/docker <contrib/docker>`_. Further information on
+this including configuration options is available in the README on
+hub.docker.com.
+
+Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
+Dockerfile to automate a synapse server in a single Docker image, at
+https://hub.docker.com/r/avhost/docker-matrix/tags/
+
+Configuring synapse
+-------------------
+
+Before you can start Synapse, you will need to generate a configuration
+file. To do this, run (in your virtualenv, as before)::
+
+    cd ~/.synapse
+    python -m synapse.app.homeserver \
+        --server-name my.domain.name \
+        --config-path homeserver.yaml \
+        --generate-config \
+        --report-stats=[yes|no]
+
+... substituting an appropriate value for ``--server-name``. The server name
+determines the "domain" part of user-ids for users on your server: these will
+all be of the format ``@user:my.domain.name``. It also determines how other
+matrix servers will reach yours for `Federation`_. For a test configuration,
+set this to the hostname of your server. For a more production-ready setup, you
+will probably want to specify your domain (``example.com``) rather than a
+matrix-specific hostname here (in the same way that your email address is
+probably ``user@example.com`` rather than ``user@email.example.com``) - but
+doing so may require more advanced setup - see `Setting up
+Federation`_. Beware that the server name cannot be changed later.
+
+This command will generate you a config file that you can then customise, but it will
+also generate a set of keys for you. These keys will allow your Home Server to
+identify itself to other Home Servers, so don't lose or delete them. It would be
+wise to back them up somewhere safe. (If, for whatever reason, you do need to
+change your Home Server's keys, you may find that other Home Servers have the
+old key cached. If you update the signing key, you should change the name of the
+key in the ``<server name>.signing.key`` file (the second word) to something
+different. See `the spec`__ for more information on key management.)
+
+.. __: `key_management`_
+
+The default configuration exposes two HTTP ports: 8008 and 8448. Port 8008 is
+configured without TLS; it should be behind a reverse proxy for TLS/SSL
+termination on port 443 which in turn should be used for clients. Port 8448
+is configured to use TLS with a self-signed certificate. If you would like
+to do initial test with a client without having to setup a reverse proxy,
+you can temporarly use another certificate. (Note that a self-signed
+certificate is fine for `Federation`_). You can do so by changing
+``tls_certificate_path``, ``tls_private_key_path`` and ``tls_dh_params_path``
+in ``homeserver.yaml``; alternatively, you can use a reverse-proxy, but be sure
+to read `Using a reverse proxy with Synapse`_ when doing so.
+
+Apart from port 8448 using TLS, both ports are the same in the default
+configuration.
+
+Registering a user
+------------------
+
+You will need at least one user on your server in order to use a Matrix
+client. Users can be registered either `via a Matrix client`__, or via a
+commandline script.
+
+.. __: `client-user-reg`_
+
+To get started, it is easiest to use the command line to register new users::
+
+    $ source ~/.synapse/bin/activate
+    $ synctl start # if not already running
+    $ register_new_matrix_user -c homeserver.yaml https://localhost:8448
+    New user localpart: erikj
+    Password:
+    Confirm password:
+    Make admin [no]:
+    Success!
+
+This process uses a setting ``registration_shared_secret`` in
+``homeserver.yaml``, which is shared between Synapse itself and the
+``register_new_matrix_user`` script. It doesn't matter what it is (a random
+value is generated by ``--generate-config``), but it should be kept secret, as
+anyone with knowledge of it can register users on your server even if
+``enable_registration`` is ``false``.
+
+Setting up a TURN server
+------------------------
+
+For reliable VoIP calls to be routed via this homeserver, you MUST configure
+a TURN server.  See `<docs/turn-howto.rst>`_ for details.
+
+IPv6
+----
+
+As of Synapse 0.19 we finally support IPv6, many thanks to @kyrias and @glyph
+for providing PR #1696.
+
+However, for federation to work on hosts with IPv6 DNS servers you **must**
+be running Twisted 17.1.0 or later - see https://github.com/matrix-org/synapse/issues/1002
+for details.  We can't make Synapse depend on Twisted 17.1 by default
+yet as it will break most older distributions (see https://github.com/matrix-org/synapse/pull/1909)
+so if you are using operating system dependencies you'll have to install your
+own Twisted 17.1 package via pip or backports etc.
+
+If you're running in a virtualenv then pip should have installed the newest
+Twisted automatically, but if your virtualenv is old you will need to manually
+upgrade to a newer Twisted dependency via:
+
+    pip install Twisted>=17.1.0
+
+
+Running Synapse
+===============
+
+To actually run your new homeserver, pick a working directory for Synapse to
+run (e.g. ``~/.synapse``), and::
+
+    cd ~/.synapse
+    source ./bin/activate
+    synctl start
 
 
 Connecting to Synapse from a client
 ===================================
 
 The easiest way to try out your new Synapse installation is by connecting to it
-from a web client.
+from a web client. The easiest option is probably the one at
+https://riot.im/app. You will need to specify a "Custom server" when you log on
+or register: set this to ``https://domain.tld`` if you setup a reverse proxy
+following the recommended setup, or ``https://localhost:8448`` - remember to specify the
+port (``:8448``) if not ``:443`` unless you changed the configuration. (Leave the identity
+server as the default - see `Identity servers`_.)
 
-Unless you are running a test instance of Synapse on your local machine, in
-general, you will need to enable TLS support before you can successfully
-connect from a client: see `<INSTALL.md#tls-certificates>`_.
-
-An easy way to get started is to login or register via Riot at
-https://riot.im/app/#/login or https://riot.im/app/#/register respectively.
-You will need to change the server you are logging into from ``matrix.org``
-and instead specify a Homeserver URL of ``https://<server_name>:8448``
-(or just ``https://<server_name>`` if you are using a reverse proxy).
-(Leave the identity server as the default - see `Identity servers`_.)
-If you prefer to use another client, refer to our
-`client breakdown <https://matrix.org/docs/projects/clients-matrix>`_.
+If using port 8448 you will run into errors until you accept the self-signed
+certificate. You can easily do this by going to ``https://localhost:8448``
+directly with your browser and accept the presented certificate. You can then
+go back in your web client and proceed further.
 
 If all goes well you should at least be able to log in, create a room, and
 start sending messages.
 
+(The homeserver runs a web client by default at https://localhost:8448/, though
+as of the time of writing it is somewhat outdated and not really recommended -
+https://github.com/matrix-org/synapse/issues/1527).
+
 .. _`client-user-reg`:
 
 Registering a new user from a client
@@ -120,9 +315,9 @@ recommended to also set up CAPTCHA - see `<docs/CAPTCHA_SETUP.rst>`_.)
 Once ``enable_registration`` is set to ``true``, it is possible to register a
 user via `riot.im <https://riot.im/app/#/register>`_ or other Matrix clients.
 
-Your new user name will be formed partly from the ``server_name``, and partly
-from a localpart you specify when you create the account. Your name will take
-the form of::
+Your new user name will be formed partly from the ``server_name`` (see
+`Configuring synapse`_), and partly from a localpart you specify when you
+create the account. Your name will take the form of::
 
     @localpart:my.domain.name
 
@@ -131,12 +326,6 @@ the form of::
 As when logging in, you will need to specify a "Custom server".  Specify your
 desired ``localpart`` in the 'User name' box.
 
-ACME setup
-==========
-
-For details on having Synapse manage your federation TLS certificates
-automatically, please see `<docs/ACME.md>`_.
-
 
 Security Note
 =============
@@ -151,10 +340,231 @@ content served to web browsers a matrix API from being able to attack webapps ho
 on the same domain.  This is particularly true of sharing a matrix webclient and
 server on the same domain.
 
-See https://github.com/vector-im/riot-web/issues/1977 and
+See https://github.com/vector-im/vector-web/issues/1977 and
 https://developer.github.com/changes/2014-04-25-user-content-security for more details.
 
 
+Platform-Specific Instructions
+==============================
+
+Debian
+------
+
+Matrix provides official Debian packages via apt from https://matrix.org/packages/debian/.
+Note that these packages do not include a client - choose one from
+https://matrix.org/docs/projects/try-matrix-now.html (or build your own with one of our SDKs :)
+
+Fedora
+------
+
+Synapse is in the Fedora repositories as ``matrix-synapse``::
+
+    sudo dnf install matrix-synapse
+
+Oleg Girko provides Fedora RPMs at
+https://obs.infoserver.lv/project/monitor/matrix-synapse
+
+OpenSUSE
+--------
+
+Synapse is in the OpenSUSE repositories as ``matrix-synapse``::
+
+    sudo zypper install matrix-synapse
+
+SUSE Linux Enterprise Server
+----------------------------
+
+Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 repository at
+https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/
+
+ArchLinux
+---------
+
+The quickest way to get up and running with ArchLinux is probably with the community package
+https://www.archlinux.org/packages/community/any/matrix-synapse/, which should pull in most of
+the necessary dependencies. If the default web client is to be served (enabled by default in
+the generated config),
+https://www.archlinux.org/packages/community/any/python2-matrix-angular-sdk/ will also need to
+be installed.
+
+Alternatively, to install using pip a few changes may be needed as ArchLinux
+defaults to python 3, but synapse currently assumes python 2.7 by default:
+
+pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 )::
+
+    sudo pip2.7 install --upgrade pip
+
+You also may need to explicitly specify python 2.7 again during the install
+request::
+
+    pip2.7 install https://github.com/matrix-org/synapse/tarball/master
+
+If you encounter an error with lib bcrypt causing an Wrong ELF Class:
+ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly
+compile it under the right architecture. (This should not be needed if
+installing under virtualenv)::
+
+    sudo pip2.7 uninstall py-bcrypt
+    sudo pip2.7 install py-bcrypt
+
+During setup of Synapse you need to call python2.7 directly again::
+
+    cd ~/.synapse
+    python2.7 -m synapse.app.homeserver \
+      --server-name machine.my.domain.name \
+      --config-path homeserver.yaml \
+      --generate-config
+
+...substituting your host and domain name as appropriate.
+
+FreeBSD
+-------
+
+Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from:
+
+ - Ports: ``cd /usr/ports/net-im/py-matrix-synapse && make install clean``
+ - Packages: ``pkg install py27-matrix-synapse``
+
+
+OpenBSD
+-------
+
+There is currently no port for OpenBSD. Additionally, OpenBSD's security
+settings require a slightly more difficult installation process.
+
+1) Create a new directory in ``/usr/local`` called ``_synapse``. Also, create a
+   new user called ``_synapse`` and set that directory as the new user's home.
+   This is required because, by default, OpenBSD only allows binaries which need
+   write and execute permissions on the same memory space to be run from
+   ``/usr/local``.
+2) ``su`` to the new ``_synapse`` user and change to their home directory.
+3) Create a new virtualenv: ``virtualenv -p python2.7 ~/.synapse``
+4) Source the virtualenv configuration located at
+   ``/usr/local/_synapse/.synapse/bin/activate``. This is done in ``ksh`` by
+   using the ``.`` command, rather than ``bash``'s ``source``.
+5) Optionally, use ``pip`` to install ``lxml``, which Synapse needs to parse
+   webpages for their titles.
+6) Use ``pip`` to install this repository: ``pip install
+   https://github.com/matrix-org/synapse/tarball/master``
+7) Optionally, change ``_synapse``'s shell to ``/bin/false`` to reduce the
+   chance of a compromised Synapse server being used to take over your box.
+
+After this, you may proceed with the rest of the install directions.
+
+NixOS
+-----
+
+Robin Lambertz has packaged Synapse for NixOS at:
+https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-synapse.nix
+
+Windows Install
+---------------
+
+If you wish to run or develop Synapse on Windows, the Windows Subsystem For
+Linux provides a Linux environment on Windows 10 which is capable of using the
+Debian, Fedora, or source installation methods. More information about WSL can
+be found at https://docs.microsoft.com/en-us/windows/wsl/install-win10 for
+Windows 10 and https://docs.microsoft.com/en-us/windows/wsl/install-on-server
+for Windows Server.
+
+Troubleshooting
+===============
+
+Troubleshooting Installation
+----------------------------
+
+Synapse requires pip 1.7 or later, so if your OS provides too old a version you
+may need to manually upgrade it::
+
+    sudo pip install --upgrade pip
+
+Installing may fail with ``Could not find any downloads that satisfy the requirement pymacaroons-pynacl (from matrix-synapse==0.12.0)``.
+You can fix this by manually upgrading pip and virtualenv::
+
+    sudo pip install --upgrade virtualenv
+
+You can next rerun ``virtualenv -p python2.7 synapse`` to update the virtual env.
+
+Installing may fail during installing virtualenv with ``InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.``
+You can fix this  by manually installing ndg-httpsclient::
+
+    pip install --upgrade ndg-httpsclient
+
+Installing may fail with ``mock requires setuptools>=17.1. Aborting installation``.
+You can fix this by upgrading setuptools::
+
+    pip install --upgrade setuptools
+
+If pip crashes mid-installation for reason (e.g. lost terminal), pip may
+refuse to run until you remove the temporary installation directory it
+created. To reset the installation::
+
+    rm -rf /tmp/pip_install_matrix
+
+pip seems to leak *lots* of memory during installation.  For instance, a Linux
+host with 512MB of RAM may run out of memory whilst installing Twisted.  If this
+happens, you will have to individually install the dependencies which are
+failing, e.g.::
+
+    pip install twisted
+
+On OS X, if you encounter clang: error: unknown argument: '-mno-fused-madd' you
+will need to export CFLAGS=-Qunused-arguments.
+
+Troubleshooting Running
+-----------------------
+
+If synapse fails with ``missing "sodium.h"`` crypto errors, you may need
+to manually upgrade PyNaCL, as synapse uses NaCl (https://nacl.cr.yp.to/) for
+encryption and digital signatures.
+Unfortunately PyNACL currently has a few issues
+(https://github.com/pyca/pynacl/issues/53) and
+(https://github.com/pyca/pynacl/issues/79) that mean it may not install
+correctly, causing all tests to fail with errors about missing "sodium.h". To
+fix try re-installing from PyPI or directly from
+(https://github.com/pyca/pynacl)::
+
+    # Install from PyPI
+    pip install --user --upgrade --force pynacl
+
+    # Install from github
+    pip install --user https://github.com/pyca/pynacl/tarball/master
+
+Running out of File Handles
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If synapse runs out of filehandles, it typically fails badly - live-locking
+at 100% CPU, and/or failing to accept new TCP connections (blocking the
+connecting client).  Matrix currently can legitimately use a lot of file handles,
+thanks to busy rooms like #matrix:matrix.org containing hundreds of participating
+servers.  The first time a server talks in a room it will try to connect
+simultaneously to all participating servers, which could exhaust the available
+file descriptors between DNS queries & HTTPS sockets, especially if DNS is slow
+to respond.  (We need to improve the routing algorithm used to be better than
+full mesh, but as of June 2017 this hasn't happened yet).
+
+If you hit this failure mode, we recommend increasing the maximum number of
+open file handles to be at least 4096 (assuming a default of 1024 or 256).
+This is typically done by editing ``/etc/security/limits.conf``
+
+Separately, Synapse may leak file handles if inbound HTTP requests get stuck
+during processing - e.g. blocked behind a lock or talking to a remote server etc.
+This is best diagnosed by matching up the 'Received request' and 'Processed request'
+log lines and looking for any 'Processed request' lines which take more than
+a few seconds to execute.  Please let us know at #matrix-dev:matrix.org if
+you see this failure mode so we can help debug it, however.
+
+ArchLinux
+~~~~~~~~~
+
+If running `$ synctl start` fails with 'returned non-zero exit status 1',
+you will need to explicitly call Python2.7 - either running as::
+
+    python2.7 -m synapse.app.homeserver --daemonize -c homeserver.yaml
+
+...or by editing synctl with the correct python executable.
+
+
 Upgrading an existing Synapse
 =============================
 
@@ -164,19 +574,100 @@ versions of synapse.
 
 .. _UPGRADE.rst: UPGRADE.rst
 
+.. _federation:
+
+Setting up Federation
+=====================
+
+Federation is the process by which users on different servers can participate
+in the same room. For this to work, those other servers must be able to contact
+yours to send messages.
+
+As explained in `Configuring synapse`_, the ``server_name`` in your
+``homeserver.yaml`` file determines the way that other servers will reach
+yours. By default, they will treat it as a hostname and try to connect to
+port 8448. This is easy to set up and will work with the default configuration,
+provided you set the ``server_name`` to match your machine's public DNS
+hostname.
+
+For a more flexible configuration, you can set up a DNS SRV record. This allows
+you to run your server on a machine that might not have the same name as your
+domain name. For example, you might want to run your server at
+``synapse.example.com``, but have your Matrix user-ids look like
+``@user:example.com``. (A SRV record also allows you to change the port from
+the default 8448. However, if you are thinking of using a reverse-proxy on the
+federation port, which is not recommended, be sure to read
+`Reverse-proxying the federation port`_ first.)
+
+To use a SRV record, first create your SRV record and publish it in DNS. This
+should have the format ``_matrix._tcp.<yourdomain.com> <ttl> IN SRV 10 0 <port>
+<synapse.server.name>``. The DNS record should then look something like::
+
+    $ dig -t srv _matrix._tcp.example.com
+    _matrix._tcp.example.com. 3600    IN      SRV     10 0 8448 synapse.example.com.
+
+Note that the server hostname cannot be an alias (CNAME record): it has to point
+directly to the server hosting the synapse instance.
+
+You can then configure your homeserver to use ``<yourdomain.com>`` as the domain in
+its user-ids, by setting ``server_name``::
+
+    python -m synapse.app.homeserver \
+        --server-name <yourdomain.com> \
+        --config-path homeserver.yaml \
+        --generate-config
+    python -m synapse.app.homeserver --config-path homeserver.yaml
+
+If you've already generated the config file, you need to edit the ``server_name``
+in your ``homeserver.yaml`` file. If you've already started Synapse and a
+database has been created, you will have to recreate the database.
+
+If all goes well, you should be able to `connect to your server with a client`__,
+and then join a room via federation. (Try ``#matrix-dev:matrix.org`` as a first
+step. "Matrix HQ"'s sheer size and activity level tends to make even the
+largest boxes pause for thought.)
+
+.. __: `Connecting to Synapse from a client`_
+
+Troubleshooting
+---------------
+
+You can use the federation tester to check if your homeserver is all set:
+``https://matrix.org/federationtester/api/report?server_name=<your_server_name>``
+If any of the attributes under "checks" is false, federation won't work.
+
+The typical failure mode with federation is that when you try to join a room,
+it is rejected with "401: Unauthorized". Generally this means that other
+servers in the room couldn't access yours. (Joining a room over federation is a
+complicated dance which requires connections in both directions).
+
+So, things to check are:
+
+* If you are trying to use a reverse-proxy, read `Reverse-proxying the
+  federation port`_.
+* If you are not using a SRV record, check that your ``server_name`` (the part
+  of your user-id after the ``:``) matches your hostname, and that port 8448 on
+  that hostname is reachable from outside your network.
+* If you *are* using a SRV record, check that it matches your ``server_name``
+  (it should be ``_matrix._tcp.<server_name>``), and that the port and hostname
+  it specifies are reachable from outside your network.
+
+Running a Demo Federation of Synapses
+-------------------------------------
+
+If you want to get up and running quickly with a trio of homeservers in a
+private federation, there is a script in the ``demo`` directory. This is mainly
+useful just for development purposes. See `<demo/README>`_.
+
 
 Using PostgreSQL
 ================
 
-Synapse offers two database engines:
- * `SQLite <https://sqlite.org/>`_
- * `PostgreSQL <https://www.postgresql.org>`_
+As of Synapse 0.9, `PostgreSQL <https://www.postgresql.org>`_ is supported as an
+alternative to the `SQLite <https://sqlite.org/>`_ database that Synapse has
+traditionally used for convenience and simplicity.
 
-By default Synapse uses SQLite in and doing so trades performance for convenience.
-SQLite is only recommended in Synapse for testing purposes or for servers with
-light workloads.
-
-Almost all installations should opt to use PostreSQL. Advantages include:
+The advantages of Postgres include:
 
 * significant performance improvements due to the superior threading and
   caching model, smarter query optimiser
@@ -188,6 +679,7 @@ Almost all installations should opt to use PostreSQL. Advantages include:
 For information on how to install and use PostgreSQL, please see
 `docs/postgres.rst <docs/postgres.rst>`_.
 
+
 .. _reverse-proxy:
 
 Using a reverse proxy with Synapse
@@ -195,13 +687,114 @@ Using a reverse proxy with Synapse
 
 It is recommended to put a reverse proxy such as
 `nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
-`Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_,
-`Caddy <https://caddyserver.com/docs/proxy>`_ or
+`Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_ or
 `HAProxy <https://www.haproxy.org/>`_ in front of Synapse. One advantage of
 doing so is that it means that you can expose the default https port (443) to
 Matrix clients without needing to run Synapse with root privileges.
 
-For information on configuring one, see `<docs/reverse_proxy.rst>`_.
+The most important thing to know here is that Matrix clients and other Matrix
+servers do not necessarily need to connect to your server via the same
+port. Indeed, clients will use port 443 by default, whereas servers default to
+port 8448. Where these are different, we refer to the 'client port' and the
+'federation port'.
+
+The next most important thing to know is that using a reverse-proxy on the
+federation port has a number of pitfalls. It is possible, but be sure to read
+`Reverse-proxying the federation port`_.
+
+The recommended setup is therefore to configure your reverse-proxy on port 443
+to port 8008 of synapse for client connections, but to also directly expose port
+8448 for server-server connections. All the Matrix endpoints begin ``/_matrix``,
+so an example nginx configuration might look like::
+
+  server {
+      listen 443 ssl;
+      listen [::]:443 ssl;
+      server_name matrix.example.com;
+
+      location /_matrix {
+          proxy_pass http://localhost:8008;
+          proxy_set_header X-Forwarded-For $remote_addr;
+      }
+  }
+
+and an example apache configuration may look like::
+
+    <VirtualHost *:443>
+        SSLEngine on
+        ServerName matrix.example.com;
+
+        <Location /_matrix>
+            ProxyPass http://127.0.0.1:8008/_matrix nocanon
+            ProxyPassReverse http://127.0.0.1:8008/_matrix
+        </Location>
+    </VirtualHost>
+
+You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true``
+for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are
+recorded correctly.
+
+Having done so, you can then use ``https://matrix.example.com`` (instead of
+``https://matrix.example.com:8448``) as the "Custom server" when `Connecting to
+Synapse from a client`_.
+
+Reverse-proxying the federation port
+------------------------------------
+
+There are two issues to consider before using a reverse-proxy on the federation
+port:
+
+* Due to the way SSL certificates are managed in the Matrix federation protocol
+  (see `spec`__), Synapse needs to be configured with the path to the SSL
+  certificate, *even if you do not terminate SSL at Synapse*.
+
+  .. __: `key_management`_
+
+* Synapse does not currently support SNI on the federation protocol
+  (`bug #1491 <https://github.com/matrix-org/synapse/issues/1491>`_), which
+  means that using name-based virtual hosting is unreliable.
+
+Furthermore, a number of the normal reasons for using a reverse-proxy do not
+apply:
+
+* Other servers will connect on port 8448 by default, so there is no need to
+  listen on port 443 (for federation, at least), which avoids the need for root
+  privileges and virtual hosting.
+
+* A self-signed SSL certificate is fine for federation, so there is no need to
+  automate renewals. (The certificate generated by ``--generate-config`` is
+  valid for 10 years.)
+
+If you want to set up a reverse-proxy on the federation port despite these
+caveats, you will need to do the following:
+
+* In ``homeserver.yaml``, set ``tls_certificate_path`` to the path to the SSL
+  certificate file used by your reverse-proxy, and set ``no_tls`` to ``True``.
+  (``tls_private_key_path`` will be ignored if ``no_tls`` is ``True``.)
+
+* In your reverse-proxy configuration:
+
+  * If there are other virtual hosts on the same port, make sure that the
+    *default* one uses the certificate configured above.
+
+  * Forward ``/_matrix`` to Synapse.
+
+* If your reverse-proxy is not listening on port 8448, publish a SRV record to
+  tell other servers how to find you. See `Setting up Federation`_.
+
+When updating the SSL certificate, just update the file pointed to by
+``tls_certificate_path``: there is no need to restart synapse. (You may like to
+use a symbolic link to help make this process atomic.)
+
+The most common mistake when setting up federation is not to tell Synapse about
+your SSL certificate. To check it, you can visit
+``https://matrix.org/federationtester/api/report?server_name=<your_server_name>``.
+Unfortunately, there is no UI for this yet, but, you should see
+``"MatchingTLSFingerprint": true``. If not, check that
+``Certificates[0].SHA256Fingerprint`` (the fingerprint of the certificate
+presented by your reverse-proxy) matches ``Keys.tls_fingerprints[0].sha256``
+(the fingerprint of the certificate Synapse is using).
+
 
 Identity Servers
 ================
@@ -233,17 +826,36 @@ an email address with your account, or send an invite to another user via their
 email address.
 
 
+URL Previews
+============
+
+Synapse 0.15.0 introduces a new API for previewing URLs at
+``/_matrix/media/r0/preview_url``.  This is disabled by default.  To turn it on
+you must enable the ``url_preview_enabled: True`` config parameter and
+explicitly specify the IP ranges that Synapse is not allowed to spider for
+previewing in the ``url_preview_ip_range_blacklist`` configuration parameter.
+This is critical from a security perspective to stop arbitrary Matrix users
+spidering 'internal' URLs on your network.  At the very least we recommend that
+your loopback and RFC1918 IP addresses are blacklisted.
+
+This also requires the optional lxml and netaddr python dependencies to be
+installed.  This in turn requires the libxml2 library to be available - on
+Debian/Ubuntu this means ``apt-get install libxml2-dev``, or equivalent for
+your OS.
+
+
 Password reset
 ==============
 
 If a user has registered an email address to their account using an identity
-server, they can request a password-reset token via clients such as Riot.
+server, they can request a password-reset token via clients such as Vector.
 
 A manual password reset can be done via direct database access as follows.
 
 First calculate the hash of the new password::
 
-    $ ~/synapse/env/bin/hash_password
+    $ source ~/.synapse/bin/activate
+    $ ./scripts/hash_password
     Password:
     Confirm password:
     $2a$12$xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
@@ -259,7 +871,7 @@ Synapse Development
 
 Before setting up a development environment for synapse, make sure you have the
 system dependencies (such as the python header files) installed - see
-`Installing from source <INSTALL.md#installing-from-source>`_.
+`Installing from source`_.
 
 To check out a synapse for development, clone the git repo into a working
 directory of your choice::
@@ -270,9 +882,10 @@ directory of your choice::
 Synapse has a number of external dependencies, that are easiest
 to install using pip and a virtualenv::
 
-    virtualenv -p python3 env
+    virtualenv -p python2.7 env
     source env/bin/activate
-    python -m pip install --no-pep-517 -e .[all]
+    python -m synapse.python_dependencies | xargs pip install
+    pip install lxml mock
 
 This will run a process of downloading and installing all the needed
 dependencies into a virtual env.
@@ -280,7 +893,7 @@ dependencies into a virtual env.
 Once this is done, you may wish to run Synapse's unit tests, to
 check that everything is installed as it should be::
 
-    python -m twisted.trial tests
+    PYTHONPATH="." trial tests
 
 This should end with a 'PASSED' result::
 
@@ -313,42 +926,16 @@ Building internal API documentation::
 
     python setup.py build_sphinx
 
-Troubleshooting
-===============
-
-Running out of File Handles
----------------------------
-
-If synapse runs out of file handles, it typically fails badly - live-locking
-at 100% CPU, and/or failing to accept new TCP connections (blocking the
-connecting client).  Matrix currently can legitimately use a lot of file handles,
-thanks to busy rooms like #matrix:matrix.org containing hundreds of participating
-servers.  The first time a server talks in a room it will try to connect
-simultaneously to all participating servers, which could exhaust the available
-file descriptors between DNS queries & HTTPS sockets, especially if DNS is slow
-to respond. (We need to improve the routing algorithm used to be better than
-full mesh, but as of March 2019 this hasn't happened yet).
-
-If you hit this failure mode, we recommend increasing the maximum number of
-open file handles to be at least 4096 (assuming a default of 1024 or 256).
-This is typically done by editing ``/etc/security/limits.conf``
-
-Separately, Synapse may leak file handles if inbound HTTP requests get stuck
-during processing - e.g. blocked behind a lock or talking to a remote server etc.
-This is best diagnosed by matching up the 'Received request' and 'Processed request'
-log lines and looking for any 'Processed request' lines which take more than
-a few seconds to execute. Please let us know at #synapse:matrix.org if
-you see this failure mode so we can help debug it, however.
 
 Help!! Synapse eats all my RAM!
--------------------------------
+===============================
 
 Synapse's architecture is quite RAM hungry currently - we deliberately
 cache a lot of recent room data and metadata in RAM in order to speed up
-common requests. We'll improve this in the future, but for now the easiest
+common requests.  We'll improve this in future, but for now the easiest
 way to either reduce the RAM usage (at the risk of slowing things down)
 is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment
-variable. The default is 0.5, which can be decreased to reduce RAM usage
+variable.  The default is 0.5, which can be decreased to reduce RAM usage
 in memory constrained enviroments, or increased if performance starts to
 degrade.
 
@@ -361,5 +948,4 @@ by installing the ``libjemalloc1`` package and adding this line to
 
     LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
 
-This can make a significant difference on Python 2.7 - it's unclear how
-much of an improvement it provides on Python 3.x.
+.. _`key_management`: https://matrix.org/docs/spec/server_server/unstable.html#retrieving-server-keys

UPGRADE.rst

@@ -5,20 +5,20 @@ Before upgrading check if any special steps are required to upgrade from the
 what you currently have installed to current version of synapse. The extra
 instructions that may be required are listed later in this document.
 
-1. If synapse was installed in a virtualenv then activate that virtualenv before
-   upgrading. If synapse is installed in a virtualenv in ``~/synapse/env`` then
+1. If synapse was installed in a virtualenv then active that virtualenv before
+   upgrading. If synapse is installed in a virtualenv in ``~/.synapse/`` then
    run:
 
    .. code:: bash
 
-       source ~/synapse/env/bin/activate
+       source ~/.synapse/bin/activate
 
 2. If synapse was installed using pip then upgrade to the latest version by
    running:
 
    .. code:: bash
 
-       pip install --upgrade matrix-synapse[all]
+       pip install --upgrade --process-dependency-links https://github.com/matrix-org/synapse/tarball/master
 
        # restart synapse
        synctl restart
@@ -31,15 +31,14 @@ instructions that may be required are listed later in this document.
 
        # Pull the latest version of the master branch.
        git pull
-
-       # Update synapse and its python dependencies.
-       pip install --upgrade .[all]
+       # Update the versions of synapse's python dependencies.
+       python synapse/python_dependencies.py | xargs pip install --upgrade
 
        # restart synapse
        ./synctl restart
 
 
-To check whether your update was successful, you can check the Server header
+To check whether your update was sucessful, you can check the Server header
 returned by the Client-Server API:
 
 .. code:: bash
@@ -49,163 +48,11 @@ returned by the Client-Server API:
     # configured on port 443.
     curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"
 
-Upgrading to v1.0
-=================
-
-Validation of TLS certificates
-------------------------------
-
-Synapse v1.0 is the first release to enforce
-validation of TLS certificates for the federation API. It is therefore
-essential that your certificates are correctly configured. See the `FAQ
-<docs/MSC1711_certificates_FAQ.md>`_ for more information.
-
-Note, v1.0 installations will also no longer be able to federate with servers
-that have not correctly configured their certificates.
-
-In rare cases, it may be desirable to disable certificate checking: for
-example, it might be essential to be able to federate with a given legacy
-server in a closed federation. This can be done in one of two ways:-
-
-* Configure the global switch ``federation_verify_certificates`` to ``false``.
-* Configure a whitelist of server domains to trust via ``federation_certificate_verification_whitelist``.
-
-See the `sample configuration file <docs/sample_config.yaml>`_
-for more details on these settings. 
-
-Email
------
-When a user requests a password reset, Synapse will send an email to the 
-user to confirm the request.
-
-Previous versions of Synapse delegated the job of sending this email to an
-identity server. If the identity server was somehow malicious or became
-compromised, it would be theoretically possible to hijack an account through
-this means.
-
-Therefore, by default, Synapse v1.0 will send the confirmation email itself. If
-Synapse is not configured with an SMTP server, password reset via email will be
-disabled.
-
-To configure an SMTP server for Synapse, modify the configuration section
-headed ``email``, and be sure to have at least the ``smtp_host``, ``smtp_port``
-and ``notif_from`` fields filled out. You may also need to set ``smtp_user``,
-``smtp_pass``, and ``require_transport_security``.
-
-If you are absolutely certain that you wish to continue using an identity
-server for password resets, set ``trust_identity_server_for_password_resets`` to ``true``.
-
-See the `sample configuration file <docs/sample_config.yaml>`_
-for more details on these settings.
-
-Upgrading to v0.99.0
-====================
-
-Please be aware that, before Synapse v1.0 is released around March 2019, you
-will need to replace any self-signed certificates with those verified by a
-root CA. Information on how to do so can be found at `the ACME docs
-<docs/ACME.md>`_.
-
-For more information on configuring TLS certificates see the `FAQ <docs/MSC1711_certificates_FAQ.md>`_.
-
-Upgrading to v0.34.0
-====================
-
-1. This release is the first to fully support Python 3. Synapse will now run on
-   Python versions 3.5, or 3.6 (as well as 2.7). We recommend switching to
-   Python 3, as it has been shown to give performance improvements.
-
-   For users who have installed Synapse into a virtualenv, we recommend doing
-   this by creating a new virtualenv. For example::
-
-       virtualenv -p python3 ~/synapse/env3
-       source ~/synapse/env3/bin/activate
-       pip install matrix-synapse
-
-   You can then start synapse as normal, having activated the new virtualenv::
-
-       cd ~/synapse
-       source env3/bin/activate
-       synctl start
-
-   Users who have installed from distribution packages should see the relevant
-   package documentation. See below for notes on Debian packages.
-
-   * When upgrading to Python 3, you **must** make sure that your log files are
-     configured as UTF-8, by adding ``encoding: utf8`` to the
-     ``RotatingFileHandler`` configuration (if you have one) in your
-     ``<server>.log.config`` file. For example, if your ``log.config`` file
-     contains::
-
-       handlers:
-         file:
-           class: logging.handlers.RotatingFileHandler
-           formatter: precise
-           filename: homeserver.log
-           maxBytes: 104857600
-           backupCount: 10
-           filters: [context]
-         console:
-           class: logging.StreamHandler
-           formatter: precise
-           filters: [context]
-
-     Then you should update this to be::
-
-       handlers:
-         file:
-           class: logging.handlers.RotatingFileHandler
-           formatter: precise
-           filename: homeserver.log
-           maxBytes: 104857600
-           backupCount: 10
-           filters: [context]
-           encoding: utf8
-         console:
-           class: logging.StreamHandler
-           formatter: precise
-           filters: [context]
-
-     There is no need to revert this change if downgrading to Python 2.
-
-   We are also making available Debian packages which will run Synapse on
-   Python 3. You can switch to these packages with ``apt-get install
-   matrix-synapse-py3``, however, please read `debian/NEWS
-   <https://github.com/matrix-org/synapse/blob/release-v0.34.0/debian/NEWS>`_
-   before doing so. The existing ``matrix-synapse`` packages will continue to
-   use Python 2 for the time being.
-
-2. This release removes the ``riot.im`` from the default list of trusted
-   identity servers.
-
-   If ``riot.im`` is in your homeserver's list of
-   ``trusted_third_party_id_servers``, you should remove it. It was added in
-   case a hypothetical future identity server was put there. If you don't
-   remove it, users may be unable to deactivate their accounts.
-
-3. This release no longer installs the (unmaintained) Matrix Console web client
-   as part of the default installation. It is possible to re-enable it by
-   installing it separately and setting the ``web_client_location`` config
-   option, but please consider switching to another client.
-
-Upgrading to v0.33.7
-====================
-
-This release removes the example email notification templates from
-``res/templates`` (they are now internal to the python package). This should
-only affect you if you (a) deploy your Synapse instance from a git checkout or
-a github snapshot URL, and (b) have email notifications enabled.
-
-If you have email notifications enabled, you should ensure that
-``email.template_dir`` is either configured to point at a directory where you
-have installed customised templates, or leave it unset to use the default
-templates.
-
-Upgrading to v0.27.3
+Upgrading to $NEXT_VERSION
 ====================
 
 This release expands the anonymous usage stats sent if the opt-in
-``report_stats`` configuration is set to ``true``. We now capture RSS memory
+``report_stats`` configuration is set to ``true``. We now capture RSS memory 
 and cpu use at a very coarse level. This requires administrators to install
 the optional ``psutil`` python module.
 

changelog.d/3578.bugfix

@@ -0,0 +1 @@
+Fix problem when playing media from Chrome using direct URL (thanks @remjey!)

changelog.d/3699.misc

@@ -0,0 +1,2 @@
+Unit tests can now be run under PostgreSQL in Docker using 
+``test_postgresql.sh``.

changelog.d/3794.misc

@@ -0,0 +1 @@
+Speed up calculation of typing updates for replication

changelog.d/3836.bugfix

@@ -0,0 +1 @@
+support registering regular users non-interactively with register_new_matrix_user script

changelog.d/3868.bugfix

@@ -0,0 +1 @@
+Fix broken invite email links for self hosted riots 

changelog.d/3873.misc

@@ -0,0 +1,2 @@
+Remove documentation regarding installation on Cygwin, the use of WSL is 
+recommended instead.

changelog.d/3879.bugfix

@@ -0,0 +1 @@
+Don't ratelimit autojoins

changelog.d/3883.feature

@@ -0,0 +1 @@
+Adding the ability to change MAX_UPLOAD_SIZE for the docker container variables.

changelog.d/3889.bugfix

@@ -0,0 +1 @@
+Fix 500 error when deleting unknown room alias

changelog.d/3892.bugfix

@@ -0,0 +1 @@
+Fix some b'abcd' noise in logs and metrics

changelog.d/3894.feature

@@ -0,0 +1 @@
+Report "python_version" in the phone home stats

changelog.d/3895.bugfix

@@ -0,0 +1 @@
+Fix some b'abcd' noise in logs and metrics

changelog.d/3897.misc

@@ -0,0 +1 @@
+Fix typo in README, synaspse -> synapse

changelog.d/3899.bugfix

@@ -0,0 +1 @@
+When we join a room, always try the server we used for the alias lookup first, to avoid unresponsive and out-of-date servers.

changelog.d/3903.misc

@@ -0,0 +1 @@
+Increase the timeout when filling missing events in federation requests

changelog.d/3904.misc

@@ -0,0 +1 @@
+Improve the logging when handling a federation transaction

changelog.d/3906.misc

@@ -0,0 +1 @@
+Improve logging of outbound federation requests

changelog.d/3907.bugfix

@@ -0,0 +1 @@
+Fix incorrect server-name indication for outgoing federation requests

changelog.d/3908.bugfix

@@ -0,0 +1 @@
+Fix adding client IPs to the database failing on Python 3.

changelog.d/3909.misc

@@ -0,0 +1 @@
+Improve logging of outbound federation requests

changelog.d/3910.bugfix

@@ -0,0 +1 @@
+Fix bug where things occaisonally were not being timed out correctly.

changelog.d/3911.misc

@@ -0,0 +1 @@
+Fix the docker image building on python 3

changelog.d/3912.misc

@@ -0,0 +1 @@
+Add a regression test for logging failed HTTP requests on Python 3.

changelog.d/3914.bugfix

@@ -0,0 +1 @@
+Fix bug where outbound federation would stop talking to some servers when using workers

changelog.d/3916.feature

@@ -0,0 +1 @@
+Always LL ourselves if we're in a room

changelog.d/3924.misc

@@ -0,0 +1 @@
+Comments and interface cleanup for on_receive_pdu

changelog.d/3925.misc

@@ -0,0 +1 @@
+Fix spurious exceptions when remote http client closes conncetion

changelog.d/3927.misc

@@ -0,0 +1 @@
+Log exceptions thrown by background tasks

changelog.d/3932.bugfix

@@ -0,0 +1 @@
+Fix some instances of ExpiringCache not expiring cache items

changelog.d/3936.bugfix

@@ -0,0 +1 @@
+Fix out-of-bounds error when LLing yourself

changelog.d/3946.misc

@@ -0,0 +1 @@
+Automate pushes to docker hub

changelog.d/3947.misc

@@ -0,0 +1 @@
+Require attrs 16.0.0 or later

changelog.d/3948.misc

@@ -0,0 +1 @@
+Fix incompatibility with python3 on alpine

changelog.d/3952.misc

@@ -0,0 +1 @@
+Run the test suite on the oldest supported versions of our dependencies in CI.

changelog.d/3956.bugfix

@@ -0,0 +1 @@
+Fix exceptions from metrics handler

changelog.d/3957.misc

@@ -0,0 +1 @@
+CircleCI now only runs merged jobs on PRs, and commit jobs on develop, master, and release branches.

changelog.d/3958.misc

@@ -0,0 +1 @@
+Fix docstrings and add tests for state store methods

changelog.d/3959.feature

@@ -0,0 +1 @@
+Include eventid in log lines when processing incoming federation transactions

changelog.d/3961.bugfix

@@ -0,0 +1 @@
+Fix errors due to concurrent monthly_active_user upserts 

changelog.d/3963.misc

@@ -0,0 +1 @@
+fix docstring for FederationClient.get_state_for_room

changelog.d/3965.misc

@@ -0,0 +1 @@
+Run notify_app_services as a bg process

changelog.d/3966.misc

@@ -0,0 +1 @@
+Improve the logging when handling a federation transaction

changelog.d/3967.misc

@@ -0,0 +1 @@
+Clarifications in FederationHandler

changelog.d/3970.bugfix

@@ -0,0 +1 @@
+Replaced all occurences of e.message with str(e). Contributed by Schnuffle

changelog.d/5083.feature

@@ -1 +0,0 @@
-Adds auth_profile_reqs option to require access_token to GET /profile endpoints on CS API.

changelog.d/5098.misc

@@ -1 +0,0 @@
-Add workarounds for pep-517 install errors.

changelog.d/5214.feature

@@ -1 +0,0 @@
-Allow server admins to define and enforce a password policy (MSC2000).

changelog.d/5363.feature

@@ -1 +0,0 @@
-Allow expired user to trigger renewal email sending manually.

changelog.d/5378.misc

@@ -1 +0,0 @@
-Track deactivated accounts in the database.

changelog.d/5394.bugfix

@@ -1 +0,0 @@
-Fix a bug where deactivated users could receive renewal emails if the account validity feature is on.

changelog.d/5416.misc

@@ -1 +0,0 @@
-Add unique index to the profile_replication_status table.

changelog.d/5420.feature

@@ -1 +0,0 @@
-Add configuration option to hide new users from the user directory.

changelog.d/5440.feature

@@ -1 +0,0 @@
-Allow server admins to define implementations of extra rules for allowing or denying incoming events.

changelog.d/5464.bugfix

@@ -1 +0,0 @@
-Fix missing invite state after exchanging 3PID invites over federaton.

changelog.d/5465.misc

@@ -1,2 +0,0 @@
-Track deactivated accounts in the database.
-

changelog.d/5474.feature

@@ -1 +0,0 @@
-Allow server admins to define implementations of extra rules for allowing or denying incoming events.

changelog.d/5477.feature

@@ -1 +0,0 @@
-Allow server admins to define implementations of extra rules for allowing or denying incoming events.

changelog.d/5493.misc

@@ -1 +0,0 @@
-Track deactivated accounts in the database.

changelog.d/5534.feature

@@ -1 +0,0 @@
-Split public rooms directory auth config in two settings, in order to manage client auth independently from the federation part of it. Obsoletes the "restrict_public_rooms_to_local_users" configuration setting. If "restrict_public_rooms_to_local_users" is set in the config, Synapse will act as if both new options are enabled, i.e. require authentication through the client API and deny federation requests.

changelog.d/5576.bugfix

@@ -1 +0,0 @@
-Fix a bug that would cause invited users to receive several emails for a single 3PID invite in case the inviter is rate limited.

changelog.d/5610.feature

@@ -1 +0,0 @@
-Implement new custom event rules for power levels.

changelog.d/5644.bugfix

@@ -1 +0,0 @@
-Fix newly-registered users not being able to lookup their own profile without joining a room.

changelog.d/5702.bugfix

@@ -1 +0,0 @@
-Fix 3PID invite to invite association detection in the Tchap room access rules.

changelog.d/5760.feature

@@ -1 +0,0 @@
-Force the access rule to be "restricted" if the join rule is "public".

changelog.d/5780.misc

@@ -1 +0,0 @@
-Allow looping calls to be given arguments.

changelog.d/5807.feature

@@ -1 +0,0 @@
-Allow defining HTML templates to serve the user on account renewal attempt when using the account validity feature.

changelog.d/5815.feature

@@ -1 +0,0 @@
-Implement per-room message retention policies.

contrib/docker/docker-compose.yml

@@ -6,11 +6,9 @@ version: '3'
 services:
 
   synapse:
-    build:
-        context: ../..
-        dockerfile: docker/Dockerfile
+    build: ../..
     image: docker.io/matrixdotorg/synapse:latest
-    # Since synapse does not retry to connect to the database, restart upon
+    # Since snyapse does not retry to connect to the database, restart upon
     # failure
     restart: unless-stopped
     # See the readme for a full documentation of the environment settings
@@ -37,7 +35,7 @@ services:
     labels:
       - traefik.enable=true
       - traefik.frontend.rule=Host:my.matrix.Host
-      - traefik.port=8008
+      - traefik.port=8448
 
   db:
     image: docker.io/postgres:10-alpine
@@ -49,4 +47,4 @@ services:
       # You may store the database tables in a local folder..
       - ./schemas:/var/lib/postgresql/data
       # .. or store them on some high performance storage for better results
-      # - /path/to/ssd/storage:/var/lib/postgresql/data
+      # - /path/to/ssd/storage:/var/lib/postfesql/data

contrib/example_log_config.yaml

@@ -19,7 +19,6 @@ handlers:
   # example output to console
   console:
     class: logging.StreamHandler
-    formatter: fmt
     filters: [context]
 
   # example output to file - to enable, edit 'root' config below.
@@ -30,7 +29,7 @@ handlers:
     maxBytes: 100000000
     backupCount: 3
     filters: [context]
-    encoding: utf8
+
 
 root:
     level: INFO

contrib/prometheus/README

@@ -6,10 +6,8 @@ To use it, first install prometheus by following the instructions at
   http://prometheus.io/
 
 ### for Prometheus v1
-
 Add a new job to the main prometheus.conf file:
 
-```yaml
   job: {
     name: "synapse"
 
@@ -17,12 +15,10 @@ Add a new job to the main prometheus.conf file:
       target: "http://SERVER.LOCATION.HERE:PORT/_synapse/metrics"
     }
   }
-```
 
 ### for Prometheus v2
 Add a new job to the main prometheus.yml file:
 
-```yaml
   - job_name: "synapse"
     metrics_path: "/_synapse/metrics"
     # when endpoint uses https:
@@ -30,14 +26,11 @@ Add a new job to the main prometheus.yml file:
 
     static_configs:
     - targets: ['SERVER.LOCATION:PORT']
-```
 
 To use `synapse.rules` add
 
-```yaml
     rule_files:
       - "/PATH/TO/synapse-v2.rules"
-```
 
 Metrics are disabled by default when running synapse; they must be enabled
 with the 'enable-metrics' option, either in the synapse config file or as a