Require SQLite3 version 3.15 or above

This is primarily to allow tuple comparisons in queries, though a better query optimiser and other improvements mean that using newer versions of sqlite is highly recommended anyway.
2018-05-08 13:48:45 +01:00
576 changed files with 17514 additions and 42199 deletions
@@ -1,168 +0,0 @@
-version: 2
-jobs:
-  dockerhubuploadrelease:
-    machine: true
-    steps:
-      - checkout
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} .
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 .
-      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
-      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
-  dockerhubuploadlatest:
-    machine: true
-    steps:
-      - checkout
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_SHA1} .
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_SHA1}-py3 --build-arg PYTHON_VERSION=3.6 .
-      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      - run: docker push matrixdotorg/synapse:latest
-      - run: docker push matrixdotorg/synapse:latest-py3
-  sytestpy2:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy2
-    working_directory: /src
-    steps:
-      - checkout
-      - run: /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy2postgres:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy2
-    working_directory: /src
-    steps:
-      - checkout
-      - run: POSTGRES=1 /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy2merged:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy2
-    working_directory: /src
-    steps:
-      - checkout
-      - run: bash .circleci/merge_base_branch.sh
-      - run: /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy2postgresmerged:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy2
-    working_directory: /src
-    steps:
-      - checkout
-      - run: bash .circleci/merge_base_branch.sh
-      - run: POSTGRES=1 /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-
-  sytestpy3:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy3
-    working_directory: /src
-    steps:
-      - checkout
-      - run: /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy3postgres:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy3
-    working_directory: /src
-    steps:
-      - checkout
-      - run: POSTGRES=1 /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy3merged:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy3
-    working_directory: /src
-    steps:
-      - checkout
-      - run: bash .circleci/merge_base_branch.sh
-      - run: /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy3postgresmerged:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy3
-    working_directory: /src
-    steps:
-      - checkout
-      - run: bash .circleci/merge_base_branch.sh
-      - run: POSTGRES=1 /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-
-workflows:
-  version: 2
-  build:
-    jobs:
-      - sytestpy2:
-          filters:
-            branches:
-              only: /develop|master|release-.*/
-      - sytestpy2postgres:
-          filters:
-            branches:
-              only: /develop|master|release-.*/
-      - sytestpy3:
-          filters:
-            branches:
-              only: /develop|master|release-.*/
-      - sytestpy3postgres:
-          filters:
-            branches:
-              only: /develop|master|release-.*/
-      - sytestpy2merged:
-          filters:
-            branches:
-              ignore: /develop|master|release-.*/
-      - sytestpy2postgresmerged:
-          filters:
-            branches:
-              ignore: /develop|master|release-.*/
-      - sytestpy3merged:
-          filters:
-            branches:
-              ignore: /develop|master|release-.*/
-      - sytestpy3postgresmerged:
-          filters:
-            branches:
-              ignore: /develop|master|release-.*/
-      - dockerhubuploadrelease:
-          filters:
-            tags:
-              only: /v[0-9].[0-9]+.[0-9]+.*/
-            branches:
-              ignore: /.*/
-      - dockerhubuploadlatest:
-          filters:
-            branches:
-              only: master
@@ -1,34 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-# CircleCI doesn't give CIRCLE_PR_NUMBER in the environment for non-forked PRs. Wonderful.
-# In this case, we just need to do some ~shell magic~ to strip it out of the PULL_REQUEST URL.
-echo 'export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}"' >> $BASH_ENV
-source $BASH_ENV
-
-if [[ -z "${CIRCLE_PR_NUMBER}" ]]
-then
-    echo "Can't figure out what the PR number is! Assuming merge target is develop."
-
-    # It probably hasn't had a PR opened yet. Since all PRs land on develop, we
-    # can probably assume it's based on it and will be merged into it.
-    GITBASE="develop"
-else
-    # Get the reference, using the GitHub API
-    GITBASE=`wget -O- https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
-fi
-
-# Show what we are before
-git show -s
-
-# Set up username so it can do a merge
-git config --global user.email bot@matrix.org
-git config --global user.name "A robot"
-
-# Fetch and merge. If it doesn't work, it will raise due to set -e.
-git fetch -u origin $GITBASE
-git merge --no-edit origin/$GITBASE
-
-# Show what we are after.
-git show -s
@@ -1,12 +0,0 @@
-[run]
-branch = True
-parallel = True
-source = synapse
-
-[paths]
-source=
-   coverage
-
-[report]
-precision = 2
-ignore_errors = True
@@ -1,7 +0,0 @@
-Dockerfile
-.travis.yml
-.gitignore
-demo/etc
-tox.ini
-.git/*
-.tox/*
@@ -1,9 +0,0 @@
-# EditorConfig https://EditorConfig.org
-
-# top-most EditorConfig file
-root = true
-
-# 4 space indentation
-[*.py]
-indent_style = space
-indent_size = 4
@@ -1,9 +1,3 @@
---
-name: Bug report
-about: Create a report to help us improve
-
---
-
 <!-- 

 **IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**: 
@@ -17,50 +11,37 @@ the necessary data to fix your issue.
 You can also preview your report before submitting it. You may remove sections
 that aren't relevant to your particular case.

-Text between <!-- and --> marks will be invisible in the report.
+Text between <!-- and --> marks will be invisible in the report.

 -->

 ### Description

-<!-- Describe here the problem that you are experiencing -->
+Describe here the problem that you are experiencing, or the feature you are requesting.

 ### Steps to reproduce

- list the steps
+- For bugs, list the steps
 - that reproduce the bug
 - using hyphens as bullet points

-<!-- 
 Describe how what happens differs from what you expected.

 If you can identify any relevant log snippets from _homeserver.log_, please include
-those (please be careful to remove any personal or private data). Please surround them with
-``` (three backticks, on a line on their own), so that they are formatted legibly.
-->
+those here (please be careful to remove any personal or private data):

 ### Version information

 <!-- IMPORTANT: please answer the following questions, to help us narrow down the problem -->

-<!-- Was this issue identified on matrix.org or another homeserver? -->
- **Homeserver**: 
+- **Homeserver**: Was this issue identified on matrix.org or another homeserver?

 If not matrix.org:
-
-<!-- 
-What version of Synapse is running? 
+- **Version**:        What version of Synapse is running? <!-- 
 You can find the Synapse version by inspecting the server headers (replace matrix.org with
 your own homeserver domain):
 $ curl -v https://matrix.org/_matrix/client/versions 2>&1 | grep "Server:"
 -->
- **Version**: 
-
- **Install method**: 
-<!-- examples: package manager/git clone/pip  -->
-
- **Platform**: 
-<!--
-Tell us about the environment in which your homeserver is operating
-distro, hardware, if it's running in a vm/container, etc.
-->
+- **Install method**: package manager/git clone/pip      
+- **Platform**:       Tell us about the environment in which your homeserver is operating
+                      - distro, hardware, if it's running in a vm/container, etc.
@@ -1,9 +0,0 @@
---
-name: Feature request
-about: Suggest an idea for this project
-
---
-
-**Description:**
-
-<!-- Describe here the feature you are requesting. -->
@@ -1,9 +0,0 @@
---
-name: Support request
-about: I need support for Synapse
-
---
-
-# Please ask for support in [**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org)
-
-## Don't file an issue as a support request.
@@ -1,7 +0,0 @@
-### Pull Request Checklist
-
-<!-- Please read CONTRIBUTING.rst before submitting your pull request -->
-
-* [ ] Pull request is based on the develop branch
-* [ ] Pull request includes a [changelog file](CONTRIBUTING.rst#changelog)
-* [ ] Pull request includes a [sign off](CONTRIBUTING.rst#sign-off)
@@ -1,3 +0,0 @@
-[**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org) is the official support room for Matrix, and can be accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html 
-
-It can also be access via IRC bridge at irc://irc.freenode.net/matrix or on the web here: https://webchat.freenode.net/?channels=matrix
@@ -1,11 +1,8 @@
 *.pyc
 .*.swp
-*~
-*.lock

 .DS_Store
 _trial_temp/
-_trial_temp*/
 logs/
 dbs/
 *.egg
@@ -16,7 +13,6 @@ docs/build/
 cmdclient_config.json
 homeserver*.db
 homeserver*.log
-homeserver*.log.*
 homeserver*.pid
 homeserver*.yaml

@@ -36,7 +32,6 @@ demo/media_store.*
 demo/etc

 uploads
-cache

 .idea/
 media_store/
@@ -44,9 +39,6 @@ media_store/
 *.tac

 build/
-venv/
-venv*/
-*venv/

 localhost-800*/
 static/client/register/register_config.js
@@ -56,4 +48,3 @@ env/
 *.config

 .vscode/
-.ropeproject/
@@ -1,70 +1,22 @@
 sudo: false
 language: python

-cache:
-  directories:
-    # we only bother to cache the wheels; parts of the http cache get
-    # invalidated every build (because they get served with a max-age of 600
-    # seconds), which means that we end up re-uploading the whole cache for
-    # every build, which is time-consuming In any case, it's not obvious that
-    # downloading the cache from S3 would be much faster than downloading the
-    # originals from pypi.
-    #
-    - $HOME/.cache/pip/wheels
+# tell travis to cache ~/.cache/pip
+cache: pip

-# don't clone the whole repo history, one commit will do
-git:
-  depth: 1
-
-# only build branches we care about (PRs are built seperately)
-branches:
-  only:
-    - master
-    - develop
-    - /^release-v/
-
-# When running the tox environments that call Twisted Trial, we can pass the -j
-# flag to run the tests concurrently. We set this to 2 for CPU bound tests
-# (SQLite) and 4 for I/O bound tests (PostgreSQL).
 matrix:
-  fast_finish: true
  include:
  - python: 2.7
    env: TOX_ENV=packaging

-  - python: 3.6
-    env: TOX_ENV="pep8,check_isort"
+  - python: 2.7
+    env: TOX_ENV=pep8

  - python: 2.7
-    env: TOX_ENV=py27,codecov TRIAL_FLAGS="-j 2"
-
-  - python: 2.7
-    env: TOX_ENV=py27-old TRIAL_FLAGS="-j 2"
-
-  - python: 2.7
-    env: TOX_ENV=py27-postgres,codecov TRIAL_FLAGS="-j 4"
-    services:
-      - postgresql
-
-  - python: 3.5
-    env: TOX_ENV=py35,codecov TRIAL_FLAGS="-j 2"
-
+    env: TOX_ENV=py27
+    
  - python: 3.6
-    env: TOX_ENV=py36,codecov TRIAL_FLAGS="-j 2"
-
-  - python: 3.6
-    env: TOX_ENV=py36-postgres,codecov TRIAL_FLAGS="-j 4"
-    services:
-      - postgresql
-
-  - # we only need to check for the newsfragment if it's a PR build
-    if: type = pull_request
-    python: 3.6
-    env: TOX_ENV=check-newsfragment
-    script:
-      - git remote set-branches --add origin develop
-      - git fetch origin develop
-      - tox -e $TOX_ENV
+    env: TOX_ENV=py36

 install:
  - pip install tox
@@ -60,9 +60,3 @@ Niklas Riekenbrauck <nikriek at gmail dot.com>

 Christoph Witzany <christoph at web.crofting.com>
 * Add LDAP support for authentication
-
-Pierre Jaury <pierre at jaury.eu>
-* Docker packaging
-
-Serban Constantin <serban.constantin at gmail dot com>
- * Small bug fix
@@ -30,28 +30,12 @@ use github's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.

-We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Travis CI 
-<https://travis-ci.org/matrix-org/synapse>`_ for continuous integration. All
-pull requests to synapse get automatically tested by Travis and CircleCI.
-If your change breaks the build, this will be shown in GitHub, so please
-keep an eye on the pull request for feedback.
-
-To run unit tests in a local development environment, you can use:
-
- ``tox -e py27`` (requires tox to be installed by ``pip install tox``) for
-  SQLite-backed Synapse on Python 2.7.
- ``tox -e py35`` for SQLite-backed Synapse on Python 3.5.
- ``tox -e py36`` for SQLite-backed Synapse on Python 3.6.
- ``tox -e py27-postgres`` for PostgreSQL-backed Synapse on Python 2.7
-  (requires a running local PostgreSQL with access to create databases).
- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 2.7
-  (requires Docker). Entirely self-contained, recommended if you don't want to
-  set up PostgreSQL yourself.
-
-Docker images are available for running the integration tests (SyTest) locally,
-see the `documentation in the SyTest repo
-<https://github.com/matrix-org/sytest/blob/develop/docker/README.md>`_ for more
-information.
+We use `Jenkins <http://matrix.org/jenkins>`_ and 
+`Travis <https://travis-ci.org/matrix-org/synapse>`_ for continuous
+integration. All pull requests to synapse get automatically tested by Travis; 
+the Jenkins builds require an adminstrator to start them. If your change 
+breaks the build, this will be shown in github, so please keep an eye on the 
+pull request for feedback.

 Code style
 ~~~~~~~~~~
@@ -64,27 +48,6 @@ Please ensure your changes match the cosmetic style of the existing project,
 and **never** mix cosmetic and functional changes in the same commit, as it
 makes it horribly hard to review otherwise.

-Changelog
-~~~~~~~~~
-
-All changes, even minor ones, need a corresponding changelog / newsfragment
-entry. These are managed by Towncrier
-(https://github.com/hawkowl/towncrier).
-
-To create a changelog entry, make a new file in the ``changelog.d``
-file named in the format of ``PRnumber.type``. The type can be
-one of ``feature``, ``bugfix``, ``removal`` (also used for
-deprecations), or ``misc`` (for internal-only changes). The content of
-the file is your changelog entry, which can contain Markdown
-formatting. Adding credits to the changelog is encouraged, we value
-your contributions and would like to have you shouted out in the
-release notes!
-
-For example, a fix in PR #1234 would have its changelog entry in
-``changelog.d/1234.bugfix``, and contain content like "The security levels of
-Florbs are now validated when recieved over federation. Contributed by Jane
-Matrix".
-
 Attribution
 ~~~~~~~~~~~

@@ -93,8 +56,7 @@ AUTHORS.rst file for the project in question. Please feel free to include a
 change to AUTHORS.rst in your pull request to list yourself and a short
 description of the area(s) you've worked on. Also, we sometimes have swag to
 give away to contributors - if you feel that Matrix-branded apparel is missing
-from your life, please mail us your shipping address to matrix at matrix.org and
-we'll try to fix it :)
+from your life, please mail us your shipping address to matrix at matrix.org and we'll try to fix it :)

 Sign off
 ~~~~~~~~
@@ -143,27 +105,18 @@ the contribution or otherwise have the right to contribute it to Matrix::
        personal information I submit with it, including my sign-off) is
        maintained indefinitely and may be redistributed consistent with
        this project or the open source license(s) involved.
-
+        
 If you agree to this for your contribution, then all that's needed is to
 include the line in your commit or pull request comment::

    Signed-off-by: Your Name <your@email.example.org>
-
-We accept contributions under a legally identifiable name, such as
-your name on government documentation or common-law names (names
-claimed by legitimate usage or repute). Unfortunately, we cannot
-accept anonymous contributions at this time.
-
-Git allows you to add this signoff automatically when using the ``-s``
-flag to ``git commit``, which uses the name and email set in your
-``user.name`` and ``user.email`` git configs.
+    
+...using your real name; unfortunately pseudonyms and anonymous contributions
+can't be accepted. Git makes this trivial - just use the -s flag when you do
+``git commit``, having first set ``user.name`` and ``user.email`` git configs
+(which you should have done anyway :)

 Conclusion
 ~~~~~~~~~~

-That's it!  Matrix is a very open and collaborative project as you might expect
-given our obsession with open communication.  If we're going to successfully
-matrix together all the fragmented communication technologies out there we are
-reliant on contributions and collaboration from the community to do so.  So
-please get involved - and we hope you have as much fun hacking on Matrix as we
-do!
+That's it!  Matrix is a very open and collaborative project as you might expect given our obsession with open communication.  If we're going to successfully matrix together all the fragmented communication technologies out there we are reliant on contributions and collaboration from the community to do so.  So please get involved - and we hope you have as much fun hacking on Matrix as we do!
@@ -2,7 +2,6 @@ include synctl
 include LICENSE
 include VERSION
 include *.rst
-include *.md
 include demo/README
 include demo/demo.tls.dh
 include demo/*.py
@@ -12,30 +11,21 @@ recursive-include synapse/storage/schema *.sql
 recursive-include synapse/storage/schema *.py

 recursive-include docs *
+recursive-include res *
 recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
 recursive-include tests *.py

-recursive-include synapse/res *
 recursive-include synapse/static *.css
 recursive-include synapse/static *.gif
 recursive-include synapse/static *.html
 recursive-include synapse/static *.js

-exclude Dockerfile
-exclude .dockerignore
-exclude test_postgresql.sh
-exclude .editorconfig
-
-include pyproject.toml
-recursive-include changelog.d *
+exclude jenkins.sh
+exclude jenkins*.sh
+exclude jenkins*
+recursive-exclude jenkins *.sh

 prune .github
 prune demo/etc
-prune docker
-prune .circleci
-prune .coveragerc
-
-exclude jenkins*
-recursive-exclude jenkins *.sh
@@ -0,0 +1,35 @@
+Directory Structure
+===================
+
+Warning: this may be a bit stale...
+
+::
+
+    .
+    ├── cmdclient           Basic CLI python Matrix client
+    ├── demo                Scripts for running standalone Matrix demos
+    ├── docs                All doc, including the draft Matrix API spec
+    │   ├── client-server       The client-server Matrix API spec
+    │   ├── model               Domain-specific elements of the Matrix API spec
+    │   ├── server-server       The server-server model of the Matrix API spec
+    │   └── sphinx              The internal API doc of the Synapse homeserver
+    ├── experiments         Early experiments of using Synapse's internal APIs
+    ├── graph               Visualisation of Matrix's distributed message store 
+    ├── synapse             The reference Matrix homeserver implementation
+    │   ├── api                 Common building blocks for the APIs
+    │   │   ├── events              Definition of state representation Events 
+    │   │   └── streams             Definition of streamable Event objects
+    │   ├── app                 The __main__ entry point for the homeserver
+    │   ├── crypto              The PKI client/server used for secure federation
+    │   │   └── resource            PKI helper objects (e.g. keys)
+    │   ├── federation          Server-server state replication logic
+    │   ├── handlers            The main business logic of the homeserver
+    │   ├── http                Wrappers around Twisted's HTTP server & client
+    │   ├── rest                Servlet-style RESTful API
+    │   ├── storage             Persistence subsystem (currently only sqlite3)
+    │   │   └── schema              sqlite persistence schema
+    │   └── util                Synapse-specific utilities
+    ├── tests               Unit tests for the Synapse homeserver
+    └── webclient           Basic AngularJS Matrix web client
+
+
@@ -71,7 +71,7 @@ We'd like to invite you to join #matrix:matrix.org (via
 https://matrix.org/docs/projects/try-matrix-now.html), run a homeserver, take a look
 at the `Matrix spec <https://matrix.org/docs/spec>`_, and experiment with the
 `APIs <https://matrix.org/docs/api>`_ and `Client SDKs
-<https://matrix.org/docs/projects/try-matrix-now.html#client-sdks>`_.
+<http://matrix.org/docs/projects/try-matrix-now.html#client-sdks>`_.

 Thanks for using Matrix!

@@ -81,33 +81,32 @@ Thanks for using Matrix!
 Synapse Installation
 ====================

-Synapse is the reference Python/Twisted Matrix homeserver implementation.
+Synapse is the reference python/twisted Matrix homeserver implementation.

 System requirements:

 - POSIX-compliant system (tested on Linux & OS X)
- Python 3.5, 3.6, or 2.7
+- Python 2.7
 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org

 Installing from source
 ----------------------
-
 (Prebuilt packages are available for some platforms - see `Platform-Specific
 Instructions`_.)

-Synapse is written in Python but some of the libraries it uses are written in
-C. So before we can install Synapse itself we need a working C compiler and the
-header files for Python C extensions.
+Synapse is written in python but some of the libraries it uses are written in
+C. So before we can install synapse itself we need a working C compiler and the
+header files for python C extensions.

 Installing prerequisites on Ubuntu or Debian::

-    sudo apt-get install build-essential python3-dev libffi-dev \
+    sudo apt-get install build-essential python2.7-dev libffi-dev \
                         python-pip python-setuptools sqlite3 \
                         libssl-dev python-virtualenv libjpeg-dev libxslt1-dev

 Installing prerequisites on ArchLinux::

-    sudo pacman -S base-devel python python-pip \
+    sudo pacman -S base-devel python2 python-pip \
                   python-setuptools python-virtualenv sqlite3

 Installing prerequisites on CentOS 7 or Fedora 25::
@@ -126,9 +125,12 @@ Installing prerequisites on Mac OS X::

 Installing prerequisites on Raspbian::

-    sudo apt-get install build-essential python3-dev libffi-dev \
+    sudo apt-get install build-essential python2.7-dev libffi-dev \
                         python-pip python-setuptools sqlite3 \
                         libssl-dev python-virtualenv libjpeg-dev
+    sudo pip install --upgrade pip
+    sudo pip install --upgrade ndg-httpsclient
+    sudo pip install --upgrade virtualenv

 Installing prerequisites on openSUSE::

@@ -139,46 +141,30 @@ Installing prerequisites on openSUSE::
 Installing prerequisites on OpenBSD::

    doas pkg_add python libffi py-pip py-setuptools sqlite3 py-virtualenv \
-                 libxslt jpeg
+                 libxslt

-To install the Synapse homeserver run::
+To install the synapse homeserver run::

-    mkdir -p ~/synapse
-    virtualenv -p python3 ~/synapse/env
-    source ~/synapse/env/bin/activate
+    virtualenv -p python2.7 ~/.synapse
+    source ~/.synapse/bin/activate
    pip install --upgrade pip
    pip install --upgrade setuptools
-    pip install matrix-synapse
+    pip install https://github.com/matrix-org/synapse/tarball/master

-This installs Synapse, along with the libraries it uses, into a virtual
-environment under ``~/synapse/env``.  Feel free to pick a different directory
+This installs synapse, along with the libraries it uses, into a virtual
+environment under ``~/.synapse``.  Feel free to pick a different directory
 if you prefer.

-This Synapse installation can then be later upgraded by using pip again with the
-update flag::
-
-    source ~/synapse/env/bin/activate
-    pip install -U matrix-synapse
-
 In case of problems, please see the _`Troubleshooting` section below.

-There is an offical synapse image available at
-https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with
-the docker-compose file available at `contrib/docker <contrib/docker>`_. Further information on
-this including configuration options is available in the README on
-hub.docker.com.
+Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a Dockerfile to automate the
+above in Docker at https://hub.docker.com/r/avhost/docker-matrix/tags/

-Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
-Dockerfile to automate a synapse server in a single Docker image, at
-https://hub.docker.com/r/avhost/docker-matrix/tags/
+Also, Martin Giess has created an auto-deployment process with vagrant/ansible,
+tested with VirtualBox/AWS/DigitalOcean - see https://github.com/EMnify/matrix-synapse-auto-deploy
+for details.

-Slavi Pantaleev has created an Ansible playbook,
-which installs the offical Docker image of Matrix Synapse
-along with many other Matrix-related services (Postgres database, riot-web, coturn, mxisd, SSL support, etc.).
-For more details, see
-https://github.com/spantaleev/matrix-docker-ansible-deploy
-
-Configuring Synapse
+Configuring synapse
 -------------------

 Before you can start Synapse, you will need to generate a configuration
@@ -238,7 +224,7 @@ commandline script.

 To get started, it is easiest to use the command line to register new users::

-    $ source ~/synapse/env/bin/activate
+    $ source ~/.synapse/bin/activate
    $ synctl start # if not already running
    $ register_new_matrix_user -c homeserver.yaml https://localhost:8448
    New user localpart: erikj
@@ -260,22 +246,43 @@ Setting up a TURN server
 For reliable VoIP calls to be routed via this homeserver, you MUST configure
 a TURN server.  See `<docs/turn-howto.rst>`_ for details.

+IPv6
+----
+
+As of Synapse 0.19 we finally support IPv6, many thanks to @kyrias and @glyph
+for providing PR #1696.
+
+However, for federation to work on hosts with IPv6 DNS servers you **must**
+be running Twisted 17.1.0 or later - see https://github.com/matrix-org/synapse/issues/1002
+for details.  We can't make Synapse depend on Twisted 17.1 by default
+yet as it will break most older distributions (see https://github.com/matrix-org/synapse/pull/1909)
+so if you are using operating system dependencies you'll have to install your
+own Twisted 17.1 package via pip or backports etc.
+
+If you're running in a virtualenv then pip should have installed the newest
+Twisted automatically, but if your virtualenv is old you will need to manually
+upgrade to a newer Twisted dependency via:
+
+    pip install Twisted>=17.1.0
+
+
 Running Synapse
 ===============

 To actually run your new homeserver, pick a working directory for Synapse to
-run (e.g. ``~/synapse``), and::
+run (e.g. ``~/.synapse``), and::

-    cd ~/synapse
-    source env/bin/activate
+    cd ~/.synapse
+    source ./bin/activate
    synctl start

+
 Connecting to Synapse from a client
 ===================================

 The easiest way to try out your new Synapse installation is by connecting to it
 from a web client. The easiest option is probably the one at
-https://riot.im/app. You will need to specify a "Custom server" when you log on
+http://riot.im/app. You will need to specify a "Custom server" when you log on
 or register: set this to ``https://domain.tld`` if you setup a reverse proxy
 following the recommended setup, or ``https://localhost:8448`` - remember to specify the
 port (``:8448``) if not ``:443`` unless you changed the configuration. (Leave the identity
@@ -321,7 +328,7 @@ Security Note
 =============

 Matrix serves raw user generated data in some APIs - specifically the `content
-repository endpoints <https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid>`_.
+repository endpoints <http://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid>`_.

 Whilst we have tried to mitigate against possible XSS attacks (e.g.
 https://github.com/matrix-org/synapse/pull/1021) we recommend running
@@ -330,7 +337,7 @@ content served to web browsers a matrix API from being able to attack webapps ho
 on the same domain.  This is particularly true of sharing a matrix webclient and
 server on the same domain.

-See https://github.com/vector-im/riot-web/issues/1977 and
+See https://github.com/vector-im/vector-web/issues/1977 and
 https://developer.github.com/changes/2014-04-25-user-content-security for more details.


@@ -340,7 +347,7 @@ Platform-Specific Instructions
 Debian
 ------

-Matrix provides official Debian packages via apt from https://matrix.org/packages/debian/.
+Matrix provides official Debian packages via apt from http://matrix.org/packages/debian/.
 Note that these packages do not include a client - choose one from
 https://matrix.org/docs/projects/try-matrix-now.html (or build your own with one of our SDKs :)

@@ -354,19 +361,6 @@ Synapse is in the Fedora repositories as ``matrix-synapse``::
 Oleg Girko provides Fedora RPMs at
 https://obs.infoserver.lv/project/monitor/matrix-synapse

-OpenSUSE
--------
-
-Synapse is in the OpenSUSE repositories as ``matrix-synapse``::
-
-    sudo zypper install matrix-synapse
-
-SUSE Linux Enterprise Server
----------------------------
-
-Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 repository at
-https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/
-
 ArchLinux
 ---------

@@ -377,17 +371,35 @@ the generated config),
 https://www.archlinux.org/packages/community/any/python2-matrix-angular-sdk/ will also need to
 be installed.

+Alternatively, to install using pip a few changes may be needed as ArchLinux
+defaults to python 3, but synapse currently assumes python 2.7 by default:
+
 pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 )::

-    sudo pip install --upgrade pip
+    sudo pip2.7 install --upgrade pip
+
+You also may need to explicitly specify python 2.7 again during the install
+request::
+
+    pip2.7 install https://github.com/matrix-org/synapse/tarball/master

 If you encounter an error with lib bcrypt causing an Wrong ELF Class:
 ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly
 compile it under the right architecture. (This should not be needed if
 installing under virtualenv)::

-    sudo pip uninstall py-bcrypt
-    sudo pip install py-bcrypt
+    sudo pip2.7 uninstall py-bcrypt
+    sudo pip2.7 install py-bcrypt
+
+During setup of Synapse you need to call python2.7 directly again::
+
+    cd ~/.synapse
+    python2.7 -m synapse.app.homeserver \
+      --server-name machine.my.domain.name \
+      --config-path homeserver.yaml \
+      --generate-config
+
+...substituting your host and domain name as appropriate.

 FreeBSD
 -------
@@ -416,7 +428,8 @@ settings require a slightly more difficult installation process.
   using the ``.`` command, rather than ``bash``'s ``source``.
 5) Optionally, use ``pip`` to install ``lxml``, which Synapse needs to parse
   webpages for their titles.
-6) Use ``pip`` to install this repository: ``pip install matrix-synapse``
+6) Use ``pip`` to install this repository: ``pip install
+   https://github.com/matrix-org/synapse/tarball/master``
 7) Optionally, change ``_synapse``'s shell to ``/bin/false`` to reduce the
   chance of a compromised Synapse server being used to take over your box.

@@ -430,13 +443,37 @@ https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-

 Windows Install
 ---------------
+Synapse can be installed on Cygwin. It requires the following Cygwin packages:
+
+- gcc
+- git
+- libffi-devel
+- openssl (and openssl-devel, python-openssl)
+- python
+- python-setuptools
+
+The content repository requires additional packages and will be unable to process
+uploads without them:
+
+- libjpeg8
+- libjpeg8-devel
+- zlib
+
+If you choose to install Synapse without these packages, you will need to reinstall
+``pillow`` for changes to be applied, e.g. ``pip uninstall pillow`` ``pip install
+pillow --user``
+
+Troubleshooting:
+
+- You may need to upgrade ``setuptools`` to get this to work correctly:
+  ``pip install setuptools --upgrade``.
+- You may encounter errors indicating that ``ffi.h`` is missing, even with
+  ``libffi-devel`` installed. If you do, copy the ``.h`` files:
+  ``cp /usr/lib/libffi-3.0.13/include/*.h /usr/include``
+- You may need to install libsodium from source in order to install PyNacl. If
+  you do, you may need to create a symlink to ``libsodium.a`` so ``ld`` can find
+  it: ``ln -s /usr/local/lib/libsodium.a /usr/lib/libsodium.a``

-If you wish to run or develop Synapse on Windows, the Windows Subsystem For
-Linux provides a Linux environment on Windows 10 which is capable of using the
-Debian, Fedora, or source installation methods. More information about WSL can
-be found at https://docs.microsoft.com/en-us/windows/wsl/install-win10 for
-Windows 10 and https://docs.microsoft.com/en-us/windows/wsl/install-on-server
-for Windows Server.

 Troubleshooting
 ===============
@@ -444,7 +481,7 @@ Troubleshooting
 Troubleshooting Installation
 ----------------------------

-Synapse requires pip 8 or later, so if your OS provides too old a version you
+Synapse requires pip 1.7 or later, so if your OS provides too old a version you
 may need to manually upgrade it::

    sudo pip install --upgrade pip
@@ -454,7 +491,7 @@ You can fix this by manually upgrading pip and virtualenv::

    sudo pip install --upgrade virtualenv

-You can next rerun ``virtualenv -p python3 synapse`` to update the virtual env.
+You can next rerun ``virtualenv -p python2.7 synapse`` to update the virtual env.

 Installing may fail during installing virtualenv with ``InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.``
 You can fix this  by manually installing ndg-httpsclient::
@@ -479,6 +516,28 @@ failing, e.g.::

    pip install twisted

+On OS X, if you encounter clang: error: unknown argument: '-mno-fused-madd' you
+will need to export CFLAGS=-Qunused-arguments.
+
+Troubleshooting Running
+-----------------------
+
+If synapse fails with ``missing "sodium.h"`` crypto errors, you may need
+to manually upgrade PyNaCL, as synapse uses NaCl (http://nacl.cr.yp.to/) for
+encryption and digital signatures.
+Unfortunately PyNACL currently has a few issues
+(https://github.com/pyca/pynacl/issues/53) and
+(https://github.com/pyca/pynacl/issues/79) that mean it may not install
+correctly, causing all tests to fail with errors about missing "sodium.h". To
+fix try re-installing from PyPI or directly from
+(https://github.com/pyca/pynacl)::
+
+    # Install from PyPI
+    pip install --user --upgrade --force pynacl
+
+    # Install from github
+    pip install --user https://github.com/pyca/pynacl/tarball/master
+
 Running out of File Handles
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -503,6 +562,16 @@ log lines and looking for any 'Processed request' lines which take more than
 a few seconds to execute.  Please let us know at #matrix-dev:matrix.org if
 you see this failure mode so we can help debug it, however.

+ArchLinux
+~~~~~~~~~
+
+If running `$ synctl start` fails with 'returned non-zero exit status 1',
+you will need to explicitly call Python2.7 - either running as::
+
+    python2.7 -m synapse.app.homeserver --daemonize -c homeserver.yaml
+
+...or by editing synctl with the correct python executable.
+

 Upgrading an existing Synapse
 =============================
@@ -602,8 +671,8 @@ useful just for development purposes. See `<demo/README>`_.
 Using PostgreSQL
 ================

-As of Synapse 0.9, `PostgreSQL <https://www.postgresql.org>`_ is supported as an
-alternative to the `SQLite <https://sqlite.org/>`_ database that Synapse has
+As of Synapse 0.9, `PostgreSQL <http://www.postgresql.org>`_ is supported as an
+alternative to the `SQLite <http://sqlite.org/>`_ database that Synapse has
 traditionally used for convenience and simplicity.

 The advantages of Postgres include:
@@ -626,9 +695,8 @@ Using a reverse proxy with Synapse

 It is recommended to put a reverse proxy such as
 `nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
-`Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_,
-`Caddy <https://caddyserver.com/docs/proxy>`_ or
-`HAProxy <https://www.haproxy.org/>`_ in front of Synapse. One advantage of
+`Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_ or
+`HAProxy <http://www.haproxy.org/>`_ in front of Synapse. One advantage of
 doing so is that it means that you can expose the default https port (443) to
 Matrix clients without needing to run Synapse with root privileges.

@@ -658,26 +726,6 @@ so an example nginx configuration might look like::
      }
  }

-an example Caddy configuration might look like::
-
-    matrix.example.com {
-      proxy /_matrix http://localhost:8008 {
-        transparent
-      }
-    }
-
-and an example Apache configuration might look like::
-
-    <VirtualHost *:443>
-        SSLEngine on
-        ServerName matrix.example.com;
-
-        <Location /_matrix>
-            ProxyPass http://127.0.0.1:8008/_matrix nocanon
-            ProxyPassReverse http://127.0.0.1:8008/_matrix
-        </Location>
-    </VirtualHost>
-
 You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true``
 for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are
 recorded correctly.
@@ -698,10 +746,9 @@ port:

  .. __: `key_management`_

-* Until v0.33.3, Synapse did not support SNI on the federation port
-  (`bug #1491 <https://github.com/matrix-org/synapse/issues/1491>`_). This bug
-  is now fixed, but means that federating with older servers can be unreliable
-  when using name-based virtual hosting.
+* Synapse does not currently support SNI on the federation protocol
+  (`bug #1491 <https://github.com/matrix-org/synapse/issues/1491>`_), which
+  means that using name-based virtual hosting is unreliable.

 Furthermore, a number of the normal reasons for using a reverse-proxy do not
 apply:
@@ -797,7 +844,7 @@ Password reset
 ==============

 If a user has registered an email address to their account using an identity
-server, they can request a password-reset token via clients such as Riot.
+server, they can request a password-reset token via clients such as Vector.

 A manual password reset can be done via direct database access as follows.

@@ -833,7 +880,7 @@ to install using pip and a virtualenv::

    virtualenv -p python2.7 env
    source env/bin/activate
-    python -m synapse.python_dependencies | xargs pip install
+    python synapse/python_dependencies.py | xargs pip install
    pip install lxml mock

 This will run a process of downloading and installing all the needed
@@ -888,13 +935,5 @@ variable.  The default is 0.5, which can be decreased to reduce RAM usage
 in memory constrained enviroments, or increased if performance starts to
 degrade.

-Using `libjemalloc <http://jemalloc.net/>`_ can also yield a significant
-improvement in overall amount, and especially in terms of giving back RAM
-to the OS. To use it, the library must simply be put in the LD_PRELOAD
-environment variable when launching Synapse. On Debian, this can be done
-by installing the ``libjemalloc1`` package and adding this line to
-``/etc/default/matrix-synapse``::
-
-    LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1

 .. _`key_management`: https://matrix.org/docs/spec/server_server/unstable.html#retrieving-server-keys
@@ -18,7 +18,7 @@ instructions that may be required are listed later in this document.

   .. code:: bash

-       pip install --upgrade --process-dependency-links matrix-synapse
+       pip install --upgrade --process-dependency-links https://github.com/matrix-org/synapse/tarball/master

       # restart synapse
       synctl restart
@@ -48,92 +48,11 @@ returned by the Client-Server API:
    # configured on port 443.
    curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"

-Upgrading to v0.34.0
-====================
-
-1. This release is the first to fully support Python 3. We recommend switching
-   to Python 3, as it has been shown to give performance improvements.
-
-   For users who have installed Synapse into a virtualenv, we recommend doing
-   this by creating a new virtualenv. For example::
-
-       virtualenv -p python3 ~/synapse/env3
-       source ~/synapse/env3/bin/activate
-       pip install matrix-synapse
-
-   You can then start synapse as normal, having activated the new virtualenv::
-
-       cd ~/synapse
-       source env3/bin/activate
-       synctl start
-
-   Users who have installed from distribution packages should see the relevant
-   package documentation.
-
-   * When upgrading to Python 3, you **must** make sure that your log files are
-     configured as UTF-8, by adding ``encoding: utf8`` to the
-     ``RotatingFileHandler`` configuration (if you have one) in your
-     ``<server>.log.config`` file. For example, if your ``log.config`` file
-     contains::
-
-       handlers:
-         file:
-           class: logging.handlers.RotatingFileHandler
-           formatter: precise
-           filename: homeserver.log
-           maxBytes: 104857600
-           backupCount: 10
-           filters: [context]
-         console:
-           class: logging.StreamHandler
-           formatter: precise
-           filters: [context]
-
-     Then you should update this to be::
-
-       handlers:
-         file:
-           class: logging.handlers.RotatingFileHandler
-           formatter: precise
-           filename: homeserver.log
-           maxBytes: 104857600
-           backupCount: 10
-           filters: [context]
-           encoding: utf8
-         console:
-           class: logging.StreamHandler
-           formatter: precise
-           filters: [context]
-
-     There is no need to revert this change if downgrading to Python 2.
-
-2. This release removes the ``riot.im`` from the default list of trusted
-   identity servers.
-
-   If ``riot.im`` is in your homeserver's list of
-   ``trusted_third_party_id_servers``, you should remove it. It was added in
-   case a hypothetical future identity server was put there. If you don't
-   remove it, users may be unable to deactivate their accounts.
-
-
-Upgrading to v0.33.7
-====================
-
-This release removes the example email notification templates from
-``res/templates`` (they are now internal to the python package). This should
-only affect you if you (a) deploy your Synapse instance from a git checkout or
-a github snapshot URL, and (b) have email notifications enabled.
-
-If you have email notifications enabled, you should ensure that
-``email.template_dir`` is either configured to point at a directory where you
-have installed customised templates, or leave it unset to use the default
-templates.
-
-Upgrading to v0.27.3
+Upgrading to $NEXT_VERSION
 ====================

 This release expands the anonymous usage stats sent if the opt-in
-``report_stats`` configuration is set to ``true``. We now capture RSS memory
+``report_stats`` configuration is set to ``true``. We now capture RSS memory 
 and cpu use at a very coarse level. This requires administrators to install
 the optional ``psutil`` python module.

@@ -1 +0,0 @@
-!.gitignore
@@ -1 +0,0 @@
-Add option to track MAU stats (but not limit people)
@@ -1 +0,0 @@
-Add an option to enable recording IPs for appservice users
@@ -1 +0,0 @@
-Pushrules can now again be made with non-ASCII rule IDs.
@@ -1 +0,0 @@
-add cross-signing support
@@ -1 +0,0 @@
-The media repository now no longer fails to decode UTF-8 filenames when downloading remote media.
@@ -1 +0,0 @@
-A coveragerc file, as well as the py36-coverage tox target, have been added.
@@ -1 +0,0 @@
-Add a GitHub pull request template and add multiple issue templates
@@ -1 +0,0 @@
-URL previews now correctly decode non-UTF-8 text if the header contains a `<meta http-equiv="Content-Type"` header.
@@ -1 +0,0 @@
-Update README to reflect the fact that #1491 is fixed
@@ -1,2 +0,0 @@
-Run the AS senders as background processes to fix warnings
-
@@ -1 +0,0 @@
-Add some diagnostics to the tests to detect logcontext problems
@@ -1 +0,0 @@
-Fix an issue where public consent URLs had two slashes.
@@ -1 +0,0 @@
-Add missing `jpeg` package prerequisite for OpenBSD in README.
@@ -1 +0,0 @@
-Fallback auth now accepts the session parameter on Python 3.
@@ -1 +0,0 @@
-Add a note saying you need to manually reclaim disk space after using the Purge History API
@@ -1 +0,0 @@
-Fix logcontext leaks in EmailPusher and in tests
@@ -1 +0,0 @@
-More logcontext checking in unittests
@@ -1 +0,0 @@
-Remove riot.im from the list of trusted Identity Servers in the default configuration
@@ -1 +0,0 @@
-Fix logcontext leaks in EmailPusher and in tests
@@ -1,2 +0,0 @@
-fix start up failure when mau_limit_reserved_threepids set and db is postgres 
-
@@ -1 +0,0 @@
-Ignore __pycache__ directories in the database schema folder
@@ -1 +0,0 @@
-Rename login type m.login.cas to m.login.sso
@@ -1 +0,0 @@
-Fix auto join failures for servers that require user consent
@@ -1 +0,0 @@
-Add note to UPGRADE.rst about removing riot.im from list of trusted identity servers
@@ -1 +0,0 @@
-Added automated coverage reporting to CI.
@@ -1 +0,0 @@
-Garbage-collect after each unit test to fix logcontext leaks
@@ -1 +0,0 @@
-Add an option to disable search for homeservers that may not be interested in it.
@@ -1 +0,0 @@
-add more detail to logging regarding "More than one row matched" error
@@ -1 +0,0 @@
-Fix exception caused by non-ascii event IDs
@@ -1 +0,0 @@
-Drop sent_transactions table
@@ -1 +0,0 @@
-Pushers can now be unsubscribed from on Python 3.
@@ -1 +0,0 @@
-Fix UnicodeDecodeError when postgres is configured to give non-English errors
@@ -1 +0,0 @@
-Add a basic .editorconfig
@@ -1 +0,0 @@
-Update README.rst and UPGRADE.rst for Python 3.
@@ -1 +0,0 @@
-Remove obsolete `verbose` and `log_file` settings from `homeserver.yaml` for Docker image.
@@ -1 +0,0 @@
-Support for serving .well-known files
@@ -1 +0,0 @@
-drop undocumented dependency on dateutil
@@ -1,41 +0,0 @@
-# Synapse Docker
-
-### Automated configuration
-
-It is recommended that you use Docker Compose to run your containers, including
-this image and a Postgres server. A sample ``docker-compose.yml`` is provided,
-including example labels for reverse proxying and other artifacts.
-
-Read the section about environment variables and set at least mandatory variables,
-then run the server:
-
-```
-docker-compose up -d
-```
-
-If secrets are not specified in the environment variables, they will be generated
-as part of the startup. Please ensure these secrets are kept between launches of the
-Docker container, as their loss may require users to log in again.
-
-### Manual configuration
-
-A sample ``docker-compose.yml`` is provided, including example labels for
-reverse proxying and other artifacts. The docker-compose file is an example,
-please comment/uncomment sections that are not suitable for your usecase.
-
-Specify a ``SYNAPSE_CONFIG_PATH``, preferably to a persistent path,
-to use manual configuration. To generate a fresh ``homeserver.yaml``, simply run:
-
-```
-docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host synapse generate
-```
-
-Then, customize your configuration and run the server:
-
-```
-docker-compose up -d
-```
-
-### More information
-
-For more information on required environment variables and mounts, see the main docker documentation at [/docker/README.md](../../docker/README.md)
@@ -1,52 +0,0 @@
-# This compose file is compatible with Compose itself, it might need some
-# adjustments to run properly with stack.
-
-version: '3'
-
-services:
-
-  synapse:
-    build:
-        context: ../..
-        dockerfile: docker/Dockerfile
-    image: docker.io/matrixdotorg/synapse:latest
-    # Since synapse does not retry to connect to the database, restart upon
-    # failure
-    restart: unless-stopped
-    # See the readme for a full documentation of the environment settings
-    environment:
-      - SYNAPSE_SERVER_NAME=my.matrix.host
-      - SYNAPSE_REPORT_STATS=no
-      - SYNAPSE_ENABLE_REGISTRATION=yes
-      - SYNAPSE_LOG_LEVEL=INFO
-      - POSTGRES_PASSWORD=changeme
-    volumes:
-      # You may either store all the files in a local folder
-      - ./files:/data
-      # .. or you may split this between different storage points
-      # - ./files:/data
-      # - /path/to/ssd:/data/uploads
-      # - /path/to/large_hdd:/data/media
-    depends_on:
-      - db
-    # In order to expose Synapse, remove one of the following, you might for
-    # instance expose the TLS port directly:
-    ports:
-      - 8448:8448/tcp
-    # ... or use a reverse proxy, here is an example for traefik:
-    labels:
-      - traefik.enable=true
-      - traefik.frontend.rule=Host:my.matrix.Host
-      - traefik.port=8448
-
-  db:
-    image: docker.io/postgres:10-alpine
-    # Change that password, of course!
-    environment:
-      - POSTGRES_USER=synapse
-      - POSTGRES_PASSWORD=changeme
-    volumes:
-      # You may store the database tables in a local folder..
-      - ./schemas:/var/lib/postgresql/data
-      # .. or store them on some high performance storage for better results
-      # - /path/to/ssd/storage:/var/lib/postgresql/data
@@ -1,6 +0,0 @@
-# Using the Synapse Grafana dashboard
-
-0. Set up Prometheus and Grafana. Out of scope for this readme. Useful documentation about using Grafana with Prometheus: http://docs.grafana.org/features/datasources/prometheus/
-1. Have your Prometheus scrape your Synapse. https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.rst
-2. Import dashboard into Grafana. Download `synapse.json`. Import it to Grafana and select the correct Prometheus datasource. http://docs.grafana.org/reference/export_import/
-3. Set up additional recording rules
@@ -1,16 +0,0 @@
-Purge history API examples
-==========================
-
-# `purge_history.sh`
-
-A bash file, that uses the [purge history API](/docs/admin_api/README.rst) to 
-purge all messages in a list of rooms up to a certain event. You can select a 
-timeframe or a number of messages that you want to keep in the room.
-
-Just configure the variables DOMAIN, ADMIN, ROOMS_ARRAY and TIME at the top of
-the script.
-
-# `purge_remote_media.sh`
-
-A bash file, that uses the [purge history API](/docs/admin_api/README.rst) to 
-purge all old cached remote media.
@@ -1,141 +0,0 @@
-#!/bin/bash
-
-# this script will use the api:
-#    https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst
-# 
-# It will purge all messages in a list of rooms up to a cetrain event
-
-###################################################################################################
-# define your domain and admin user
-###################################################################################################
-# add this user as admin in your home server:
-DOMAIN=yourserver.tld
-# add this user as admin in your home server:
-ADMIN="@you_admin_username:$DOMAIN"
-
-API_URL="$DOMAIN:8008/_matrix/client/r0"
-
-###################################################################################################
-#choose the rooms to prune old messages from (add a free comment at the end)
-###################################################################################################
-# the room_id's you can get e.g. from your Riot clients "View Source" button on each message
-ROOMS_ARRAY=(
-'!DgvjtOljKujDBrxyHk:matrix.org#riot:matrix.org'
-'!QtykxKocfZaZOUrTwp:matrix.org#Matrix HQ'
-)
-
-# ALTERNATIVELY:
-# you can select all the rooms that are not encrypted and loop over the result:
-# SELECT room_id FROM rooms WHERE room_id NOT IN (SELECT DISTINCT room_id FROM events WHERE type ='m.room.encrypted')
-# or
-# select all rooms with at least 100 members:
-# SELECT q.room_id FROM (select count(*) as numberofusers, room_id FROM current_state_events WHERE type ='m.room.member'
-#   GROUP BY room_id) AS q LEFT JOIN room_aliases a ON q.room_id=a.room_id WHERE q.numberofusers > 100 ORDER BY numberofusers desc
-
-###################################################################################################
-# evaluate the EVENT_ID before which should be pruned
-###################################################################################################
-# choose a time before which the messages should be pruned:
-TIME='12 months ago'
-# ALTERNATIVELY:
-# a certain time:
-# TIME='2016-08-31 23:59:59'
-
-# creates a timestamp from the given time string:
-UNIX_TIMESTAMP=$(date +%s%3N --date='TZ="UTC+2" '"$TIME")
-
-# ALTERNATIVELY:
-# prune all messages that are older than 1000 messages ago:
-# LAST_MESSAGES=1000
-# SQL_GET_EVENT="SELECT event_id from events WHERE type='m.room.message' AND room_id ='$ROOM' ORDER BY received_ts DESC LIMIT 1 offset $(($LAST_MESSAGES - 1))"
-
-# ALTERNATIVELY:
-# select the EVENT_ID manually:
-#EVENT_ID='$1471814088343495zpPNI:matrix.org' # an example event from 21st of Aug 2016 by Matthew
-
-###################################################################################################
-# make the admin user a server admin in the database with
-###################################################################################################
-# psql -A -t --dbname=synapse -c "UPDATE users SET admin=1 WHERE name LIKE '$ADMIN'"
-
-###################################################################################################
-# database function
-###################################################################################################
-sql (){
-  # for sqlite3:
-  #sqlite3 homeserver.db "pragma busy_timeout=20000;$1" | awk '{print $2}'
-  # for postgres:
-  psql -A -t --dbname=synapse -c "$1" | grep -v 'Pager'
-}
-
-###################################################################################################
-# get an access token
-###################################################################################################
-# for example externally by watching Riot in your browser's network inspector
-# or internally on the server locally, use this:
-TOKEN=$(sql "SELECT token FROM access_tokens WHERE user_id='$ADMIN' ORDER BY id DESC LIMIT 1")
-AUTH="Authorization: Bearer $TOKEN"
-
-###################################################################################################
-# check, if your TOKEN works. For example this works: 
-###################################################################################################
-# $ curl --header "$AUTH" "$API_URL/rooms/$ROOM/state/m.room.power_levels" 
-
-###################################################################################################
-# finally start pruning the room:
-###################################################################################################
-POSTDATA='{"delete_local_events":"true"}' # this will really delete local events, so the messages in the room really disappear unless they are restored by remote federation
-
-for ROOM in "${ROOMS_ARRAY[@]}"; do
-    echo "########################################### $(date) ################# "
-    echo "pruning room: $ROOM ..."
-    ROOM=${ROOM%#*}
-    #set -x
-    echo "check for alias in db..."
-    # for postgres:
-    sql "SELECT * FROM room_aliases WHERE room_id='$ROOM'"
-    echo "get event..."
-    # for postgres:
-    EVENT_ID=$(sql "SELECT event_id FROM events WHERE type='m.room.message' AND received_ts<'$UNIX_TIMESTAMP' AND room_id='$ROOM' ORDER BY received_ts DESC LIMIT 1;")
-    if [ "$EVENT_ID" == "" ]; then
-      echo "no event $TIME"
-    else
-      echo "event: $EVENT_ID"
-      SLEEP=2
-      set -x
-      # call purge
-      OUT=$(curl --header "$AUTH" -s -d $POSTDATA POST "$API_URL/admin/purge_history/$ROOM/$EVENT_ID")
-      PURGE_ID=$(echo "$OUT" |grep purge_id|cut -d'"' -f4 )
-      if [ "$PURGE_ID" == "" ]; then
-        # probably the history purge is already in progress for $ROOM
-        : "continuing with next room"
-      else
-        while : ; do
-          # get status of purge and sleep longer each time if still active
-          sleep $SLEEP
-          STATUS=$(curl --header "$AUTH" -s GET "$API_URL/admin/purge_history_status/$PURGE_ID" |grep status|cut -d'"' -f4)
-          : "$ROOM --> Status: $STATUS"
-          [[ "$STATUS" == "active" ]] || break 
-          SLEEP=$((SLEEP + 1))
-        done 
-      fi
-      set +x
-      sleep 1
-    fi  
-done
-
-
-###################################################################################################
-# additionally
-###################################################################################################
-# to benefit from pruning large amounts of data, you need to call VACUUM to free the unused space.
-# This can take a very long time (hours) and the client have to be stopped while you do so:
-# $ synctl stop
-# $ sqlite3 -line homeserver.db "vacuum;"
-# $ synctl start
-
-# This could be set, so you don't need to prune every time after deleting some rows:
-# $ sqlite3 homeserver.db "PRAGMA auto_vacuum = FULL;"
-# be cautious, it could make the database somewhat slow if there are a lot of deletions
-
-exit
@@ -1,54 +0,0 @@
-#!/bin/bash
-
-DOMAIN=yourserver.tld
-# add this user as admin in your home server:
-ADMIN="@you_admin_username:$DOMAIN"
-
-API_URL="$DOMAIN:8008/_matrix/client/r0"
-
-# choose a time before which the messages should be pruned:
-# TIME='2016-08-31 23:59:59'
-TIME='12 months ago'
-
-# creates a timestamp from the given time string:
-UNIX_TIMESTAMP=$(date +%s%3N --date='TZ="UTC+2" '"$TIME")
-
-
-###################################################################################################
-# database function
-###################################################################################################
-sql (){
-  # for sqlite3:
-  #sqlite3 homeserver.db "pragma busy_timeout=20000;$1" | awk '{print $2}'
-  # for postgres:
-  psql -A -t --dbname=synapse -c "$1" | grep -v 'Pager'
-}
-
-###############################################################################
-# make the admin user a server admin in the database with
-###############################################################################
-# sql "UPDATE users SET admin=1 WHERE name LIKE '$ADMIN'"
-
-###############################################################################
-# get an access token
-###############################################################################
-# for example externally by watching Riot in your browser's network inspector
-# or internally on the server locally, use this:
-TOKEN=$(sql "SELECT token FROM access_tokens WHERE user_id='$ADMIN' ORDER BY id DESC LIMIT 1")
-
-###############################################################################
-# check, if your TOKEN works. For example this works: 
-###############################################################################
-# curl --header "Authorization: Bearer $TOKEN" "$API_URL/rooms/$ROOM/state/m.room.power_levels"
-
-###############################################################################
-# optional check size before
-###############################################################################
-# echo calculate used storage before ...
-# du -shc ../.synapse/media_store/*
-
-###############################################################################
-# finally start pruning media:
-###############################################################################
-set -x # for debugging the generated string
-curl --header "Authorization: Bearer $TOKEN" -v POST "$API_URL/admin/purge_media_cache/?before_ts=$UNIX_TIMESTAMP"
@@ -1,63 +0,0 @@
-ARG PYTHON_VERSION=2
-
-###
-### Stage 0: builder
-###
-FROM docker.io/python:${PYTHON_VERSION}-alpine3.8 as builder
-
-# install the OS build deps
-
-RUN apk add \
-        build-base \
-        libffi-dev \
-        libjpeg-turbo-dev \
-        libressl-dev \
-        libxslt-dev \
-        linux-headers \
-        postgresql-dev \
-        zlib-dev
-
-# build things which have slow build steps, before we copy synapse, so that
-# the layer can be cached.
-#
-# (we really just care about caching a wheel here, as the "pip install" below
-# will install them again.)
-
-RUN pip install --prefix="/install" --no-warn-script-location \
-        cryptography \
-        msgpack-python \
-        pillow \
-        pynacl
-
-# now install synapse and all of the python deps to /install.
-
-COPY . /synapse
-RUN pip install --prefix="/install" --no-warn-script-location \
-        lxml \
-        psycopg2 \
-        /synapse
-
-###
-### Stage 1: runtime
-###
-
-FROM docker.io/python:${PYTHON_VERSION}-alpine3.8
-
-RUN apk add --no-cache --virtual .runtime_deps \
-        libffi \
-        libjpeg-turbo \
-        libressl \
-        libxslt \
-        libpq \
-        zlib \
-        su-exec
-
-COPY --from=builder /install /usr/local
-COPY ./docker/start.py /start.py
-COPY ./docker/conf /conf
-
-VOLUME ["/data"]
-
-EXPOSE 8008/tcp 8448/tcp
-
-ENTRYPOINT ["/start.py"]
@@ -1,12 +0,0 @@
-# Use the Sytest image that comes with a lot of the build dependencies
-# pre-installed
-FROM matrixdotorg/sytest:latest
-
-# The Sytest image doesn't come with python, so install that
-RUN apt-get -qq install -y python python-dev python-pip
-
-# We need tox to run the tests in run_pg_tests.sh
-RUN pip install tox
-
-ADD run_pg_tests.sh /pg_tests.sh
-ENTRYPOINT /pg_tests.sh
@@ -1,125 +0,0 @@
-# Synapse Docker
-
-This Docker image will run Synapse as a single process. It does not provide a database
-server or a TURN server, you should run these separately.
-
-## Run
-
-We do not currently offer a `latest` image, as this has somewhat undefined semantics.
-We instead release only tagged versions so upgrading between releases is entirely
-within your control.
-
-### Using docker-compose (easier)
-
-This image is designed to run either with an automatically generated configuration
-file or with a custom configuration that requires manual editing.
-
-An easy way to make use of this image is via docker-compose. See the
-[contrib/docker](../contrib/docker)
-section of the synapse project for examples.
-
-### Without Compose (harder)
-
-If you do not wish to use Compose, you may still run this image using plain
-Docker commands. Note that the following is just a guideline and you may need
-to add parameters to the docker run command to account for the network situation
-with your postgres database.
-
-```
-docker run \
-    -d \
-    --name synapse \
-    -v ${DATA_PATH}:/data \
-    -e SYNAPSE_SERVER_NAME=my.matrix.host \
-    -e SYNAPSE_REPORT_STATS=yes \
-    docker.io/matrixdotorg/synapse:latest
-```
-
-## Volumes
-
-The image expects a single volume, located at ``/data``, that will hold:
-
-* temporary files during uploads;
-* uploaded media and thumbnails;
-* the SQLite database if you do not configure postgres;
-* the appservices configuration.
-
-You are free to use separate volumes depending on storage endpoints at your
-disposal. For instance, ``/data/media`` coud be stored on a large but low
-performance hdd storage while other files could be stored on high performance
-endpoints.
-
-In order to setup an application service, simply create an ``appservices``
-directory in the data volume and write the application service Yaml
-configuration file there. Multiple application services are supported.
-
-## Environment
-
-Unless you specify a custom path for the configuration file, a very generic
-file will be generated, based on the following environment settings.
-These are a good starting point for setting up your own deployment.
-
-Global settings:
-
-* ``UID``, the user id Synapse will run as [default 991]
-* ``GID``, the group id Synapse will run as [default 991]
-* ``SYNAPSE_CONFIG_PATH``, path to a custom config file
-
-If ``SYNAPSE_CONFIG_PATH`` is set, you should generate a configuration file
-then customize it manually. No other environment variable is required.
-
-Otherwise, a dynamic configuration file will be used. The following environment
-variables are available for configuration:
-
-* ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname.
-* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous
-  statistics reporting back to the Matrix project which helps us to get funding.
-* ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if
-  you run your own TLS-capable reverse proxy).
-* ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on
-  the Synapse instance.
-* ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server.
-* ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`].
-* ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`].
-* ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public
-  key in order to enable recaptcha upon registration.
-* ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private
-  key in order to enable recaptcha upon registration.
-* ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN
-  uris to enable TURN for this homeserver.
-* ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required.
-* ``SYNAPSE_MAX_UPLOAD_SIZE``, set this variable to change the max upload size [default `10M`].
-
-Shared secrets, that will be initialized to random values if not set:
-
-* ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if
-  registration is disable.
-* ``SYNAPSE_MACAROON_SECRET_KEY`` secret for signing access tokens
-  to the server.
-
-Database specific values (will use SQLite if not set):
-
-* `POSTGRES_DB` - The database name for the synapse postgres database. [default: `synapse`]
-* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`]
-* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy.
-* `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`]
-
-Mail server specific values (will not send emails if not set):
-
-* ``SYNAPSE_SMTP_HOST``, hostname to the mail server.
-* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default ``25``].
-* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if any.
-* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail server if any.
-
-## Build
-
-Build the docker image with the `docker build` command from the root of the synapse repository.
-
-```
-docker build -t docker.io/matrixdotorg/synapse . -f docker/Dockerfile
-```
-
-The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:<version>` where `<version>` is the same as the release tag in the synapse git repository.
-
-You may have a local Python wheel cache available, in which case copy the relevant
-packages in the ``cache/`` directory at the root of the project.
@@ -1,221 +0,0 @@
-# vim:ft=yaml
-
-## TLS ##
-
-tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt"
-tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key"
-tls_dh_params_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.dh"
-no_tls: {{ "True" if SYNAPSE_NO_TLS else "False" }}
-tls_fingerprints: []
-
-## Server ##
-
-server_name: "{{ SYNAPSE_SERVER_NAME }}"
-pid_file: /homeserver.pid
-web_client: False
-soft_file_limit: 0
-log_config: "/compiled/log.config"
-
-## Ports ##
-
-listeners:
-  {% if not SYNAPSE_NO_TLS %}
-  -
-    port: 8448
-    bind_addresses: ['::']
-    type: http
-    tls: true
-    x_forwarded: false
-    resources:
-      - names: [client]
-        compress: true
-      - names: [federation]  # Federation APIs
-        compress: false
-  {% endif %}
-
-  - port: 8008
-    tls: false
-    bind_addresses: ['::']
-    type: http
-    x_forwarded: false
-
-    resources:
-      - names: [client]
-        compress: true
-      - names: [federation]
-        compress: false
-
-## Database ##
-
-{% if POSTGRES_PASSWORD %}
-database:
-  name: "psycopg2"
-  args:
-    user: "{{ POSTGRES_USER or "synapse" }}"
-    password: "{{ POSTGRES_PASSWORD }}"
-    database: "{{ POSTGRES_DB or "synapse" }}"
-    host: "{{ POSTGRES_HOST or "db" }}"
-    port: "{{ POSTGRES_PORT or "5432" }}"
-    cp_min: 5
-    cp_max: 10
-{% else %}
-database:
-  name: "sqlite3"
-  args:
-    database: "/data/homeserver.db"
-{% endif %}
-
-## Performance ##
-
-event_cache_size: "{{ SYNAPSE_EVENT_CACHE_SIZE or "10K" }}"
-
-## Ratelimiting ##
-
-rc_messages_per_second: 0.2
-rc_message_burst_count: 10.0
-federation_rc_window_size: 1000
-federation_rc_sleep_limit: 10
-federation_rc_sleep_delay: 500
-federation_rc_reject_limit: 50
-federation_rc_concurrent: 3
-
-## Files ##
-
-media_store_path: "/data/media"
-uploads_path: "/data/uploads"
-max_upload_size: "{{ SYNAPSE_MAX_UPLOAD_SIZE or "10M" }}"
-max_image_pixels: "32M"
-dynamic_thumbnails: false
-
-# List of thumbnail to precalculate when an image is uploaded.
-thumbnail_sizes:
- width: 32
-  height: 32
-  method: crop
- width: 96
-  height: 96
-  method: crop
- width: 320
-  height: 240
-  method: scale
- width: 640
-  height: 480
-  method: scale
- width: 800
-  height: 600
-  method: scale
-
-url_preview_enabled: False
-max_spider_size: "10M"
-
-## Captcha ##
-
-{% if SYNAPSE_RECAPTCHA_PUBLIC_KEY %}
-recaptcha_public_key: "{{ SYNAPSE_RECAPTCHA_PUBLIC_KEY }}"
-recaptcha_private_key: "{{ SYNAPSE_RECAPTCHA_PRIVATE_KEY }}"
-enable_registration_captcha: True
-recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify"
-{% else %}
-recaptcha_public_key: "YOUR_PUBLIC_KEY"
-recaptcha_private_key: "YOUR_PRIVATE_KEY"
-enable_registration_captcha: False
-recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify"
-{% endif %}
-
-## Turn ##
-
-{% if SYNAPSE_TURN_URIS %}
-turn_uris:
-{% for uri in SYNAPSE_TURN_URIS.split(',') %}    - "{{ uri }}"
-{% endfor %}
-turn_shared_secret: "{{ SYNAPSE_TURN_SECRET }}"
-turn_user_lifetime: "1h"
-turn_allow_guests: True
-{% else %}
-turn_uris: []
-turn_shared_secret: "YOUR_SHARED_SECRET"
-turn_user_lifetime: "1h"
-turn_allow_guests: True
-{% endif %}
-
-## Registration ##
-
-enable_registration: {{ "True" if SYNAPSE_ENABLE_REGISTRATION else "False" }}
-registration_shared_secret: "{{ SYNAPSE_REGISTRATION_SHARED_SECRET }}"
-bcrypt_rounds: 12
-allow_guest_access: {{ "True" if SYNAPSE_ALLOW_GUEST else "False" }}
-enable_group_creation: true
-
-# The list of identity servers trusted to verify third party
-# identifiers by this server.
-#
-# Also defines the ID server which will be called when an account is
-# deactivated (one will be picked arbitrarily).
-trusted_third_party_id_servers:
-    - matrix.org
-    - vector.im
-
-## Metrics ###
-
-{% if SYNAPSE_REPORT_STATS.lower() == "yes" %}
-enable_metrics: True
-report_stats: True
-{% else %}
-enable_metrics: False
-report_stats: False
-{% endif %}
-
-## API Configuration ##
-
-room_invite_state_types:
-    - "m.room.join_rules"
-    - "m.room.canonical_alias"
-    - "m.room.avatar"
-    - "m.room.name"
-
-{% if SYNAPSE_APPSERVICES %}
-app_service_config_files:
-{% for appservice in SYNAPSE_APPSERVICES %}    - "{{ appservice }}"
-{% endfor %}
-{% else %}
-app_service_config_files: []
-{% endif %}
-
-macaroon_secret_key: "{{ SYNAPSE_MACAROON_SECRET_KEY }}"
-expire_access_token: False
-
-## Signing Keys ##
-
-signing_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.signing.key"
-old_signing_keys: {}
-key_refresh_interval: "1d" # 1 Day.
-
-# The trusted servers to download signing keys from.
-perspectives:
-  servers:
-    "matrix.org":
-      verify_keys:
-        "ed25519:auto":
-          key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"
-
-password_config:
-   enabled: true
-
-{% if SYNAPSE_SMTP_HOST %}
-email:
-   enable_notifs: false
-   smtp_host: "{{ SYNAPSE_SMTP_HOST }}"
-   smtp_port: {{ SYNAPSE_SMTP_PORT or "25" }}
-   smtp_user: "{{ SYNAPSE_SMTP_USER }}"
-   smtp_pass: "{{ SYNAPSE_SMTP_PASSWORD }}"
-   require_transport_security: False
-   notif_from: "{{ SYNAPSE_SMTP_FROM or "hostmaster@" + SYNAPSE_SERVER_NAME }}"
-   app_name: Matrix
-   # if template_dir is unset, uses the example templates that are part of
-   # the Synapse distribution.
-   #template_dir: res/templates
-   notif_template_html: notif_mail.html
-   notif_template_text: notif_mail.txt
-   notif_for_new_users: True
-   riot_base_url: "https://{{ SYNAPSE_SERVER_NAME }}"
-{% endif %}
@@ -1,29 +0,0 @@
-version: 1
-
-formatters:
-  precise:
-   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s'
-
-filters:
-  context:
-    (): synapse.util.logcontext.LoggingContextFilter
-    request: ""
-
-handlers:
-  console:
-    class: logging.StreamHandler
-    formatter: precise
-    filters: [context]
-
-loggers:
-    synapse:
-        level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }}
-
-    synapse.storage.SQL:
-        # beware: increasing this to DEBUG will make synapse log sensitive
-        # information such as access tokens.
-        level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }}
-
-root:
-    level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }}
-    handlers: [console]
@@ -1,20 +0,0 @@
-#!/bin/bash
-
-# This script runs the PostgreSQL tests inside a Docker container. It expects
-# the relevant source files to be mounted into /src (done automatically by the
-# caller script). It will set up the database, run it, and then use the tox
-# configuration to run the tests.
-
-set -e
-
-# Set PGUSER so Synapse's tests know what user to connect to the database with
-export PGUSER=postgres
-
-# Initialise & start the database
-su -c '/usr/lib/postgresql/9.6/bin/initdb -D /var/lib/postgresql/data -E "UTF-8" --lc-collate="en_US.UTF-8" --lc-ctype="en_US.UTF-8" --username=postgres' postgres
-su -c '/usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres
-
-# Run the tests
-cd /src
-export TRIAL_FLAGS="-j 4"
-tox --workdir=/tmp -e py27-postgres
@@ -1,67 +0,0 @@
-#!/usr/local/bin/python
-
-import jinja2
-import os
-import sys
-import subprocess
-import glob
-import codecs
-
-# Utility functions
-convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ))
-
-def check_arguments(environ, args):
-    for argument in args:
-        if argument not in environ:
-            print("Environment variable %s is mandatory, exiting." % argument)
-            sys.exit(2)
-
-def generate_secrets(environ, secrets):
-    for name, secret in secrets.items():
-        if secret not in environ:
-            filename = "/data/%s.%s.key" % (environ["SYNAPSE_SERVER_NAME"], name)
-            if os.path.exists(filename):
-                with open(filename) as handle: value = handle.read()
-            else:
-                print("Generating a random secret for {}".format(name))
-                value = codecs.encode(os.urandom(32), "hex").decode()
-                with open(filename, "w") as handle: handle.write(value)
-            environ[secret] = value
-
-# Prepare the configuration
-mode = sys.argv[1] if len(sys.argv) > 1 else None
-environ = os.environ.copy()
-ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991))
-args = ["python", "-m", "synapse.app.homeserver"]
-
-# In generate mode, generate a configuration, missing keys, then exit
-if mode == "generate":
-    check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_CONFIG_PATH"))
-    args += [
-        "--server-name", environ["SYNAPSE_SERVER_NAME"],
-        "--report-stats", environ["SYNAPSE_REPORT_STATS"],
-        "--config-path", environ["SYNAPSE_CONFIG_PATH"],
-        "--generate-config"
-    ]
-    os.execv("/usr/local/bin/python", args)
-
-# In normal mode, generate missing keys if any, then run synapse
-else:
-    # Parse the configuration file
-    if "SYNAPSE_CONFIG_PATH" in environ:
-        args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]]
-    else:
-        check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"))
-        generate_secrets(environ, {
-            "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET",
-            "macaroon": "SYNAPSE_MACAROON_SECRET_KEY"
-        })
-        environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml")
-        if not os.path.exists("/compiled"): os.mkdir("/compiled")
-        convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ)
-        convert("/conf/log.config", "/compiled/log.config", environ)
-        subprocess.check_output(["chown", "-R", ownership, "/data"])
-        args += ["--config-path", "/compiled/homeserver.yaml"]
-    # Generate missing keys and start synapse
-    subprocess.check_output(args + ["--generate-keys"])
-    os.execv("/sbin/su-exec", ["su-exec", ownership] + args)
@@ -61,11 +61,3 @@ the following:
    }

 The status will be one of ``active``, ``complete``, or ``failed``.
-
-Reclaim disk space (Postgres)
-----------------------------
-
-To reclaim the disk space and return it to the operating system, you need to run
-`VACUUM FULL;` on the database.
-
-https://www.postgresql.org/docs/current/sql-vacuum.html
@@ -1,63 +0,0 @@
-Shared-Secret Registration
-==========================
-
-This API allows for the creation of users in an administrative and
-non-interactive way. This is generally used for bootstrapping a Synapse
-instance with administrator accounts.
-
-To authenticate yourself to the server, you will need both the shared secret
-(``registration_shared_secret`` in the homeserver configuration), and a
-one-time nonce. If the registration shared secret is not configured, this API
-is not enabled.
-
-To fetch the nonce, you need to request one from the API::
-
-  > GET /_matrix/client/r0/admin/register
-
-  < {"nonce": "thisisanonce"}
-
-Once you have the nonce, you can make a ``POST`` to the same URL with a JSON
-body containing the nonce, username, password, whether they are an admin
-(optional, False by default), and a HMAC digest of the content.
-
-As an example::
-
-  > POST /_matrix/client/r0/admin/register
-  > {
-     "nonce": "thisisanonce",
-     "username": "pepper_roni",
-     "password": "pizza",
-     "admin": true,
-     "mac": "mac_digest_here"
-    }
-
-  < {
-     "access_token": "token_here",
-     "user_id": "@pepper_roni:localhost",
-     "home_server": "test",
-     "device_id": "device_id_here"
-    }
-
-The MAC is the hex digest output of the HMAC-SHA1 algorithm, with the key being
-the shared secret and the content being the nonce, user, password, and either
-the string "admin" or "notadmin", each separated by NULs. For an example of
-generation in Python::
-
-  import hmac, hashlib
-
-  def generate_mac(nonce, user, password, admin=False):
-
-      mac = hmac.new(
-        key=shared_secret,
-        digestmod=hashlib.sha1,
-      )
-
-      mac.update(nonce.encode('utf8'))
-      mac.update(b"\x00")
-      mac.update(user.encode('utf8'))
-      mac.update(b"\x00")
-      mac.update(password.encode('utf8'))
-      mac.update(b"\x00")
-      mac.update(b"admin" if admin else b"notadmin")
-
-      return mac.hexdigest()
@@ -44,26 +44,13 @@ Deactivate Account

 This API deactivates an account. It removes active access tokens, resets the
 password, and deletes third-party IDs (to prevent the user requesting a
-password reset). It can also mark the user as GDPR-erased (stopping their data
-from distributed further, and deleting it entirely if there are no other
-references to it).
+password reset).

 The api is::

    POST /_matrix/client/r0/admin/deactivate/<user_id>

-with a body of:
-
-.. code:: json
-
-    {
-        "erase": true
-    }
-
-including an ``access_token`` of a server admin.
-
-The erase parameter is optional and defaults to 'false'.
-An empty body may be passed for backwards compatibility.
+including an ``access_token`` of a server admin, and an empty request body.


 Reset password
@@ -16,7 +16,7 @@
      print("I am a fish %s" %
            "moo")

-    and this::
+      and this::

        print(
            "I am a fish %s" %
@@ -1,197 +0,0 @@
-Support in Synapse for tracking agreement to server terms and conditions
-========================================================================
-
-Synapse 0.30 introduces support for tracking whether users have agreed to the
-terms and conditions set by the administrator of a server - and blocking access
-to the server until they have.
-
-There are several parts to this functionality; each requires some specific
-configuration in `homeserver.yaml` to be enabled.
-
-Note that various parts of the configuation and this document refer to the
-"privacy policy": agreement with a privacy policy is one particular use of this
-feature, but of course adminstrators can specify other terms and conditions
-unrelated to "privacy" per se.
-
-Collecting policy agreement from a user
---------------------------------------
-
-Synapse can be configured to serve the user a simple policy form with an
-"accept" button. Clicking "Accept" records the user's acceptance in the
-database and shows a success page.
-
-To enable this, first create templates for the policy and success pages.
-These should be stored on the local filesystem.
-
-These templates use the [Jinja2](http://jinja.pocoo.org) templating language,
-and [docs/privacy_policy_templates](privacy_policy_templates) gives
-examples of the sort of thing that can be done.
-
-Note that the templates must be stored under a name giving the language of the
-template - currently this must always be `en` (for "English");
-internationalisation support is intended for the future.
-
-The template for the policy itself should be versioned and named according to
-the version: for example `1.0.html`. The version of the policy which the user
-has agreed to is stored in the database.
-
-Once the templates are in place, make the following changes to `homeserver.yaml`:
-
- 1. Add a `user_consent` section, which should look like:
-
-    ```yaml
-    user_consent:
-      template_dir: privacy_policy_templates
-      version: 1.0
-    ```
-
-    `template_dir` points to the directory containing the policy
-    templates. `version` defines the version of the policy which will be served
-    to the user. In the example above, Synapse will serve
-    `privacy_policy_templates/en/1.0.html`.
-
-
- 2. Add a `form_secret` setting at the top level:
-
-
-    ```yaml
-    form_secret: "<unique secret>"
-    ```
-
-    This should be set to an arbitrary secret string (try `pwgen -y 30` to
-    generate suitable secrets).
-
-    More on what this is used for below.
-
- 3. Add `consent` wherever the `client` resource is currently enabled in the
-    `listeners` configuration. For example:
-
-    ```yaml
-    listeners:
-      - port: 8008
-        resources:
-          - names:
-            - client
-            - consent
-    ```
-
-
-Finally, ensure that `jinja2` is installed. If you are using a virtualenv, this
-should be a matter of `pip install Jinja2`. On debian, try `apt-get install
-python-jinja2`.
-
-Once this is complete, and the server has been restarted, try visiting
-`https://<server>/_matrix/consent`. If correctly configured, this should give
-an error "Missing string query parameter 'u'". It is now possible to manually
-construct URIs where users can give their consent.
-
-### Enabling consent tracking at registration
-
-1. Add the following to your configuration:
-
-   ```yaml
-   user_consent:
-     require_at_registration: true
-     policy_name: "Privacy Policy" # or whatever you'd like to call the policy
-   ```
-
-2. In your consent templates, make use of the `public_version` variable to
-   see if an unauthenticated user is viewing the page. This is typically
-   wrapped around the form that would be used to actually agree to the document:
-
-   ```
-   {% if not public_version %}
-     <!-- The variables used here are only provided when the 'u' param is given to the homeserver -->
-     <form method="post" action="consent">
-       <input type="hidden" name="v" value="{{version}}"/>
-       <input type="hidden" name="u" value="{{user}}"/>
-       <input type="hidden" name="h" value="{{userhmac}}"/>
-       <input type="submit" value="Sure thing!"/>
-     </form>
-   {% endif %}
-   ```
-
-3. Restart Synapse to apply the changes.
-
-Visiting `https://<server>/_matrix/consent` should now give you a view of the privacy
-document. This is what users will be able to see when registering for accounts.
-
-### Constructing the consent URI
-
-It may be useful to manually construct the "consent URI" for a given user - for
-instance, in order to send them an email asking them to consent. To do this,
-take the base `https://<server>/_matrix/consent` URL and add the following
-query parameters:
-
- * `u`: the user id of the user. This can either be a full MXID
-   (`@user:server.com`) or just the localpart (`user`).
-
- * `h`: hex-encoded HMAC-SHA256 of `u` using the `form_secret` as a key. It is
-   possible to calculate this on the commandline with something like:
-
-   ```bash
-   echo -n '<user>' | openssl sha256 -hmac '<form_secret>'
-   ```
-
-   This should result in a URI which looks something like:
-   `https://<server>/_matrix/consent?u=<user>&h=68a152465a4d...`.
-
-
-Note that not providing a `u` parameter will be interpreted as wanting to view
-the document from an unauthenticated perspective, such as prior to registration.
-Therefore, the `h` parameter is not required in this scenario. To enable this
-behaviour, set `require_at_registration` to `true` in your `user_consent` config.
-
-
-Sending users a server notice asking them to agree to the policy
----------------------------------------------------------------
-
-It is possible to configure Synapse to send a [server
-notice](server_notices.md) to anybody who has not yet agreed to the current
-version of the policy. To do so:
-
- * ensure that the consent resource is configured, as in the previous section
-
- * ensure that server notices are configured, as in [server_notices.md](server_notices.md).
-
- * Add `server_notice_content` under `user_consent` in `homeserver.yaml`. For
-   example:
-
-   ```yaml
-   user_consent:
-     server_notice_content:
-       msgtype: m.text
-       body: >-
-         Please give your consent to the privacy policy at %(consent_uri)s.
-   ```
-
-   Synapse automatically replaces the placeholder `%(consent_uri)s` with the
-   consent uri for that user.
-
- * ensure that `public_baseurl` is set in `homeserver.yaml`, and gives the base
-   URI that clients use to connect to the server. (It is used to construct
-   `consent_uri` in the server notice.)
-
-
-Blocking users from using the server until they agree to the policy
-------------------------------------------------------------------
-
-Synapse can be configured to block any attempts to join rooms or send messages
-until the user has given their agreement to the policy. (Joining the server
-notices room is exempted from this).
-
-To enable this, add `block_events_error` under `user_consent`. For example:
-
-```yaml
-user_consent:
-  block_events_error: >-
-    You can't send any messages until you consent to the privacy policy at
-    %(consent_uri)s.
-```
-
-Synapse automatically replaces the placeholder `%(consent_uri)s` with the
-consent uri for that user.
-
-ensure that `public_baseurl` is set in `homeserver.yaml`, and gives the base
-URI that clients use to connect to the server. (It is used to construct
-`consent_uri` in the error.)
@@ -163,7 +163,7 @@ the logcontext was set, this will make things work out ok: provided
 It's all too easy to forget to ``yield``: for instance if we forgot that
 ``do_some_stuff`` returned a deferred, we might plough on regardless. This
 leads to a mess; it will probably work itself out eventually, but not before
-a load of stuff has been logged against the wrong context. (Normally, other
+a load of stuff has been logged against the wrong content. (Normally, other
 things will break, more obviously, if you forget to ``yield``, so this tends
 not to be a major problem in practice.)

@@ -440,59 +440,3 @@ To conclude: I think this scheme would have worked equally well, with less
 danger of messing it up, and probably made some more esoteric code easier to
 write. But again — changing the conventions of the entire Synapse codebase is
 not a sensible option for the marginal improvement offered.
-
-
-A note on garbage-collection of Deferred chains
-----------------------------------------------
-
-It turns out that our logcontext rules do not play nicely with Deferred
-chains which get orphaned and garbage-collected.
-
-Imagine we have some code that looks like this:
-
-.. code:: python
-
-    listener_queue = []
-
-    def on_something_interesting():
-        for d in listener_queue:
-            d.callback("foo")
-
-    @defer.inlineCallbacks
-    def await_something_interesting():
-        new_deferred = defer.Deferred()
-        listener_queue.append(new_deferred)
-
-        with PreserveLoggingContext():
-            yield new_deferred
-
-Obviously, the idea here is that we have a bunch of things which are waiting
-for an event. (It's just an example of the problem here, but a relatively
-common one.)
-
-Now let's imagine two further things happen. First of all, whatever was
-waiting for the interesting thing goes away. (Perhaps the request times out,
-or something *even more* interesting happens.)
-
-Secondly, let's suppose that we decide that the interesting thing is never
-going to happen, and we reset the listener queue:
-
-.. code:: python
-
-    def reset_listener_queue():
-        listener_queue.clear()
-
-So, both ends of the deferred chain have now dropped their references, and the
-deferred chain is now orphaned, and will be garbage-collected at some point.
-Note that ``await_something_interesting`` is a generator function, and when
-Python garbage-collects generator functions, it gives them a chance to clean
-up by making the ``yield`` raise a ``GeneratorExit`` exception. In our case,
-that means that the ``__exit__`` handler of ``PreserveLoggingContext`` will
-carefully restore the request context, but there is now nothing waiting for
-its return, so the request context is never cleared.
-
-To reiterate, this problem only arises when *both* ends of a deferred chain
-are dropped. Dropping the the reference to a deferred you're supposed to be
-calling is probably bad practice, so this doesn't actually happen too much.
-Unfortunately, when it does happen, it will lead to leaked logcontexts which
-are incredibly hard to track down.
@@ -1,43 +0,0 @@
-Using the synapse manhole
-=========================
-
-The "manhole" allows server administrators to access a Python shell on a running
-Synapse installation. This is a very powerful mechanism for administration and
-debugging.
-
-To enable it, first uncomment the `manhole` listener configuration in
-`homeserver.yaml`:
-
-```yaml
-listeners:
-  - port: 9000
-    bind_addresses: ['::1', '127.0.0.1']
-    type: manhole
-```
-
-(`bind_addresses` in the above is important: it ensures that access to the
-manhole is only possible for local users).
-
-Note that this will give administrative access to synapse to **all users** with
-shell access to the server. It should therefore **not** be enabled in
-environments where untrusted users have shell access.
-
-Then restart synapse, and point an ssh client at port 9000 on localhost, using
-the username `matrix`:
-
-```bash
-ssh -p9000 matrix@localhost
-```
-
-The password is `rabbithole`.
-
-This gives a Python REPL in which `hs` gives access to the
-`synapse.server.HomeServer` object - which in turn gives access to many other
-parts of the process.
-
-As a simple example, retrieving an event from the database:
-
-```
->>> hs.get_datastore().get_event('$1416420717069yeQaw:matrix.org')
-<Deferred at 0x7ff253fc6998 current result: <FrozenEvent event_id='$1416420717069yeQaw:matrix.org', type='m.room.create', state_key=''>>
-```
@@ -1,47 +1,25 @@
 How to monitor Synapse metrics using Prometheus
 ===============================================

-1. Install Prometheus:
+1. Install prometheus:

   Follow instructions at http://prometheus.io/docs/introduction/install/

-2. Enable Synapse metrics:
+2. Enable synapse metrics:

-   There are two methods of enabling metrics in Synapse.
+   Simply setting a (local) port number will enable it. Pick a port.
+   prometheus itself defaults to 9090, so starting just above that for
+   locally monitored services seems reasonable. E.g. 9092:

-   The first serves the metrics as a part of the usual web server and can be
-   enabled by adding the "metrics" resource to the existing listener as such::
+   Add to homeserver.yaml::

-     resources:
-       - names:
-         - client
-         - metrics
+     metrics_port: 9092

-   This provides a simple way of adding metrics to your Synapse installation,
-   and serves under ``/_synapse/metrics``. If you do not wish your metrics be
-   publicly exposed, you will need to either filter it out at your load
-   balancer, or use the second method.
+   Also ensure that ``enable_metrics`` is set to ``True``.

-   The second method runs the metrics server on a different port, in a
-   different thread to Synapse. This can make it more resilient to heavy load
-   meaning metrics cannot be retrieved, and can be exposed to just internal
-   networks easier. The served metrics are available over HTTP only, and will
-   be available at ``/``.
+   Restart synapse.

-   Add a new listener to homeserver.yaml::
-
-     listeners:
-       - type: metrics
-         port: 9000
-         bind_addresses:
-           - '0.0.0.0'
-
-   For both options, you will need to ensure that ``enable_metrics`` is set to
-   ``True``.
-
-   Restart Synapse.
-
-3. Add a Prometheus target for Synapse.
+3. Add a prometheus target for synapse.

   It needs to set the ``metrics_path`` to a non-default value (under ``scrape_configs``)::

@@ -53,50 +31,7 @@ How to monitor Synapse metrics using Prometheus
   If your prometheus is older than 1.5.2, you will need to replace
   ``static_configs`` in the above with ``target_groups``.

-   Restart Prometheus.
-
-
-Removal of deprecated metrics & time based counters becoming histograms in 0.31.0
---------------------------------------------------------------------------------
-
-The duplicated metrics deprecated in Synapse 0.27.0 have been removed.
-
-All time duration-based metrics have been changed to be seconds. This affects:
-
-+----------------------------------+
-| msec -> sec metrics              |
-+==================================+
-| python_gc_time                   |
-+----------------------------------+
-| python_twisted_reactor_tick_time |
-+----------------------------------+
-| synapse_storage_query_time       |
-+----------------------------------+
-| synapse_storage_schedule_time    |
-+----------------------------------+
-| synapse_storage_transaction_time |
-+----------------------------------+
-
-Several metrics have been changed to be histograms, which sort entries into
-buckets and allow better analysis. The following metrics are now histograms:
-
-+-------------------------------------------+
-| Altered metrics                           |
-+===========================================+
-| python_gc_time                            |
-+-------------------------------------------+
-| python_twisted_reactor_pending_calls      |
-+-------------------------------------------+
-| python_twisted_reactor_tick_time          |
-+-------------------------------------------+
-| synapse_http_server_response_time_seconds |
-+-------------------------------------------+
-| synapse_storage_query_time                |
-+-------------------------------------------+
-| synapse_storage_schedule_time             |
-+-------------------------------------------+
-| synapse_storage_transaction_time          |
-+-------------------------------------------+
+   Restart prometheus.


 Block and response metrics renamed for 0.27.0
@@ -6,22 +6,16 @@ Postgres version 9.4 or later is known to work.
 Set up database
 ===============

-Assuming your PostgreSQL database user is called ``postgres``, create a user
-``synapse_user`` with::
-
-   su - postgres
-   createuser --pwprompt synapse_user
-
-The PostgreSQL database used *must* have the correct encoding set, otherwise it
+The PostgreSQL database used *must* have the correct encoding set, otherwise
 would not be able to store UTF8 strings. To create a database with the correct
 encoding use, e.g.::

-   CREATE DATABASE synapse
-    ENCODING 'UTF8'
-    LC_COLLATE='C'
-    LC_CTYPE='C'
-    template=template0
-    OWNER synapse_user;
+ CREATE DATABASE synapse
+  ENCODING 'UTF8'
+  LC_COLLATE='C'
+  LC_CTYPE='C'
+  template=template0
+  OWNER synapse_user;

 This would create an appropriate database named ``synapse`` owned by the
 ``synapse_user`` user (which must already exist).
@@ -52,8 +46,8 @@ As with Debian/Ubuntu, postgres support depends on the postgres python connector
 Synapse config
 ==============

-When you are ready to start using PostgreSQL, edit the ``database`` section in
-your config file to match the following lines::
+When you are ready to start using PostgreSQL, add the following line to your
+config file::

    database:
        name: psycopg2
@@ -102,12 +96,9 @@ complete, restart synapse.  For instance::
    cp homeserver.db homeserver.db.snapshot
    ./synctl start

-Copy the old config file into a new config file::
-
-    cp homeserver.yaml homeserver-postgres.yaml
-
-Edit the database section as described in the section *Synapse config* above
-and with the SQLite snapshot located at ``homeserver.db.snapshot`` simply run::
+Assuming your new config file (as described in the section *Synapse config*)
+is named ``homeserver-postgres.yaml`` and the SQLite snapshot is at
+``homeserver.db.snapshot`` then simply run::

    synapse_port_db --sqlite-database homeserver.db.snapshot \
        --postgres-config homeserver-postgres.yaml
@@ -126,11 +117,6 @@ run::
        --postgres-config homeserver-postgres.yaml

 Once that has completed, change the synapse config to point at the PostgreSQL
-database configuration file ``homeserver-postgres.yaml``::
-
-    ./synctl stop
-    mv homeserver.yaml homeserver-old-sqlite.yaml 
-    mv homeserver-postgres.yaml homeserver.yaml 
-    ./synctl start
-
-Synapse should now be running against PostgreSQL.
+database configuration file ``homeserver-postgres.yaml`` (i.e. rename it to 
+``homeserver.yaml``) and restart synapse. Synapse should now be running against
+PostgreSQL.
@@ -1,26 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <title>Matrix.org Privacy policy</title>
-  </head>
-  <body>
-  {% if has_consented %}
-    <p>
-      Your base already belong to us.
-    </p>
-  {% else %}
-    <p>
-      All your base are belong to us.
-    </p>
-    {% if not public_version %}
-      <!-- The variables used here are only provided when the 'u' param is given to the homeserver -->
-      <form method="post" action="consent">
-        <input type="hidden" name="v" value="{{version}}"/>
-        <input type="hidden" name="u" value="{{user}}"/>
-        <input type="hidden" name="h" value="{{userhmac}}"/>
-        <input type="submit" value="Sure thing!"/>
-      </form>
-    {% endif %}
-  {% endif %}
-  </body>
-</html>
@@ -1,11 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <title>Matrix.org Privacy policy</title>
-  </head>
-  <body>
-    <p>
-      Sweet.
-    </p>
-  </body>
-</html>
@@ -1,74 +0,0 @@
-Server Notices
-==============
-
-'Server Notices' are a new feature introduced in Synapse 0.30. They provide a
-channel whereby server administrators can send messages to users on the server.
-
-They are used as part of communication of the server polices(see
-[consent_tracking.md](consent_tracking.md)), however the intention is that
-they may also find a use for features such as "Message of the day".
-
-This is a feature specific to Synapse, but it uses standard Matrix
-communication mechanisms, so should work with any Matrix client.
-
-User experience
---------------
-
-When the user is first sent a server notice, they will get an invitation to a
-room (typically called 'Server Notices', though this is configurable in
-`homeserver.yaml`). They will be **unable to reject** this invitation -
-attempts to do so will receive an error.
-
-Once they accept the invitation, they will see the notice message in the room
-history; it will appear to have come from the 'server notices user' (see
-below).
-
-The user is prevented from sending any messages in this room by the power
-levels.
-
-Having joined the room, the user can leave the room if they want. Subsequent
-server notices will then cause a new room to be created.
-
-Synapse configuration
---------------------
-
-Server notices come from a specific user id on the server. Server
-administrators are free to choose the user id - something like `server` is
-suggested, meaning the notices will come from
-`@server:<your_server_name>`. Once the Server Notices user is configured, that
-user id becomes a special, privileged user, so administrators should ensure
-that **it is not already allocated**.
-
-In order to support server notices, it is necessary to add some configuration
-to the `homeserver.yaml` file. In particular, you should add a `server_notices`
-section, which should look like this:
-
-```yaml
-server_notices:
-   system_mxid_localpart: server
-   system_mxid_display_name: "Server Notices"
-   system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
-   room_name: "Server Notices"
-```
-
-The only compulsory setting is `system_mxid_localpart`, which defines the user
-id of the Server Notices user, as above. `room_name` defines the name of the
-room which will be created.
-
-`system_mxid_display_name` and `system_mxid_avatar_url` can be used to set the
-displayname and avatar of the Server Notices user.
-
-Sending notices
---------------
-
-As of the current version of synapse, there is no convenient interface for
-sending notices (other than the automated ones sent as part of consent
-tracking).
-
-In the meantime, it is possible to test this feature using the manhole. Having
-gone into the manhole as described in [manhole.md](manhole.md), a notice can be
-sent with something like:
-
-```
->>> hs.get_server_notices_manager().send_notice('@user:server.com', {'msgtype':'m.text', 'body':'foo'})
-```
@@ -74,7 +74,7 @@ replication endpoints that it's talking to on the main synapse process.
 ``worker_replication_port`` should point to the TCP replication listener port and
 ``worker_replication_http_port`` should point to the HTTP replication port.

-Currently, the ``event_creator`` and ``federation_reader`` workers require specifying
+Currently, only the ``event_creator`` worker requires specifying
 ``worker_replication_http_port``.

 For instance::
@@ -173,23 +173,10 @@ endpoints matching the following regular expressions::
    ^/_matrix/federation/v1/backfill/
    ^/_matrix/federation/v1/get_missing_events/
    ^/_matrix/federation/v1/publicRooms
-    ^/_matrix/federation/v1/query/
-    ^/_matrix/federation/v1/make_join/
-    ^/_matrix/federation/v1/make_leave/
-    ^/_matrix/federation/v1/send_join/
-    ^/_matrix/federation/v1/send_leave/
-    ^/_matrix/federation/v1/invite/
-    ^/_matrix/federation/v1/query_auth/
-    ^/_matrix/federation/v1/event_auth/
-    ^/_matrix/federation/v1/exchange_third_party_invite/
-    ^/_matrix/federation/v1/send/

 The above endpoints should all be routed to the federation_reader worker by the
 reverse-proxy configuration.

-The `^/_matrix/federation/v1/send/` endpoint must only be handled by a single
-instance.
-
 ``synapse.app.federation_sender``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -219,10 +206,6 @@ Handles client API endpoints. It can handle REST endpoints matching the
 following regular expressions::

    ^/_matrix/client/(api/v1|r0|unstable)/publicRooms$
-    ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/joined_members$
-    ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/context/.*$
-    ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$
-    ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$

 ``synapse.app.user_dir``
 ~~~~~~~~~~~~~~~~~~~~~~~~
@@ -241,14 +224,6 @@ regular expressions::

    ^/_matrix/client/(api/v1|r0|unstable)/keys/upload

-If ``use_presence`` is False in the homeserver config, it can also handle REST
-endpoints matching the following regular expressions::
-
-    ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status
-
-This "stub" presence handler will pass through ``GET`` request but make the
-``PUT`` effectively a no-op.
-
 It will proxy any requests it cannot handle to the main synapse instance. It
 must therefore be configured with the location of the main instance, via
 the ``worker_main_http_uri`` setting in the frontend_proxy worker configuration
@@ -265,7 +240,6 @@ Handles some event creation. It can handle REST endpoints matching::
    ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send
    ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$
    ^/_matrix/client/(api/v1|r0|unstable)/join/
-    ^/_matrix/client/(api/v1|r0|unstable)/profile/

 It will create events locally and then send them on to the main synapse
 instance to be persisted and handled.
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -eux
+
+: ${WORKSPACE:="$(pwd)"}
+
+export WORKSPACE
+export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
+
+export HAPROXY_BIN=/home/haproxy/haproxy-1.6.11/haproxy
+
+./jenkins/prepare_synapse.sh
+./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git
+./jenkins/clone.sh dendron https://github.com/matrix-org/dendron.git
+./dendron/jenkins/build_dendron.sh
+./sytest/jenkins/prep_sytest_for_postgres.sh
+
+./sytest/jenkins/install_and_run.sh \
+    --python $WORKSPACE/.tox/py27/bin/python \
+    --synapse-directory $WORKSPACE \
+    --dendron $WORKSPACE/dendron/bin/dendron \
+    --haproxy \
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -eux
+
+: ${WORKSPACE:="$(pwd)"}
+
+export WORKSPACE
+export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
+
+./jenkins/prepare_synapse.sh
+./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git
+./jenkins/clone.sh dendron https://github.com/matrix-org/dendron.git
+./dendron/jenkins/build_dendron.sh
+./sytest/jenkins/prep_sytest_for_postgres.sh
+
+./sytest/jenkins/install_and_run.sh \
+    --python $WORKSPACE/.tox/py27/bin/python \
+    --synapse-directory $WORKSPACE \
+    --dendron $WORKSPACE/dendron/bin/dendron \
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -eux
+
+: ${WORKSPACE:="$(pwd)"}
+
+export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
+
+# Output test results as junit xml
+export TRIAL_FLAGS="--reporter=subunit"
+export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml"
+# Write coverage reports to a separate file for each process
+export COVERAGE_OPTS="-p"
+export DUMP_COVERAGE_COMMAND="coverage help"
+
+# Output flake8 violations to violations.flake8.log
+export PEP8SUFFIX="--output-file=violations.flake8.log"
+
+rm .coverage* || echo "No coverage files to remove"
+
+tox -e packaging -e pep8
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -eux
+
+: ${WORKSPACE:="$(pwd)"}
+
+export WORKSPACE
+export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
+
+./jenkins/prepare_synapse.sh
+./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git
+
+./sytest/jenkins/prep_sytest_for_postgres.sh
+
+./sytest/jenkins/install_and_run.sh \
+    --python $WORKSPACE/.tox/py27/bin/python \
+    --synapse-directory $WORKSPACE \
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -eux
+
+: ${WORKSPACE:="$(pwd)"}
+
+export WORKSPACE
+export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
+
+./jenkins/prepare_synapse.sh
+./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git
+
+./sytest/jenkins/install_and_run.sh \
+    --python $WORKSPACE/.tox/py27/bin/python \
+    --synapse-directory $WORKSPACE \
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -eux
+
+: ${WORKSPACE:="$(pwd)"}
+
+export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
+
+# Output test results as junit xml
+export TRIAL_FLAGS="--reporter=subunit"
+export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml"
+# Write coverage reports to a separate file for each process
+export COVERAGE_OPTS="-p"
+export DUMP_COVERAGE_COMMAND="coverage help"
+
+# Output flake8 violations to violations.flake8.log
+# Don't exit with non-0 status code on Jenkins,
+# so that the build steps continue and a later step can decided whether to
+# UNSTABLE or FAILURE this build.
+export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?"
+
+rm .coverage* || echo "No coverage files to remove"
+
+tox --notest -e py27
+TOX_BIN=$WORKSPACE/.tox/py27/bin
+python synapse/python_dependencies.py | xargs -n1 $TOX_BIN/pip install
+$TOX_BIN/pip install lxml
+
+tox -e py27
@@ -0,0 +1,44 @@
+#! /bin/bash
+
+# This clones a project from github into a named subdirectory
+# If the project has a branch with the same name as this branch
+# then it will checkout that branch after cloning.
+# Otherwise it will checkout "origin/develop."
+# The first argument is the name of the directory to checkout
+# the branch into.
+# The second argument is the URL of the remote repository to checkout.
+# Usually something like https://github.com/matrix-org/sytest.git
+
+set -eux
+
+NAME=$1
+PROJECT=$2
+BASE=".$NAME-base"
+
+# Update our mirror.
+if [ ! -d ".$NAME-base" ]; then
+  # Create a local mirror of the source repository.
+  # This saves us from having to download the entire repository
+  # when this script is next run.
+  git clone "$PROJECT" "$BASE" --mirror
+else
+  # Fetch any updates from the source repository.
+  (cd "$BASE"; git fetch -p)
+fi
+
+# Remove the existing repository so that we have a clean copy
+rm -rf "$NAME"
+# Cloning with --shared means that we will share portions of the
+# .git directory with our local mirror.
+git clone "$BASE" "$NAME" --shared
+
+# Jenkins may have supplied us with the name of the branch in the
+# environment. Otherwise we will have to guess based on the current
+# commit.
+: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"}
+cd "$NAME"
+# check out the relevant branch
+git checkout "${GIT_BRANCH}" || (
+    echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop"
+    git checkout "origin/develop"
+)
@@ -14,3 +14,22 @@ fi

 # set up the virtualenv
 tox -e py27 --notest -v
+
+TOX_BIN=$TOX_DIR/py27/bin
+
+# cryptography 2.2 requires setuptools >= 18.5.
+#
+# older versions of virtualenv (?) give us a virtualenv with the same version
+# of setuptools as is installed on the system python (and tox runs virtualenv
+# under python3, so we get the version of setuptools that is installed on that).
+#
+# anyway, make sure that we have a recent enough setuptools.
+$TOX_BIN/pip install 'setuptools>=18.5'
+
+# we also need a semi-recent version of pip, because old ones fail to install
+# the "enum34" dependency of cryptography.
+$TOX_BIN/pip install 'pip>=10'
+
+{ python synapse/python_dependencies.py
+  echo lxml psycopg2
+} | xargs $TOX_BIN/pip install
@@ -1,30 +0,0 @@
-[tool.towncrier]
-    package = "synapse"
-    filename = "CHANGES.md"
-    directory = "changelog.d"
-    issue_format = "[\\#{issue}](https://github.com/matrix-org/synapse/issues/{issue})"
-
-    [[tool.towncrier.type]]
-        directory = "feature"
-        name = "Features"
-        showcontent = true
-
-    [[tool.towncrier.type]]
-        directory = "bugfix"
-        name = "Bugfixes"
-        showcontent = true
-
-    [[tool.towncrier.type]]
-        directory = "doc"
-        name = "Improved Documentation"
-        showcontent = true
-
-    [[tool.towncrier.type]]
-        directory = "removal"
-        name = "Deprecations and Removals"
-        showcontent = true
-
-    [[tool.towncrier.type]]
-        directory = "misc"
-        name = "Internal Changes"
-        showcontent = true
--- a/Show More
+++ b/Show More
				`@@ -1 +0,0 @@`
				`Add option to track MAU stats (but not limit people)`
				`@@ -1 +0,0 @@`
				`Add an option to enable recording IPs for appservice users`
				`@@ -1 +0,0 @@`
				`Pushrules can now again be made with non-ASCII rule IDs.`
				`@@ -1 +0,0 @@`
				`The media repository now no longer fails to decode UTF-8 filenames when downloading remote media.`
				`@@ -1 +0,0 @@`
				`A coveragerc file, as well as the py36-coverage tox target, have been added.`
				`@@ -1 +0,0 @@`
				`Add a GitHub pull request template and add multiple issue templates`
				`@@ -1 +0,0 @@`
				URL previews now correctly decode non-UTF-8 text if the header contains a `<meta http-equiv="Content-Type"` header.
				`@@ -1 +0,0 @@`
				`Update README to reflect the fact that #1491 is fixed`
				`@@ -1,2 +0,0 @@`
				`Run the AS senders as background processes to fix warnings`
				`@@ -1 +0,0 @@`
				`Add some diagnostics to the tests to detect logcontext problems`