Remove stranded code

Remove old proxy binary
Catch room profile errors and anything else that can go wrong
2019-05-16 18:35:25 +01:00 · 2019-02-13 21:58:39 +00:00 · 2019-02-13 21:01:32 +00:00 · 2019-02-13 20:54:34 +00:00 · 2019-02-13 20:54:34 +00:00 · 2019-02-13 20:54:34 +00:00
178 changed files with 7303 additions and 2839 deletions
@@ -23,99 +23,106 @@ jobs:
      - run: docker push matrixdotorg/synapse:latest
      - run: docker push matrixdotorg/synapse:latest-py3
  sytestpy2:
-    machine: true
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
    steps:
      - checkout
-      - run: docker pull matrixdotorg/sytest-synapsepy2
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2
+      - run: /synapse_sytest.sh
      - store_artifacts:
-          path: ~/project/logs
+          path: /logs
          destination: logs
      - store_test_results:
-          path: logs
+          path: /logs
  sytestpy2postgres:
-    machine: true
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
    steps:
      - checkout
-      - run: docker pull matrixdotorg/sytest-synapsepy2
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2
+      - run: POSTGRES=1 /synapse_sytest.sh
      - store_artifacts:
-          path: ~/project/logs
+          path: /logs
          destination: logs
      - store_test_results:
-          path: logs
+          path: /logs
  sytestpy2merged:
-    machine: true
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
    steps:
      - checkout
      - run: bash .circleci/merge_base_branch.sh
-      - run: docker pull matrixdotorg/sytest-synapsepy2
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2
+      - run: /synapse_sytest.sh
      - store_artifacts:
-          path: ~/project/logs
+          path: /logs
          destination: logs
      - store_test_results:
-          path: logs
-
+          path: /logs
  sytestpy2postgresmerged:
-    machine: true
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
    steps:
      - checkout
      - run: bash .circleci/merge_base_branch.sh
-      - run: docker pull matrixdotorg/sytest-synapsepy2
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2
+      - run: POSTGRES=1 /synapse_sytest.sh
      - store_artifacts:
-          path: ~/project/logs
+          path: /logs
          destination: logs
      - store_test_results:
-          path: logs
+          path: /logs

  sytestpy3:
-    machine: true
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
    steps:
      - checkout
-      - run: docker pull matrixdotorg/sytest-synapsepy3
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
+      - run: /synapse_sytest.sh
      - store_artifacts:
-          path: ~/project/logs
+          path: /logs
          destination: logs
      - store_test_results:
-          path: logs
+          path: /logs
  sytestpy3postgres:
-    machine: true
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
    steps:
      - checkout
-      - run: docker pull matrixdotorg/sytest-synapsepy3
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3
+      - run: POSTGRES=1 /synapse_sytest.sh
      - store_artifacts:
-          path: ~/project/logs
+          path: /logs
          destination: logs
      - store_test_results:
-          path: logs
+          path: /logs
  sytestpy3merged:
-    machine: true
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
    steps:
      - checkout
      - run: bash .circleci/merge_base_branch.sh
-      - run: docker pull matrixdotorg/sytest-synapsepy3
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
+      - run: /synapse_sytest.sh
      - store_artifacts:
-          path: ~/project/logs
+          path: /logs
          destination: logs
      - store_test_results:
-          path: logs
+          path: /logs
  sytestpy3postgresmerged:
-    machine: true
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
    steps:
      - checkout
      - run: bash .circleci/merge_base_branch.sh
-      - run: docker pull matrixdotorg/sytest-synapsepy3
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3
+      - run: POSTGRES=1 /synapse_sytest.sh
      - store_artifacts:
-          path: ~/project/logs
+          path: /logs
          destination: logs
      - store_test_results:
-          path: logs
+          path: /logs

 workflows:
  version: 2
@@ -16,7 +16,7 @@ then
    GITBASE="develop"
 else
    # Get the reference, using the GitHub API
-    GITBASE=`curl -q https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
+    GITBASE=`wget -O- https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
 fi

 # Show what we are before
@@ -31,4 +31,4 @@ git fetch -u origin $GITBASE
 git merge --no-edit origin/$GITBASE

 # Show what we are after.
-git show -s
+git show -s
@@ -1,27 +1,45 @@
 sudo: false
 language: python

-# tell travis to cache ~/.cache/pip
-cache: pip
+cache:
+  directories:
+    # we only bother to cache the wheels; parts of the http cache get
+    # invalidated every build (because they get served with a max-age of 600
+    # seconds), which means that we end up re-uploading the whole cache for
+    # every build, which is time-consuming In any case, it's not obvious that
+    # downloading the cache from S3 would be much faster than downloading the
+    # originals from pypi.
+    #
+    - $HOME/.cache/pip/wheels

-before_script:
-  - git remote set-branches --add origin develop
-  - git fetch origin develop
+# don't clone the whole repo history, one commit will do
+git:
+  depth: 1

+# only build branches we care about (PRs are built seperately)
+branches:
+  only:
+    - master
+    - develop
+    - /^release-v/
+
+# When running the tox environments that call Twisted Trial, we can pass the -j
+# flag to run the tests concurrently. We set this to 2 for CPU bound tests
+# (SQLite) and 4 for I/O bound tests (PostgreSQL).
 matrix:
  fast_finish: true
  include:
  - python: 2.7
    env: TOX_ENV=packaging

-  - python: 2.7
-    env: TOX_ENV=pep8
+  - python: 3.6
+    env: TOX_ENV="pep8,check_isort"

  - python: 2.7
-    env: TOX_ENV=py27
+    env: TOX_ENV=py27 TRIAL_FLAGS="-j 2"

  - python: 2.7
-    env: TOX_ENV=py27-old
+    env: TOX_ENV=py27-old TRIAL_FLAGS="-j 2"

  - python: 2.7
    env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
@@ -29,21 +47,24 @@ matrix:
      - postgresql

  - python: 3.5
-    env: TOX_ENV=py35
+    env: TOX_ENV=py35 TRIAL_FLAGS="-j 2"

  - python: 3.6
-    env: TOX_ENV=py36
+    env: TOX_ENV=py36 TRIAL_FLAGS="-j 2"

  - python: 3.6
    env: TOX_ENV=py36-postgres TRIAL_FLAGS="-j 4"
    services:
      - postgresql

-  - python: 3.6
-    env: TOX_ENV=check_isort
-
-  - python: 3.6
+  - # we only need to check for the newsfragment if it's a PR build
+    if: type = pull_request
+    python: 3.6
    env: TOX_ENV=check-newsfragment
+    script:
+      - git remote set-branches --add origin develop
+      - git fetch origin develop
+      - tox -e $TOX_ENV

 install:
  - pip install tox
@@ -1,3 +1,127 @@
+Synapse 0.33.9 (2018-11-19)
+===========================
+
+No significant changes.
+
+
+Synapse 0.33.9rc1 (2018-11-14)
+==============================
+
+Features
+--------
+
+- Include flags to optionally add `m.login.terms` to the registration flow when consent tracking is enabled. ([\#4004](https://github.com/matrix-org/synapse/issues/4004), [\#4133](https://github.com/matrix-org/synapse/issues/4133), [\#4142](https://github.com/matrix-org/synapse/issues/4142), [\#4184](https://github.com/matrix-org/synapse/issues/4184))
+- Support for replacing rooms with new ones ([\#4091](https://github.com/matrix-org/synapse/issues/4091), [\#4099](https://github.com/matrix-org/synapse/issues/4099), [\#4100](https://github.com/matrix-org/synapse/issues/4100), [\#4101](https://github.com/matrix-org/synapse/issues/4101))
+
+
+Bugfixes
+--------
+
+- Fix exceptions when using the email mailer on Python 3. ([\#4095](https://github.com/matrix-org/synapse/issues/4095))
+- Fix e2e key backup with more than 9 backup versions ([\#4113](https://github.com/matrix-org/synapse/issues/4113))
+- Searches that request profile info now no longer fail with a 500. ([\#4122](https://github.com/matrix-org/synapse/issues/4122))
+- fix return code of empty key backups ([\#4123](https://github.com/matrix-org/synapse/issues/4123))
+- If the typing stream ID goes backwards (as on a worker when the master restarts), the worker's typing handler will no longer erroneously report rooms containing new typing events. ([\#4127](https://github.com/matrix-org/synapse/issues/4127))
+- Fix table lock of device_lists_remote_cache which could freeze the application ([\#4132](https://github.com/matrix-org/synapse/issues/4132))
+- Fix exception when using state res v2 algorithm ([\#4135](https://github.com/matrix-org/synapse/issues/4135))
+- Generating the user consent URI no longer fails on Python 3. ([\#4140](https://github.com/matrix-org/synapse/issues/4140), [\#4163](https://github.com/matrix-org/synapse/issues/4163))
+- Loading URL previews from the DB cache on Postgres will no longer cause Unicode type errors when responding to the request, and URL previews will no longer fail if the remote server returns a Content-Type header with the chartype in quotes. ([\#4157](https://github.com/matrix-org/synapse/issues/4157))
+- The hash_password script now works on Python 3. ([\#4161](https://github.com/matrix-org/synapse/issues/4161))
+- Fix noop checks when updating device keys, reducing spurious device list update notifications. ([\#4164](https://github.com/matrix-org/synapse/issues/4164))
+
+
+Deprecations and Removals
+-------------------------
+
+- The disused and un-specced identicon generator has been removed. ([\#4106](https://github.com/matrix-org/synapse/issues/4106))
+- The obsolete and non-functional /pull federation endpoint has been removed. ([\#4118](https://github.com/matrix-org/synapse/issues/4118))
+- The deprecated v1 key exchange endpoints have been removed. ([\#4119](https://github.com/matrix-org/synapse/issues/4119))
+- Synapse will no longer fetch keys using the fallback deprecated v1 key exchange method and will now always use v2. ([\#4120](https://github.com/matrix-org/synapse/issues/4120))
+
+
+Internal Changes
+----------------
+
+- Fix build of Docker image with docker-compose ([\#3778](https://github.com/matrix-org/synapse/issues/3778))
+- Delete unreferenced state groups during history purge ([\#4006](https://github.com/matrix-org/synapse/issues/4006))
+- The "Received rdata" log messages on workers is now logged at DEBUG, not INFO. ([\#4108](https://github.com/matrix-org/synapse/issues/4108))
+- Reduce replication traffic for device lists ([\#4109](https://github.com/matrix-org/synapse/issues/4109))
+- Fix `synapse_replication_tcp_protocol_*_commands` metric label to be full command name, rather than just the first character ([\#4110](https://github.com/matrix-org/synapse/issues/4110))
+- Log some bits about room creation ([\#4121](https://github.com/matrix-org/synapse/issues/4121))
+- Fix `tox` failure on old systems ([\#4124](https://github.com/matrix-org/synapse/issues/4124))
+- Add STATE_V2_TEST room version ([\#4128](https://github.com/matrix-org/synapse/issues/4128))
+- Clean up event accesses and tests ([\#4137](https://github.com/matrix-org/synapse/issues/4137))
+- The default logging config will now set an explicit log file encoding of UTF-8. ([\#4138](https://github.com/matrix-org/synapse/issues/4138))
+- Add helpers functions for getting prev and auth events of an event ([\#4139](https://github.com/matrix-org/synapse/issues/4139))
+- Add some tests for the HTTP pusher. ([\#4149](https://github.com/matrix-org/synapse/issues/4149))
+- add purge_history.sh and purge_remote_media.sh scripts to contrib/ ([\#4155](https://github.com/matrix-org/synapse/issues/4155))
+- HTTP tests have been refactored to contain less boilerplate. ([\#4156](https://github.com/matrix-org/synapse/issues/4156))
+- Drop incoming events from federation for unknown rooms ([\#4165](https://github.com/matrix-org/synapse/issues/4165))
+
+
+Synapse 0.33.8 (2018-11-01)
+===========================
+
+No significant changes.
+
+
+Synapse 0.33.8rc2 (2018-10-31)
+==============================
+
+Bugfixes
+--------
+
+- Searches that request profile info now no longer fail with a 500. Fixes 
+  a regression in 0.33.8rc1. ([\#4122](https://github.com/matrix-org/synapse/issues/4122))
+
+
+Synapse 0.33.8rc1 (2018-10-29)
+==============================
+
+Features
+--------
+
+- Servers with auto-join rooms will now automatically create those rooms when the first user registers ([\#3975](https://github.com/matrix-org/synapse/issues/3975))
+- Add config option to control alias creation ([\#4051](https://github.com/matrix-org/synapse/issues/4051))
+- The register_new_matrix_user script is now ported to Python 3. ([\#4085](https://github.com/matrix-org/synapse/issues/4085))
+- Configure Docker image to listen on both ipv4 and ipv6. ([\#4089](https://github.com/matrix-org/synapse/issues/4089))
+
+
+Bugfixes
+--------
+
+- Fix HTTP error response codes for federated group requests. ([\#3969](https://github.com/matrix-org/synapse/issues/3969))
+- Fix issue where Python 3 users couldn't paginate /publicRooms ([\#4046](https://github.com/matrix-org/synapse/issues/4046))
+- Fix URL previewing to work in Python 3.7 ([\#4050](https://github.com/matrix-org/synapse/issues/4050))
+- synctl will use the right python executable to run worker processes ([\#4057](https://github.com/matrix-org/synapse/issues/4057))
+- Manhole now works again on Python 3, instead of failing with a "couldn't match all kex parts" when connecting. ([\#4060](https://github.com/matrix-org/synapse/issues/4060), [\#4067](https://github.com/matrix-org/synapse/issues/4067))
+- Fix some metrics being racy and causing exceptions when polled by Prometheus. ([\#4061](https://github.com/matrix-org/synapse/issues/4061))
+- Fix bug which prevented email notifications from being sent unless an absolute path was given for `email_templates`. ([\#4068](https://github.com/matrix-org/synapse/issues/4068))
+- Correctly account for cpu usage by background threads ([\#4074](https://github.com/matrix-org/synapse/issues/4074))
+- Fix race condition where config defined reserved users were not being added to
+  the monthly active user list prior to the homeserver reactor firing up ([\#4081](https://github.com/matrix-org/synapse/issues/4081))
+- Fix bug which prevented backslashes being used in event field filters ([\#4083](https://github.com/matrix-org/synapse/issues/4083))
+
+
+Internal Changes
+----------------
+
+- Add information about the [matrix-docker-ansible-deploy](https://github.com/spantaleev/matrix-docker-ansible-deploy) playbook ([\#3698](https://github.com/matrix-org/synapse/issues/3698))
+- Add initial implementation of new state resolution algorithm ([\#3786](https://github.com/matrix-org/synapse/issues/3786))
+- Reduce database load when fetching state groups ([\#4011](https://github.com/matrix-org/synapse/issues/4011))
+- Various cleanups in the federation client code ([\#4031](https://github.com/matrix-org/synapse/issues/4031))
+- Run the CircleCI builds in docker containers ([\#4041](https://github.com/matrix-org/synapse/issues/4041))
+- Only colourise synctl output when attached to tty ([\#4049](https://github.com/matrix-org/synapse/issues/4049))
+- Refactor room alias creation code ([\#4063](https://github.com/matrix-org/synapse/issues/4063))
+- Make the Python scripts in the top-level scripts folders meet pep8 and pass flake8. ([\#4068](https://github.com/matrix-org/synapse/issues/4068))
+- The README now contains example for the Caddy web server. Contributed by steamp0rt. ([\#4072](https://github.com/matrix-org/synapse/issues/4072))
+- Add psutil as an explicit dependency ([\#4073](https://github.com/matrix-org/synapse/issues/4073))
+- Clean up threading and logcontexts in pushers ([\#4075](https://github.com/matrix-org/synapse/issues/4075))
+- Correctly manage logcontexts during startup to fix some "Unexpected logging context" warnings ([\#4076](https://github.com/matrix-org/synapse/issues/4076))
+- Give some more things logcontexts ([\#4077](https://github.com/matrix-org/synapse/issues/4077))
+- Clean up some bits of code which were flagged by the linter ([\#4082](https://github.com/matrix-org/synapse/issues/4082))
+
+
 Synapse 0.33.7 (2018-10-18)
 ===========================

@@ -11,10 +135,6 @@ If you have email notifications enabled, you should ensure that
 have installed customised templates, or leave it unset to use the default
 templates.

-The configuration parser will try to detect the situation where
-`email.template_dir` is incorrectly set to `res/templates` and do the right
-thing, but will warn about this.
-
 Synapse 0.33.7rc2 (2018-10-17)
 ==============================

@@ -174,6 +174,12 @@ Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
 Dockerfile to automate a synapse server in a single Docker image, at
 https://hub.docker.com/r/avhost/docker-matrix/tags/

+Slavi Pantaleev has created an Ansible playbook,
+which installs the offical Docker image of Matrix Synapse
+along with many other Matrix-related services (Postgres database, riot-web, coturn, mxisd, SSL support, etc.).
+For more details, see
+https://github.com/spantaleev/matrix-docker-ansible-deploy
+
 Configuring Synapse
 -------------------

@@ -651,7 +657,8 @@ Using a reverse proxy with Synapse

 It is recommended to put a reverse proxy such as
 `nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
-`Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_ or
+`Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_,
+`Caddy <https://caddyserver.com/docs/proxy>`_ or
 `HAProxy <https://www.haproxy.org/>`_ in front of Synapse. One advantage of
 doing so is that it means that you can expose the default https port (443) to
 Matrix clients without needing to run Synapse with root privileges.
@@ -682,7 +689,15 @@ so an example nginx configuration might look like::
      }
  }

-and an example apache configuration may look like::
+an example Caddy configuration might look like::
+
+    matrix.example.com {
+      proxy /_matrix http://localhost:8008 {
+        transparent
+      }
+    }
+
+and an example Apache configuration might look like::

    <VirtualHost *:443>
        SSLEngine on
@@ -48,6 +48,19 @@ returned by the Client-Server API:
    # configured on port 443.
    curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"

+Upgrading to v0.33.7
+====================
+
+This release removes the example email notification templates from
+``res/templates`` (they are now internal to the python package). This should
+only affect you if you (a) deploy your Synapse instance from a git checkout or
+a github snapshot URL, and (b) have email notifications enabled.
+
+If you have email notifications enabled, you should ensure that
+``email.template_dir`` is either configured to point at a directory where you
+have installed customised templates, or leave it unset to use the default
+templates.
+
 Upgrading to v0.27.3
 ====================

@@ -6,9 +6,11 @@ version: '3'
 services:

  synapse:
-    build: ../..
+    build:
+        context: ../..
+        dockerfile: docker/Dockerfile
    image: docker.io/matrixdotorg/synapse:latest
-    # Since snyapse does not retry to connect to the database, restart upon
+    # Since synapse does not retry to connect to the database, restart upon
    # failure
    restart: unless-stopped
    # See the readme for a full documentation of the environment settings
@@ -47,4 +49,4 @@ services:
      # You may store the database tables in a local folder..
      - ./schemas:/var/lib/postgresql/data
      # .. or store them on some high performance storage for better results
-      # - /path/to/ssd/storage:/var/lib/postfesql/data
+      # - /path/to/ssd/storage:/var/lib/postgresql/data
@@ -0,0 +1,16 @@
+Purge history API examples
+==========================
+
+# `purge_history.sh`
+
+A bash file, that uses the [purge history API](/docs/admin_api/README.rst) to 
+purge all messages in a list of rooms up to a certain event. You can select a 
+timeframe or a number of messages that you want to keep in the room.
+
+Just configure the variables DOMAIN, ADMIN, ROOMS_ARRAY and TIME at the top of
+the script.
+
+# `purge_remote_media.sh`
+
+A bash file, that uses the [purge history API](/docs/admin_api/README.rst) to 
+purge all old cached remote media.
@@ -0,0 +1,141 @@
+#!/bin/bash
+
+# this script will use the api:
+#    https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst
+# 
+# It will purge all messages in a list of rooms up to a cetrain event
+
+###################################################################################################
+# define your domain and admin user
+###################################################################################################
+# add this user as admin in your home server:
+DOMAIN=yourserver.tld
+# add this user as admin in your home server:
+ADMIN="@you_admin_username:$DOMAIN"
+
+API_URL="$DOMAIN:8008/_matrix/client/r0"
+
+###################################################################################################
+#choose the rooms to prune old messages from (add a free comment at the end)
+###################################################################################################
+# the room_id's you can get e.g. from your Riot clients "View Source" button on each message
+ROOMS_ARRAY=(
+'!DgvjtOljKujDBrxyHk:matrix.org#riot:matrix.org'
+'!QtykxKocfZaZOUrTwp:matrix.org#Matrix HQ'
+)
+
+# ALTERNATIVELY:
+# you can select all the rooms that are not encrypted and loop over the result:
+# SELECT room_id FROM rooms WHERE room_id NOT IN (SELECT DISTINCT room_id FROM events WHERE type ='m.room.encrypted')
+# or
+# select all rooms with at least 100 members:
+# SELECT q.room_id FROM (select count(*) as numberofusers, room_id FROM current_state_events WHERE type ='m.room.member'
+#   GROUP BY room_id) AS q LEFT JOIN room_aliases a ON q.room_id=a.room_id WHERE q.numberofusers > 100 ORDER BY numberofusers desc
+
+###################################################################################################
+# evaluate the EVENT_ID before which should be pruned
+###################################################################################################
+# choose a time before which the messages should be pruned:
+TIME='12 months ago'
+# ALTERNATIVELY:
+# a certain time:
+# TIME='2016-08-31 23:59:59'
+
+# creates a timestamp from the given time string:
+UNIX_TIMESTAMP=$(date +%s%3N --date='TZ="UTC+2" '"$TIME")
+
+# ALTERNATIVELY:
+# prune all messages that are older than 1000 messages ago:
+# LAST_MESSAGES=1000
+# SQL_GET_EVENT="SELECT event_id from events WHERE type='m.room.message' AND room_id ='$ROOM' ORDER BY received_ts DESC LIMIT 1 offset $(($LAST_MESSAGES - 1))"
+
+# ALTERNATIVELY:
+# select the EVENT_ID manually:
+#EVENT_ID='$1471814088343495zpPNI:matrix.org' # an example event from 21st of Aug 2016 by Matthew
+
+###################################################################################################
+# make the admin user a server admin in the database with
+###################################################################################################
+# psql -A -t --dbname=synapse -c "UPDATE users SET admin=1 WHERE name LIKE '$ADMIN'"
+
+###################################################################################################
+# database function
+###################################################################################################
+sql (){
+  # for sqlite3:
+  #sqlite3 homeserver.db "pragma busy_timeout=20000;$1" | awk '{print $2}'
+  # for postgres:
+  psql -A -t --dbname=synapse -c "$1" | grep -v 'Pager'
+}
+
+###################################################################################################
+# get an access token
+###################################################################################################
+# for example externally by watching Riot in your browser's network inspector
+# or internally on the server locally, use this:
+TOKEN=$(sql "SELECT token FROM access_tokens WHERE user_id='$ADMIN' ORDER BY id DESC LIMIT 1")
+AUTH="Authorization: Bearer $TOKEN"
+
+###################################################################################################
+# check, if your TOKEN works. For example this works: 
+###################################################################################################
+# $ curl --header "$AUTH" "$API_URL/rooms/$ROOM/state/m.room.power_levels" 
+
+###################################################################################################
+# finally start pruning the room:
+###################################################################################################
+POSTDATA='{"delete_local_events":"true"}' # this will really delete local events, so the messages in the room really disappear unless they are restored by remote federation
+
+for ROOM in "${ROOMS_ARRAY[@]}"; do
+    echo "########################################### $(date) ################# "
+    echo "pruning room: $ROOM ..."
+    ROOM=${ROOM%#*}
+    #set -x
+    echo "check for alias in db..."
+    # for postgres:
+    sql "SELECT * FROM room_aliases WHERE room_id='$ROOM'"
+    echo "get event..."
+    # for postgres:
+    EVENT_ID=$(sql "SELECT event_id FROM events WHERE type='m.room.message' AND received_ts<'$UNIX_TIMESTAMP' AND room_id='$ROOM' ORDER BY received_ts DESC LIMIT 1;")
+    if [ "$EVENT_ID" == "" ]; then
+      echo "no event $TIME"
+    else
+      echo "event: $EVENT_ID"
+      SLEEP=2
+      set -x
+      # call purge
+      OUT=$(curl --header "$AUTH" -s -d $POSTDATA POST "$API_URL/admin/purge_history/$ROOM/$EVENT_ID")
+      PURGE_ID=$(echo "$OUT" |grep purge_id|cut -d'"' -f4 )
+      if [ "$PURGE_ID" == "" ]; then
+        # probably the history purge is already in progress for $ROOM
+        : "continuing with next room"
+      else
+        while : ; do
+          # get status of purge and sleep longer each time if still active
+          sleep $SLEEP
+          STATUS=$(curl --header "$AUTH" -s GET "$API_URL/admin/purge_history_status/$PURGE_ID" |grep status|cut -d'"' -f4)
+          : "$ROOM --> Status: $STATUS"
+          [[ "$STATUS" == "active" ]] || break 
+          SLEEP=$((SLEEP + 1))
+        done 
+      fi
+      set +x
+      sleep 1
+    fi  
+done
+
+
+###################################################################################################
+# additionally
+###################################################################################################
+# to benefit from pruning large amounts of data, you need to call VACUUM to free the unused space.
+# This can take a very long time (hours) and the client have to be stopped while you do so:
+# $ synctl stop
+# $ sqlite3 -line homeserver.db "vacuum;"
+# $ synctl start
+
+# This could be set, so you don't need to prune every time after deleting some rows:
+# $ sqlite3 homeserver.db "PRAGMA auto_vacuum = FULL;"
+# be cautious, it could make the database somewhat slow if there are a lot of deletions
+
+exit
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+DOMAIN=yourserver.tld
+# add this user as admin in your home server:
+ADMIN="@you_admin_username:$DOMAIN"
+
+API_URL="$DOMAIN:8008/_matrix/client/r0"
+
+# choose a time before which the messages should be pruned:
+# TIME='2016-08-31 23:59:59'
+TIME='12 months ago'
+
+# creates a timestamp from the given time string:
+UNIX_TIMESTAMP=$(date +%s%3N --date='TZ="UTC+2" '"$TIME")
+
+
+###################################################################################################
+# database function
+###################################################################################################
+sql (){
+  # for sqlite3:
+  #sqlite3 homeserver.db "pragma busy_timeout=20000;$1" | awk '{print $2}'
+  # for postgres:
+  psql -A -t --dbname=synapse -c "$1" | grep -v 'Pager'
+}
+
+###############################################################################
+# make the admin user a server admin in the database with
+###############################################################################
+# sql "UPDATE users SET admin=1 WHERE name LIKE '$ADMIN'"
+
+###############################################################################
+# get an access token
+###############################################################################
+# for example externally by watching Riot in your browser's network inspector
+# or internally on the server locally, use this:
+TOKEN=$(sql "SELECT token FROM access_tokens WHERE user_id='$ADMIN' ORDER BY id DESC LIMIT 1")
+
+###############################################################################
+# check, if your TOKEN works. For example this works: 
+###############################################################################
+# curl --header "Authorization: Bearer $TOKEN" "$API_URL/rooms/$ROOM/state/m.room.power_levels"
+
+###############################################################################
+# optional check size before
+###############################################################################
+# echo calculate used storage before ...
+# du -shc ../.synapse/media_store/*
+
+###############################################################################
+# finally start pruning media:
+###############################################################################
+set -x # for debugging the generated string
+curl --header "Authorization: Bearer $TOKEN" -v POST "$API_URL/admin/purge_media_cache/?before_ts=$UNIX_TIMESTAMP"
@@ -21,7 +21,7 @@ listeners:
  {% if not SYNAPSE_NO_TLS %}
  -
    port: 8448
-    bind_addresses: ['0.0.0.0']
+    bind_addresses: ['::']
    type: http
    tls: true
    x_forwarded: false
@@ -34,7 +34,7 @@ listeners:

  - port: 8008
    tls: false
-    bind_addresses: ['0.0.0.0']
+    bind_addresses: ['::']
    type: http
    x_forwarded: false

@@ -31,7 +31,7 @@ Note that the templates must be stored under a name giving the language of the
 template - currently this must always be `en` (for "English");
 internationalisation support is intended for the future.

-The template for the policy itself should be versioned and named according to 
+The template for the policy itself should be versioned and named according to
 the version: for example `1.0.html`. The version of the policy which the user
 has agreed to is stored in the database.

@@ -85,6 +85,37 @@ Once this is complete, and the server has been restarted, try visiting
 an error "Missing string query parameter 'u'". It is now possible to manually
 construct URIs where users can give their consent.

+### Enabling consent tracking at registration
+
+1. Add the following to your configuration:
+
+   ```yaml
+   user_consent:
+     require_at_registration: true
+     policy_name: "Privacy Policy" # or whatever you'd like to call the policy
+   ```
+
+2. In your consent templates, make use of the `public_version` variable to
+   see if an unauthenticated user is viewing the page. This is typically
+   wrapped around the form that would be used to actually agree to the document:
+
+   ```
+   {% if not public_version %}
+     <!-- The variables used here are only provided when the 'u' param is given to the homeserver -->
+     <form method="post" action="consent">
+       <input type="hidden" name="v" value="{{version}}"/>
+       <input type="hidden" name="u" value="{{user}}"/>
+       <input type="hidden" name="h" value="{{userhmac}}"/>
+       <input type="submit" value="Sure thing!"/>
+     </form>
+   {% endif %}
+   ```
+
+3. Restart Synapse to apply the changes.
+
+Visiting `https://<server>/_matrix/consent` should now give you a view of the privacy
+document. This is what users will be able to see when registering for accounts.
+
 ### Constructing the consent URI

 It may be useful to manually construct the "consent URI" for a given user - for
@@ -106,6 +137,12 @@ query parameters:
   `https://<server>/_matrix/consent?u=<user>&h=68a152465a4d...`.


+Note that not providing a `u` parameter will be interpreted as wanting to view
+the document from an unauthenticated perspective, such as prior to registration.
+Therefore, the `h` parameter is not required in this scenario. To enable this
+behaviour, set `require_at_registration` to `true` in your `user_consent` config.
+
+
 Sending users a server notice asking them to agree to the policy
 ----------------------------------------------------------------

@@ -12,12 +12,15 @@
    <p>
      All your base are belong to us.
    </p>
-    <form method="post" action="consent">
-      <input type="hidden" name="v" value="{{version}}"/>
-      <input type="hidden" name="u" value="{{user}}"/>
-      <input type="hidden" name="h" value="{{userhmac}}"/>
-      <input type="submit" value="Sure thing!"/>
-    </form>
+    {% if not public_version %}
+      <!-- The variables used here are only provided when the 'u' param is given to the homeserver -->
+      <form method="post" action="consent">
+        <input type="hidden" name="v" value="{{version}}"/>
+        <input type="hidden" name="u" value="{{user}}"/>
+        <input type="hidden" name="h" value="{{userhmac}}"/>
+        <input type="submit" value="Sure thing!"/>
+      </form>
+    {% endif %}
  {% endif %}
  </body>
 </html>
@@ -14,22 +14,3 @@ fi

 # set up the virtualenv
 tox -e py27 --notest -v
-
-TOX_BIN=$TOX_DIR/py27/bin
-
-# cryptography 2.2 requires setuptools >= 18.5.
-#
-# older versions of virtualenv (?) give us a virtualenv with the same version
-# of setuptools as is installed on the system python (and tox runs virtualenv
-# under python3, so we get the version of setuptools that is installed on that).
-#
-# anyway, make sure that we have a recent enough setuptools.
-$TOX_BIN/pip install 'setuptools>=18.5'
-
-# we also need a semi-recent version of pip, because old ones fail to install
-# the "enum34" dependency of cryptography.
-$TOX_BIN/pip install 'pip>=10'
-
-{ python synapse/python_dependencies.py
-  echo lxml
-} | xargs $TOX_BIN/pip install
@@ -1,21 +1,20 @@
-from synapse.events import FrozenEvent
-from synapse.api.auth import Auth
-
-from mock import Mock
+from __future__ import print_function

 import argparse
 import itertools
 import json
 import sys

+from mock import Mock
+
+from synapse.api.auth import Auth
+from synapse.events import FrozenEvent
+

 def check_auth(auth, auth_chain, events):
    auth_chain.sort(key=lambda e: e.depth)

-    auth_map = {
-        e.event_id: e
-        for e in auth_chain
-    }
+    auth_map = {e.event_id: e for e in auth_chain}

    create_events = {}
    for e in auth_chain:
@@ -25,31 +24,26 @@ def check_auth(auth, auth_chain, events):
    for e in itertools.chain(auth_chain, events):
        auth_events_list = [auth_map[i] for i, _ in e.auth_events]

-        auth_events = {
-            (e.type, e.state_key): e
-            for e in auth_events_list
-        }
+        auth_events = {(e.type, e.state_key): e for e in auth_events_list}

        auth_events[("m.room.create", "")] = create_events[e.room_id]

        try:
            auth.check(e, auth_events=auth_events)
        except Exception as ex:
-            print "Failed:", e.event_id, e.type, e.state_key
-            print "Auth_events:", auth_events
-            print ex
-            print json.dumps(e.get_dict(), sort_keys=True, indent=4)
+            print("Failed:", e.event_id, e.type, e.state_key)
+            print("Auth_events:", auth_events)
+            print(ex)
+            print(json.dumps(e.get_dict(), sort_keys=True, indent=4))
            # raise
-        print "Success:", e.event_id, e.type, e.state_key
+        print("Success:", e.event_id, e.type, e.state_key)
+

 if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument(
-        'json',
-        nargs='?',
-        type=argparse.FileType('r'),
-        default=sys.stdin,
+        'json', nargs='?', type=argparse.FileType('r'), default=sys.stdin
    )

    args = parser.parse_args()
@@ -1,10 +1,15 @@
-from synapse.crypto.event_signing import *
-from unpaddedbase64 import encode_base64
-
 import argparse
 import hashlib
-import sys
 import json
+import logging
+import sys
+
+from unpaddedbase64 import encode_base64
+
+from synapse.crypto.event_signing import (
+    check_event_content_hash,
+    compute_event_reference_hash,
+)


 class dictobj(dict):
@@ -24,27 +29,26 @@ class dictobj(dict):

 def main():
    parser = argparse.ArgumentParser()
-    parser.add_argument("input_json", nargs="?", type=argparse.FileType('r'),
-                        default=sys.stdin)
+    parser.add_argument(
+        "input_json", nargs="?", type=argparse.FileType('r'), default=sys.stdin
+    )
    args = parser.parse_args()
    logging.basicConfig()

    event_json = dictobj(json.load(args.input_json))

-    algorithms = {
-        "sha256": hashlib.sha256,
-    }
+    algorithms = {"sha256": hashlib.sha256}

    for alg_name in event_json.hashes:
        if check_event_content_hash(event_json, algorithms[alg_name]):
-            print "PASS content hash %s" % (alg_name,)
+            print("PASS content hash %s" % (alg_name,))
        else:
-            print "FAIL content hash %s" % (alg_name,)
+            print("FAIL content hash %s" % (alg_name,))

    for algorithm in algorithms.values():
        name, h_bytes = compute_event_reference_hash(event_json, algorithm)
-        print "Reference hash %s: %s" % (name, encode_base64(h_bytes))
+        print("Reference hash %s: %s" % (name, encode_base64(h_bytes)))

-if __name__=="__main__":
+
+if __name__ == "__main__":
    main()
-
@@ -1,15 +1,15 @@

-from signedjson.sign import verify_signed_json
+import argparse
+import json
+import logging
+import sys
+import urllib2
+
+import dns.resolver
 from signedjson.key import decode_verify_key_bytes, write_signing_keys
+from signedjson.sign import verify_signed_json
 from unpaddedbase64 import decode_base64

-import urllib2
-import json
-import sys
-import dns.resolver
-import pprint
-import argparse
-import logging

 def get_targets(server_name):
    if ":" in server_name:
@@ -23,6 +23,7 @@ def get_targets(server_name):
    except dns.resolver.NXDOMAIN:
        yield (server_name, 8448)

+
 def get_server_keys(server_name, target, port):
    url = "https://%s:%i/_matrix/key/v1" % (target, port)
    keys = json.load(urllib2.urlopen(url))
@@ -33,12 +34,14 @@ def get_server_keys(server_name, target, port):
        verify_keys[key_id] = verify_key
    return verify_keys

+
 def main():

    parser = argparse.ArgumentParser()
    parser.add_argument("signature_name")
-    parser.add_argument("input_json", nargs="?", type=argparse.FileType('r'),
-                        default=sys.stdin)
+    parser.add_argument(
+        "input_json", nargs="?", type=argparse.FileType('r'), default=sys.stdin
+    )

    args = parser.parse_args()
    logging.basicConfig()
@@ -48,24 +51,23 @@ def main():
    for target, port in get_targets(server_name):
        try:
            keys = get_server_keys(server_name, target, port)
-            print "Using keys from https://%s:%s/_matrix/key/v1" % (target, port)
+            print("Using keys from https://%s:%s/_matrix/key/v1" % (target, port))
            write_signing_keys(sys.stdout, keys.values())
            break
-        except:
+        except Exception:
            logging.exception("Error talking to %s:%s", target, port)

    json_to_check = json.load(args.input_json)
-    print "Checking JSON:"
+    print("Checking JSON:")
    for key_id in json_to_check["signatures"][args.signature_name]:
        try:
            key = keys[key_id]
            verify_signed_json(json_to_check, args.signature_name, key)
-            print "PASS %s" % (key_id,)
-        except:
+            print("PASS %s" % (key_id,))
+        except Exception:
            logging.exception("Check for key %s failed" % (key_id,))
-            print "FAIL %s" % (key_id,)
+            print("FAIL %s" % (key_id,))


 if __name__ == '__main__':
    main()
-
@@ -1,13 +1,21 @@
+import hashlib
+import json
+import sys
+import time
+
+import six
+
 import psycopg2
 import yaml
-import sys
-import json
-import time
-import hashlib
-from unpaddedbase64 import encode_base64
+from canonicaljson import encode_canonical_json
 from signedjson.key import read_signing_keys
 from signedjson.sign import sign_json
-from canonicaljson import encode_canonical_json
+from unpaddedbase64 import encode_base64
+
+if six.PY2:
+    db_type = six.moves.builtins.buffer
+else:
+    db_type = memoryview


 def select_v1_keys(connection):
@@ -39,7 +47,9 @@ def select_v2_json(connection):
    cursor.close()
    results = {}
    for server_name, key_id, key_json in rows:
-        results.setdefault(server_name, {})[key_id] = json.loads(str(key_json).decode("utf-8"))
+        results.setdefault(server_name, {})[key_id] = json.loads(
+            str(key_json).decode("utf-8")
+        )
    return results


@@ -47,10 +57,7 @@ def convert_v1_to_v2(server_name, valid_until, keys, certificate):
    return {
        "old_verify_keys": {},
        "server_name": server_name,
-        "verify_keys": {
-            key_id: {"key": key}
-            for key_id, key in keys.items()
-        },
+        "verify_keys": {key_id: {"key": key} for key_id, key in keys.items()},
        "valid_until_ts": valid_until,
        "tls_fingerprints": [fingerprint(certificate)],
    }
@@ -65,7 +72,7 @@ def rows_v2(server, json):
    valid_until = json["valid_until_ts"]
    key_json = encode_canonical_json(json)
    for key_id in json["verify_keys"]:
-        yield (server, key_id, "-", valid_until, valid_until, buffer(key_json))
+        yield (server, key_id, "-", valid_until, valid_until, db_type(key_json))


 def main():
@@ -87,7 +94,7 @@ def main():

    result = {}
    for server in keys:
-        if not server in json:
+        if server not in json:
            v2_json = convert_v1_to_v2(
                server, valid_until, keys[server], certificates[server]
            )
@@ -96,10 +103,7 @@ def main():

    yaml.safe_dump(result, sys.stdout, default_flow_style=False)

-    rows = list(
-        row for server, json in result.items()
-        for row in rows_v2(server, json)
-    )
+    rows = list(row for server, json in result.items() for row in rows_v2(server, json))

    cursor = connection.cursor()
    cursor.executemany(
@@ -107,7 +111,7 @@ def main():
        " server_name, key_id, from_server,"
        " ts_added_ms, ts_valid_until_ms, key_json"
        ") VALUES (%s, %s, %s, %s, %s, %s)",
-        rows
+        rows,
    )
    connection.commit()

@@ -1,8 +1,16 @@
 #! /usr/bin/python

+from __future__ import print_function
+
+import argparse
 import ast
+import os
+import re
+import sys
+
 import yaml

+
 class DefinitionVisitor(ast.NodeVisitor):
    def __init__(self):
        super(DefinitionVisitor, self).__init__()
@@ -42,15 +50,18 @@ def non_empty(defs):
    functions = {name: non_empty(f) for name, f in defs['def'].items()}
    classes = {name: non_empty(f) for name, f in defs['class'].items()}
    result = {}
-    if functions: result['def'] = functions
-    if classes: result['class'] = classes
+    if functions:
+        result['def'] = functions
+    if classes:
+        result['class'] = classes
    names = defs['names']
    uses = []
    for name in names.get('Load', ()):
        if name not in names.get('Param', ()) and name not in names.get('Store', ()):
            uses.append(name)
    uses.extend(defs['attrs'])
-    if uses: result['uses'] = uses
+    if uses:
+        result['uses'] = uses
    result['names'] = names
    result['attrs'] = defs['attrs']
    return result
@@ -95,7 +106,6 @@ def used_names(prefix, item, defs, names):


 if __name__ == '__main__':
-    import sys, os, argparse, re

    parser = argparse.ArgumentParser(description='Find definitions.')
    parser.add_argument(
@@ -105,24 +115,28 @@ if __name__ == '__main__':
        "--ignore", action="append", metavar="REGEXP", help="Ignore a pattern"
    )
    parser.add_argument(
-        "--pattern", action="append", metavar="REGEXP",
-        help="Search for a pattern"
+        "--pattern", action="append", metavar="REGEXP", help="Search for a pattern"
    )
    parser.add_argument(
-        "directories", nargs='+', metavar="DIR",
-        help="Directories to search for definitions"
+        "directories",
+        nargs='+',
+        metavar="DIR",
+        help="Directories to search for definitions",
    )
    parser.add_argument(
-        "--referrers", default=0, type=int,
-        help="Include referrers up to the given depth"
+        "--referrers",
+        default=0,
+        type=int,
+        help="Include referrers up to the given depth",
    )
    parser.add_argument(
-        "--referred", default=0, type=int,
-        help="Include referred down to the given depth"
+        "--referred",
+        default=0,
+        type=int,
+        help="Include referred down to the given depth",
    )
    parser.add_argument(
-        "--format", default="yaml",
-        help="Output format, one of 'yaml' or 'dot'"
+        "--format", default="yaml", help="Output format, one of 'yaml' or 'dot'"
    )
    args = parser.parse_args()

@@ -162,7 +176,7 @@ if __name__ == '__main__':
            for used_by in entry.get("used", ()):
                referrers.add(used_by)
        for name, definition in names.items():
-            if not name in referrers:
+            if name not in referrers:
                continue
            if ignore and any(pattern.match(name) for pattern in ignore):
                continue
@@ -176,7 +190,7 @@ if __name__ == '__main__':
            for uses in entry.get("uses", ()):
                referred.add(uses)
        for name, definition in names.items():
-            if not name in referred:
+            if name not in referred:
                continue
            if ignore and any(pattern.match(name) for pattern in ignore):
                continue
@@ -185,12 +199,12 @@ if __name__ == '__main__':
    if args.format == 'yaml':
        yaml.dump(result, sys.stdout, default_flow_style=False)
    elif args.format == 'dot':
-        print "digraph {"
+        print("digraph {")
        for name, entry in result.items():
-            print name
+            print(name)
            for used_by in entry.get("used", ()):
                if used_by in result:
-                    print used_by, "->", name
-        print "}"
+                    print(used_by, "->", name)
+        print("}")
    else:
        raise ValueError("Unknown format %r" % (args.format))
@@ -1,8 +1,11 @@
 #!/usr/bin/env python2

-import pymacaroons
+from __future__ import print_function
+
 import sys

+import pymacaroons
+
 if len(sys.argv) == 1:
    sys.stderr.write("usage: %s macaroon [key]\n" % (sys.argv[0],))
    sys.exit(1)
@@ -11,14 +14,14 @@ macaroon_string = sys.argv[1]
 key = sys.argv[2] if len(sys.argv) > 2 else None

 macaroon = pymacaroons.Macaroon.deserialize(macaroon_string)
-print macaroon.inspect()
+print(macaroon.inspect())

-print ""
+print("")

 verifier = pymacaroons.Verifier()
 verifier.satisfy_general(lambda c: True)
 try:
    verifier.verify(macaroon, key)
-    print "Signature is correct"
+    print("Signature is correct")
 except Exception as e:
-    print str(e)
+    print(str(e))
@@ -18,21 +18,21 @@
 from __future__ import print_function

 import argparse
+import base64
+import json
+import sys
 from urlparse import urlparse, urlunparse

 import nacl.signing
-import json
-import base64
 import requests
-import sys
-
-from requests.adapters import HTTPAdapter
 import srvlookup
 import yaml
+from requests.adapters import HTTPAdapter

 # uncomment the following to enable debug logging of http requests
-#from httplib import HTTPConnection
-#HTTPConnection.debuglevel = 1
+# from httplib import HTTPConnection
+# HTTPConnection.debuglevel = 1
+

 def encode_base64(input_bytes):
    """Encode bytes as a base64 string without any padding."""
@@ -58,15 +58,15 @@ def decode_base64(input_string):

 def encode_canonical_json(value):
    return json.dumps(
-         value,
-         # Encode code-points outside of ASCII as UTF-8 rather than \u escapes
-         ensure_ascii=False,
-         # Remove unecessary white space.
-         separators=(',',':'),
-         # Sort the keys of dictionaries.
-         sort_keys=True,
-         # Encode the resulting unicode as UTF-8 bytes.
-     ).encode("UTF-8")
+        value,
+        # Encode code-points outside of ASCII as UTF-8 rather than \u escapes
+        ensure_ascii=False,
+        # Remove unecessary white space.
+        separators=(',', ':'),
+        # Sort the keys of dictionaries.
+        sort_keys=True,
+        # Encode the resulting unicode as UTF-8 bytes.
+    ).encode("UTF-8")


 def sign_json(json_object, signing_key, signing_name):
@@ -88,6 +88,7 @@ def sign_json(json_object, signing_key, signing_name):

 NACL_ED25519 = "ed25519"

+
 def decode_signing_key_base64(algorithm, version, key_base64):
    """Decode a base64 encoded signing key
    Args:
@@ -143,25 +144,25 @@ def request_json(method, origin_name, origin_key, destination, path, content):
    authorization_headers = []

    for key, sig in signed_json["signatures"][origin_name].items():
-        header = "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % (
-            origin_name, key, sig,
-        )
+        header = "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % (origin_name, key, sig)
        authorization_headers.append(bytes(header))
-        print ("Authorization: %s" % header, file=sys.stderr)
+        print("Authorization: %s" % header, file=sys.stderr)

    dest = "matrix://%s%s" % (destination, path)
-    print ("Requesting %s" % dest, file=sys.stderr)
+    print("Requesting %s" % dest, file=sys.stderr)

    s = requests.Session()
    s.mount("matrix://", MatrixConnectionAdapter())

+    headers = {"Host": destination, "Authorization": authorization_headers[0]}
+
+    if method == "POST":
+        headers["Content-Type"] = "application/json"
+
    result = s.request(
        method=method,
        url=dest,
-        headers={
-            "Host": destination,
-            "Authorization": authorization_headers[0]
-        },
+        headers=headers,
        verify=False,
        data=content,
    )
@@ -171,50 +172,50 @@ def request_json(method, origin_name, origin_key, destination, path, content):

 def main():
    parser = argparse.ArgumentParser(
-        description=
-            "Signs and sends a federation request to a matrix homeserver",
+        description="Signs and sends a federation request to a matrix homeserver"
    )

    parser.add_argument(
-        "-N", "--server-name",
+        "-N",
+        "--server-name",
        help="Name to give as the local homeserver. If unspecified, will be "
-             "read from the config file.",
+        "read from the config file.",
    )

    parser.add_argument(
-        "-k", "--signing-key-path",
+        "-k",
+        "--signing-key-path",
        help="Path to the file containing the private ed25519 key to sign the "
-             "request with.",
+        "request with.",
    )

    parser.add_argument(
-        "-c", "--config",
+        "-c",
+        "--config",
        default="homeserver.yaml",
        help="Path to server config file. Ignored if --server-name and "
-             "--signing-key-path are both given.",
+        "--signing-key-path are both given.",
    )

    parser.add_argument(
-        "-d", "--destination",
+        "-d",
+        "--destination",
        default="matrix.org",
        help="name of the remote homeserver. We will do SRV lookups and "
-             "connect appropriately.",
+        "connect appropriately.",
    )

    parser.add_argument(
-        "-X", "--method",
-        help="HTTP method to use for the request. Defaults to GET if --data is"
-             "unspecified, POST if it is."
+        "-X",
+        "--method",
+        help="HTTP method to use for the request. Defaults to GET if --body is"
+        "unspecified, POST if it is.",
    )

-    parser.add_argument(
-        "--body",
-        help="Data to send as the body of the HTTP request"
-    )
+    parser.add_argument("--body", help="Data to send as the body of the HTTP request")

    parser.add_argument(
-        "path",
-        help="request path. We will add '/_matrix/federation/v1/' to this."
+        "path", help="request path. We will add '/_matrix/federation/v1/' to this."
    )

    args = parser.parse_args()
@@ -227,13 +228,15 @@ def main():

    result = request_json(
        args.method,
-        args.server_name, key, args.destination,
+        args.server_name,
+        key,
+        args.destination,
        "/_matrix/federation/v1/" + args.path,
        content=args.body,
    )

    json.dump(result, sys.stdout)
-    print ("")
+    print("")


 def read_args_from_config(args):
@@ -253,7 +256,7 @@ class MatrixConnectionAdapter(HTTPAdapter):
            return s, 8448

        if ":" in s:
-            out = s.rsplit(":",1)
+            out = s.rsplit(":", 1)
            try:
                port = int(out[1])
            except ValueError:
@@ -263,7 +266,7 @@ class MatrixConnectionAdapter(HTTPAdapter):
        try:
            srv = srvlookup.lookup("matrix", "tcp", s)[0]
            return srv.host, srv.port
-        except:
+        except Exception:
            return s, 8448

    def get_connection(self, url, proxies=None):
@@ -272,10 +275,9 @@ class MatrixConnectionAdapter(HTTPAdapter):
        (host, port) = self.lookup(parsed.netloc)
        netloc = "%s:%d" % (host, port)
        print("Connecting to %s" % (netloc,), file=sys.stderr)
-        url = urlunparse((
-            "https", netloc, parsed.path, parsed.params, parsed.query,
-            parsed.fragment,
-        ))
+        url = urlunparse(
+            ("https", netloc, parsed.path, parsed.params, parsed.query, parsed.fragment)
+        )
        return super(MatrixConnectionAdapter, self).get_connection(url, proxies)


@@ -1,23 +1,31 @@
-from synapse.storage.pdu import PduStore
-from synapse.storage.signatures import SignatureStore
-from synapse.storage._base import SQLBaseStore
-from synapse.federation.units import Pdu
-from synapse.crypto.event_signing import (
-    add_event_pdu_content_hash, compute_pdu_event_reference_hash
-)
-from synapse.api.events.utils import prune_pdu
-from unpaddedbase64 import encode_base64, decode_base64
-from canonicaljson import encode_canonical_json
+from __future__ import print_function
+
 import sqlite3
 import sys

+from unpaddedbase64 import decode_base64, encode_base64
+
+from synapse.crypto.event_signing import (
+    add_event_pdu_content_hash,
+    compute_pdu_event_reference_hash,
+)
+from synapse.federation.units import Pdu
+from synapse.storage._base import SQLBaseStore
+from synapse.storage.pdu import PduStore
+from synapse.storage.signatures import SignatureStore
+
+
 class Store(object):
    _get_pdu_tuples = PduStore.__dict__["_get_pdu_tuples"]
    _get_pdu_content_hashes_txn = SignatureStore.__dict__["_get_pdu_content_hashes_txn"]
    _get_prev_pdu_hashes_txn = SignatureStore.__dict__["_get_prev_pdu_hashes_txn"]
-    _get_pdu_origin_signatures_txn = SignatureStore.__dict__["_get_pdu_origin_signatures_txn"]
+    _get_pdu_origin_signatures_txn = SignatureStore.__dict__[
+        "_get_pdu_origin_signatures_txn"
+    ]
    _store_pdu_content_hash_txn = SignatureStore.__dict__["_store_pdu_content_hash_txn"]
-    _store_pdu_reference_hash_txn = SignatureStore.__dict__["_store_pdu_reference_hash_txn"]
+    _store_pdu_reference_hash_txn = SignatureStore.__dict__[
+        "_store_pdu_reference_hash_txn"
+    ]
    _store_prev_pdu_hash_txn = SignatureStore.__dict__["_store_prev_pdu_hash_txn"]
    _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"]

@@ -26,9 +34,7 @@ store = Store()


 def select_pdus(cursor):
-    cursor.execute(
-        "SELECT pdu_id, origin FROM pdus ORDER BY depth ASC"
-    )
+    cursor.execute("SELECT pdu_id, origin FROM pdus ORDER BY depth ASC")

    ids = cursor.fetchall()

@@ -41,23 +47,30 @@ def select_pdus(cursor):
    for pdu in pdus:
        try:
            if pdu.prev_pdus:
-                print "PROCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus
+                print("PROCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus)
                for pdu_id, origin, hashes in pdu.prev_pdus:
                    ref_alg, ref_hsh = reference_hashes[(pdu_id, origin)]
                    hashes[ref_alg] = encode_base64(ref_hsh)
-                    store._store_prev_pdu_hash_txn(cursor,  pdu.pdu_id, pdu.origin, pdu_id, origin, ref_alg, ref_hsh)
-                print "SUCCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus
+                    store._store_prev_pdu_hash_txn(
+                        cursor, pdu.pdu_id, pdu.origin, pdu_id, origin, ref_alg, ref_hsh
+                    )
+                print("SUCCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus)
            pdu = add_event_pdu_content_hash(pdu)
            ref_alg, ref_hsh = compute_pdu_event_reference_hash(pdu)
            reference_hashes[(pdu.pdu_id, pdu.origin)] = (ref_alg, ref_hsh)
-            store._store_pdu_reference_hash_txn(cursor, pdu.pdu_id, pdu.origin, ref_alg, ref_hsh)
+            store._store_pdu_reference_hash_txn(
+                cursor, pdu.pdu_id, pdu.origin, ref_alg, ref_hsh
+            )

            for alg, hsh_base64 in pdu.hashes.items():
-                print alg, hsh_base64
-                store._store_pdu_content_hash_txn(cursor, pdu.pdu_id, pdu.origin, alg, decode_base64(hsh_base64))
+                print(alg, hsh_base64)
+                store._store_pdu_content_hash_txn(
+                    cursor, pdu.pdu_id, pdu.origin, alg, decode_base64(hsh_base64)
+                )
+
+        except Exception:
+            print("FAILED_", pdu.pdu_id, pdu.origin, pdu.prev_pdus)

-        except:
-            print "FAILED_", pdu.pdu_id, pdu.origin, pdu.prev_pdus

 def main():
    conn = sqlite3.connect(sys.argv[1])
@@ -65,5 +78,6 @@ def main():
    select_pdus(cursor)
    conn.commit()

-if __name__=='__main__':
+
+if __name__ == '__main__':
    main()
@@ -1,18 +1,17 @@
 #! /usr/bin/python

-import ast
 import argparse
+import ast
 import os
 import sys
+
 import yaml

 PATTERNS_V1 = []
 PATTERNS_V2 = []

-RESULT = {
-    "v1": PATTERNS_V1,
-    "v2": PATTERNS_V2,
-}
+RESULT = {"v1": PATTERNS_V1, "v2": PATTERNS_V2}
+

 class CallVisitor(ast.NodeVisitor):
    def visit_Call(self, node):
@@ -21,7 +20,6 @@ class CallVisitor(ast.NodeVisitor):
        else:
            return

-
        if name == "client_path_patterns":
            PATTERNS_V1.append(node.args[0].s)
        elif name == "client_v2_patterns":
@@ -42,8 +40,10 @@ def find_patterns_in_file(filepath):
 parser = argparse.ArgumentParser(description='Find url patterns.')

 parser.add_argument(
-    "directories", nargs='+', metavar="DIR",
-    help="Directories to search for definitions"
+    "directories",
+    nargs='+',
+    metavar="DIR",
+    help="Directories to search for definitions",
 )

 args = parser.parse_args()
@@ -1,39 +0,0 @@
-#!/usr/bin/env perl
-
-use strict;
-use warnings;
-
-use DBI;
-use DBD::SQLite;
-use JSON;
-use Getopt::Long;
-
-my $db; # = "homeserver.db";
-my $server = "http://localhost:8008";
-my $size = 320;
-
-GetOptions("db|d=s",     \$db,
-           "server|s=s", \$server,
-           "width|w=i",  \$size) or usage();
-
-usage() unless $db;
-
-my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","") || die $DBI::errstr;
-
-my $res = $dbh->selectall_arrayref("select token, name from access_tokens, users where access_tokens.user_id = users.id group by user_id") || die $DBI::errstr;
-
-foreach (@$res) {
-    my ($token, $mxid) = ($_->[0], $_->[1]);
-    my ($user_id) = ($mxid =~ m/@(.*):/);
-    my ($url) = $dbh->selectrow_array("select avatar_url from profiles where user_id=?", undef, $user_id);
-    if (!$url || $url =~ /#auto$/) {
-        `curl -s -o tmp.png "$server/_matrix/media/v1/identicon?name=${mxid}&width=$size&height=$size"`;
-        my $json = `curl -s -X POST -H "Content-Type: image/png" -T "tmp.png" $server/_matrix/media/v1/upload?access_token=$token`;
-        my $content_uri = from_json($json)->{content_uri};
-        `curl -X PUT -H "Content-Type: application/json" --data '{ "avatar_url": "${content_uri}#auto"}' $server/_matrix/client/api/v1/profile/${mxid}/avatar_url?access_token=$token`;
-    }
-}
-
-sub usage {
-    die "usage: ./make-identicons.pl\n\t-d database [e.g. homeserver.db]\n\t-s homeserver (default: http://localhost:8008)\n\t-w identicon size in pixels (default 320)";
-}
@@ -1,8 +1,9 @@
-import requests
 import collections
+import json
 import sys
 import time
-import json
+
+import requests

 Entry = collections.namedtuple("Entry", "name position rows")

@@ -30,11 +31,11 @@ def parse_response(content):


 def replicate(server, streams):
-    return parse_response(requests.get(
-        server + "/_synapse/replication",
-        verify=False,
-        params=streams
-    ).content)
+    return parse_response(
+        requests.get(
+            server + "/_synapse/replication", verify=False, params=streams
+        ).content
+    )


 def main():
@@ -45,16 +46,16 @@ def main():
        try:
            streams = {
                row.name: row.position
-                for row in replicate(server, {"streams":"-1"})["streams"].rows
+                for row in replicate(server, {"streams": "-1"})["streams"].rows
            }
-        except requests.exceptions.ConnectionError as e:
+        except requests.exceptions.ConnectionError:
            time.sleep(0.1)

    while True:
        try:
            results = replicate(server, streams)
-        except:
-            sys.stdout.write("connection_lost("+ repr(streams) + ")\n")
+        except Exception:
+            sys.stdout.write("connection_lost(" + repr(streams) + ")\n")
            break
        for update in results.values():
            for row in update.rows:
@@ -62,6 +63,5 @@ def main():
            streams[update.name] = update.position


-
-if __name__=='__main__':
+if __name__ == '__main__':
    main()
@@ -1,17 +1,17 @@
 #!/usr/bin/env python

 import argparse
-
+import getpass
 import sys
+import unicodedata

 import bcrypt
-import getpass
-
 import yaml

-bcrypt_rounds=12
+bcrypt_rounds = 12
 password_pepper = ""

+
 def prompt_for_pass():
    password = getpass.getpass("Password: ")

@@ -25,19 +25,27 @@ def prompt_for_pass():

    return password

+
 if __name__ == "__main__":
    parser = argparse.ArgumentParser(
-        description="Calculate the hash of a new password, so that passwords"
-                    " can be reset")
+        description=(
+            "Calculate the hash of a new password, so that passwords can be reset"
+        )
+    )
    parser.add_argument(
-        "-p", "--password",
+        "-p",
+        "--password",
        default=None,
        help="New password for user. Will prompt if omitted.",
    )
    parser.add_argument(
-        "-c", "--config",
+        "-c",
+        "--config",
        type=argparse.FileType('r'),
-        help="Path to server config file. Used to read in bcrypt_rounds and password_pepper.",
+        help=(
+            "Path to server config file. "
+            "Used to read in bcrypt_rounds and password_pepper."
+        ),
    )

    args = parser.parse_args()
@@ -51,5 +59,21 @@ if __name__ == "__main__":
    if not password:
        password = prompt_for_pass()

-    print bcrypt.hashpw(password + password_pepper, bcrypt.gensalt(bcrypt_rounds))
+    # On Python 2, make sure we decode it to Unicode before we normalise it
+    if isinstance(password, bytes):
+        try:
+            password = password.decode(sys.stdin.encoding)
+        except UnicodeDecodeError:
+            print(
+                "ERROR! Your password is not decodable using your terminal encoding (%s)."
+                % (sys.stdin.encoding,)
+            )

+    pw = unicodedata.normalize("NFKC", password)
+
+    hashed = bcrypt.hashpw(
+        pw.encode('utf8') + password_pepper.encode("utf8"),
+        bcrypt.gensalt(bcrypt_rounds),
+    ).decode('ascii')
+
+    print(hashed)
@@ -36,12 +36,9 @@ from __future__ import print_function

 import argparse
 import logging
-
-import sys
-
 import os
-
 import shutil
+import sys

 from synapse.rest.media.v1.filepath import MediaFilePaths

@@ -77,24 +74,23 @@ def move_media(origin_server, file_id, src_paths, dest_paths):
    if not os.path.exists(original_file):
        logger.warn(
            "Original for %s/%s (%s) does not exist",
-            origin_server, file_id, original_file,
+            origin_server,
+            file_id,
+            original_file,
        )
    else:
        mkdir_and_move(
-            original_file,
-            dest_paths.remote_media_filepath(origin_server, file_id),
+            original_file, dest_paths.remote_media_filepath(origin_server, file_id)
        )

    # now look for thumbnails
-    original_thumb_dir = src_paths.remote_media_thumbnail_dir(
-        origin_server, file_id,
-    )
+    original_thumb_dir = src_paths.remote_media_thumbnail_dir(origin_server, file_id)
    if not os.path.exists(original_thumb_dir):
        return

    mkdir_and_move(
        original_thumb_dir,
-        dest_paths.remote_media_thumbnail_dir(origin_server, file_id)
+        dest_paths.remote_media_thumbnail_dir(origin_server, file_id),
    )


@@ -109,24 +105,16 @@ def mkdir_and_move(original_file, dest_file):

 if __name__ == "__main__":
    parser = argparse.ArgumentParser(
-        description=__doc__,
-        formatter_class = argparse.RawDescriptionHelpFormatter,
-    )
-    parser.add_argument(
-        "-v", action='store_true', help='enable debug logging')
-    parser.add_argument(
-        "src_repo",
-        help="Path to source content repo",
-    )
-    parser.add_argument(
-        "dest_repo",
-        help="Path to source content repo",
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
    )
+    parser.add_argument("-v", action='store_true', help='enable debug logging')
+    parser.add_argument("src_repo", help="Path to source content repo")
+    parser.add_argument("dest_repo", help="Path to source content repo")
    args = parser.parse_args()

    logging_config = {
        "level": logging.DEBUG if args.v else logging.INFO,
-        "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s"
+        "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s",
    }
    logging.basicConfig(**logging_config)

@@ -14,197 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+from __future__ import print_function

-import argparse
-import getpass
-import hashlib
-import hmac
-import json
-import sys
-import urllib2
-import yaml
-
-
-def request_registration(user, password, server_location, shared_secret, admin=False):
-    req = urllib2.Request(
-        "%s/_matrix/client/r0/admin/register" % (server_location,),
-        headers={'Content-Type': 'application/json'}
-    )
-
-    try:
-        if sys.version_info[:3] >= (2, 7, 9):
-            # As of version 2.7.9, urllib2 now checks SSL certs
-            import ssl
-            f = urllib2.urlopen(req, context=ssl.SSLContext(ssl.PROTOCOL_SSLv23))
-        else:
-            f = urllib2.urlopen(req)
-        body = f.read()
-        f.close()
-        nonce = json.loads(body)["nonce"]
-    except urllib2.HTTPError as e:
-        print "ERROR! Received %d %s" % (e.code, e.reason,)
-        if 400 <= e.code < 500:
-            if e.info().type == "application/json":
-                resp = json.load(e)
-                if "error" in resp:
-                    print resp["error"]
-        sys.exit(1)
-
-    mac = hmac.new(
-        key=shared_secret,
-        digestmod=hashlib.sha1,
-    )
-
-    mac.update(nonce)
-    mac.update("\x00")
-    mac.update(user)
-    mac.update("\x00")
-    mac.update(password)
-    mac.update("\x00")
-    mac.update("admin" if admin else "notadmin")
-
-    mac = mac.hexdigest()
-
-    data = {
-        "nonce": nonce,
-        "username": user,
-        "password": password,
-        "mac": mac,
-        "admin": admin,
-    }
-
-    server_location = server_location.rstrip("/")
-
-    print "Sending registration request..."
-
-    req = urllib2.Request(
-        "%s/_matrix/client/r0/admin/register" % (server_location,),
-        data=json.dumps(data),
-        headers={'Content-Type': 'application/json'}
-    )
-    try:
-        if sys.version_info[:3] >= (2, 7, 9):
-            # As of version 2.7.9, urllib2 now checks SSL certs
-            import ssl
-            f = urllib2.urlopen(req, context=ssl.SSLContext(ssl.PROTOCOL_SSLv23))
-        else:
-            f = urllib2.urlopen(req)
-        f.read()
-        f.close()
-        print "Success."
-    except urllib2.HTTPError as e:
-        print "ERROR! Received %d %s" % (e.code, e.reason,)
-        if 400 <= e.code < 500:
-            if e.info().type == "application/json":
-                resp = json.load(e)
-                if "error" in resp:
-                    print resp["error"]
-        sys.exit(1)
-
-
-def register_new_user(user, password, server_location, shared_secret, admin):
-    if not user:
-        try:
-            default_user = getpass.getuser()
-        except:
-            default_user = None
-
-        if default_user:
-            user = raw_input("New user localpart [%s]: " % (default_user,))
-            if not user:
-                user = default_user
-        else:
-            user = raw_input("New user localpart: ")
-
-    if not user:
-        print "Invalid user name"
-        sys.exit(1)
-
-    if not password:
-        password = getpass.getpass("Password: ")
-
-        if not password:
-            print "Password cannot be blank."
-            sys.exit(1)
-
-        confirm_password = getpass.getpass("Confirm password: ")
-
-        if password != confirm_password:
-            print "Passwords do not match"
-            sys.exit(1)
-
-    if admin is None:
-        admin = raw_input("Make admin [no]: ")
-        if admin in ("y", "yes", "true"):
-            admin = True
-        else:
-            admin = False
-
-    request_registration(user, password, server_location, shared_secret, bool(admin))
-
+from synapse._scripts.register_new_matrix_user import main

 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description="Used to register new users with a given home server when"
-                    " registration has been disabled. The home server must be"
-                    " configured with the 'registration_shared_secret' option"
-                    " set.",
-    )
-    parser.add_argument(
-        "-u", "--user",
-        default=None,
-        help="Local part of the new user. Will prompt if omitted.",
-    )
-    parser.add_argument(
-        "-p", "--password",
-        default=None,
-        help="New password for user. Will prompt if omitted.",
-    )
-    admin_group = parser.add_mutually_exclusive_group()
-    admin_group.add_argument(
-        "-a", "--admin",
-        action="store_true",
-        help="Register new user as an admin. Will prompt if --no-admin is not set either.",
-    )
-    admin_group.add_argument(
-        "--no-admin",
-        action="store_true",
-        help="Register new user as a regular user. Will prompt if --admin is not set either.",
-    )
-
-    group = parser.add_mutually_exclusive_group(required=True)
-    group.add_argument(
-        "-c", "--config",
-        type=argparse.FileType('r'),
-        help="Path to server config file. Used to read in shared secret.",
-    )
-
-    group.add_argument(
-        "-k", "--shared-secret",
-        help="Shared secret as defined in server config file.",
-    )
-
-    parser.add_argument(
-        "server_url",
-        default="https://localhost:8448",
-        nargs='?',
-        help="URL to use to talk to the home server. Defaults to "
-             " 'https://localhost:8448'.",
-    )
-
-    args = parser.parse_args()
-
-    if "config" in args and args.config:
-        config = yaml.safe_load(args.config)
-        secret = config.get("registration_shared_secret", None)
-        if not secret:
-            print "No 'registration_shared_secret' defined in config."
-            sys.exit(1)
-    else:
-        secret = args.shared_secret
-
-    admin = None
-    if args.admin or args.no_admin:
-        admin = args.admin
-
-    register_new_user(args.user, args.password, args.server_url, secret, admin)
+    main()
@@ -15,23 +15,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-from twisted.internet import defer, reactor
-from twisted.enterprise import adbapi
-
-from synapse.storage._base import LoggingTransaction, SQLBaseStore
-from synapse.storage.engines import create_engine
-from synapse.storage.prepare_database import prepare_database
-
 import argparse
 import curses
 import logging
 import sys
 import time
 import traceback
-import yaml

 from six import string_types

+import yaml
+
+from twisted.enterprise import adbapi
+from twisted.internet import defer, reactor
+
+from synapse.storage._base import LoggingTransaction, SQLBaseStore
+from synapse.storage.engines import create_engine
+from synapse.storage.prepare_database import prepare_database

 logger = logging.getLogger("synapse_port_db")

@@ -105,6 +105,7 @@ class Store(object):

    *All* database interactions should go through this object.
    """
+
    def __init__(self, db_pool, engine):
        self.db_pool = db_pool
        self.database_engine = engine
@@ -135,7 +136,8 @@ class Store(object):
                        txn = conn.cursor()
                        return func(
                            LoggingTransaction(txn, desc, self.database_engine, [], []),
-                            *args, **kwargs
+                            *args,
+                            **kwargs
                        )
                    except self.database_engine.module.DatabaseError as e:
                        if self.database_engine.is_deadlock(e):
@@ -158,22 +160,20 @@ class Store(object):
        def r(txn):
            txn.execute(sql, args)
            return txn.fetchall()
+
        return self.runInteraction("execute_sql", r)

    def insert_many_txn(self, txn, table, headers, rows):
        sql = "INSERT INTO %s (%s) VALUES (%s)" % (
            table,
            ", ".join(k for k in headers),
-            ", ".join("%s" for _ in headers)
+            ", ".join("%s" for _ in headers),
        )

        try:
            txn.executemany(sql, rows)
-        except:
-            logger.exception(
-                "Failed to insert: %s",
-                table,
-            )
+        except Exception:
+            logger.exception("Failed to insert: %s", table)
            raise


@@ -206,7 +206,7 @@ class Porter(object):
                            "table_name": table,
                            "forward_rowid": 1,
                            "backward_rowid": 0,
-                        }
+                        },
                    )

                    forward_chunk = 1
@@ -221,10 +221,10 @@ class Porter(object):
                    table, forward_chunk, backward_chunk
                )
        else:
+
            def delete_all(txn):
                txn.execute(
-                    "DELETE FROM port_from_sqlite3 WHERE table_name = %s",
-                    (table,)
+                    "DELETE FROM port_from_sqlite3 WHERE table_name = %s", (table,)
                )
                txn.execute("TRUNCATE %s CASCADE" % (table,))

@@ -232,11 +232,7 @@ class Porter(object):

            yield self.postgres_store._simple_insert(
                table="port_from_sqlite3",
-                values={
-                    "table_name": table,
-                    "forward_rowid": 1,
-                    "backward_rowid": 0,
-                }
+                values={"table_name": table, "forward_rowid": 1, "backward_rowid": 0},
            )

            forward_chunk = 1
@@ -251,12 +247,16 @@ class Porter(object):
        )

    @defer.inlineCallbacks
-    def handle_table(self, table, postgres_size, table_size, forward_chunk,
-                     backward_chunk):
+    def handle_table(
+        self, table, postgres_size, table_size, forward_chunk, backward_chunk
+    ):
        logger.info(
            "Table %s: %i/%i (rows %i-%i) already ported",
-            table, postgres_size, table_size,
-            backward_chunk+1, forward_chunk-1,
+            table,
+            postgres_size,
+            table_size,
+            backward_chunk + 1,
+            forward_chunk - 1,
        )

        if not table_size:
@@ -271,7 +271,9 @@ class Porter(object):
            return

        if table in (
-            "user_directory", "user_directory_search", "users_who_share_rooms",
+            "user_directory",
+            "user_directory_search",
+            "users_who_share_rooms",
            "users_in_pubic_room",
        ):
            # We don't port these tables, as they're a faff and we can regenreate
@@ -283,37 +285,35 @@ class Porter(object):
            # We need to make sure there is a single row, `(X, null), as that is
            # what synapse expects to be there.
            yield self.postgres_store._simple_insert(
-                table=table,
-                values={"stream_id": None},
+                table=table, values={"stream_id": None}
            )
            self.progress.update(table, table_size)  # Mark table as done
            return

        forward_select = (
-            "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?"
-            % (table,)
+            "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,)
        )

        backward_select = (
-            "SELECT rowid, * FROM %s WHERE rowid <= ? ORDER BY rowid LIMIT ?"
-            % (table,)
+            "SELECT rowid, * FROM %s WHERE rowid <= ? ORDER BY rowid LIMIT ?" % (table,)
        )

        do_forward = [True]
        do_backward = [True]

        while True:
+
            def r(txn):
                forward_rows = []
                backward_rows = []
                if do_forward[0]:
-                    txn.execute(forward_select, (forward_chunk, self.batch_size,))
+                    txn.execute(forward_select, (forward_chunk, self.batch_size))
                    forward_rows = txn.fetchall()
                    if not forward_rows:
                        do_forward[0] = False

                if do_backward[0]:
-                    txn.execute(backward_select, (backward_chunk, self.batch_size,))
+                    txn.execute(backward_select, (backward_chunk, self.batch_size))
                    backward_rows = txn.fetchall()
                    if not backward_rows:
                        do_backward[0] = False
@@ -325,9 +325,7 @@ class Porter(object):

                return headers, forward_rows, backward_rows

-            headers, frows, brows = yield self.sqlite_store.runInteraction(
-                "select", r
-            )
+            headers, frows, brows = yield self.sqlite_store.runInteraction("select", r)

            if frows or brows:
                if frows:
@@ -339,9 +337,7 @@ class Porter(object):
                rows = self._convert_rows(table, headers, rows)

                def insert(txn):
-                    self.postgres_store.insert_many_txn(
-                        txn, table, headers[1:], rows
-                    )
+                    self.postgres_store.insert_many_txn(txn, table, headers[1:], rows)

                    self.postgres_store._simple_update_one_txn(
                        txn,
@@ -362,8 +358,9 @@ class Porter(object):
                return

    @defer.inlineCallbacks
-    def handle_search_table(self, postgres_size, table_size, forward_chunk,
-                            backward_chunk):
+    def handle_search_table(
+        self, postgres_size, table_size, forward_chunk, backward_chunk
+    ):
        select = (
            "SELECT es.rowid, es.*, e.origin_server_ts, e.stream_ordering"
            " FROM event_search as es"
@@ -373,8 +370,9 @@ class Porter(object):
        )

        while True:
+
            def r(txn):
-                txn.execute(select, (forward_chunk, self.batch_size,))
+                txn.execute(select, (forward_chunk, self.batch_size))
                rows = txn.fetchall()
                headers = [column[0] for column in txn.description]

@@ -402,18 +400,21 @@ class Porter(object):
                        else:
                            rows_dict.append(d)

-                    txn.executemany(sql, [
-                        (
-                            row["event_id"],
-                            row["room_id"],
-                            row["key"],
-                            row["sender"],
-                            row["value"],
-                            row["origin_server_ts"],
-                            row["stream_ordering"],
-                        )
-                        for row in rows_dict
-                    ])
+                    txn.executemany(
+                        sql,
+                        [
+                            (
+                                row["event_id"],
+                                row["room_id"],
+                                row["key"],
+                                row["sender"],
+                                row["value"],
+                                row["origin_server_ts"],
+                                row["stream_ordering"],
+                            )
+                            for row in rows_dict
+                        ],
+                    )

                    self.postgres_store._simple_update_one_txn(
                        txn,
@@ -437,7 +438,8 @@ class Porter(object):
    def setup_db(self, db_config, database_engine):
        db_conn = database_engine.module.connect(
            **{
-                k: v for k, v in db_config.get("args", {}).items()
+                k: v
+                for k, v in db_config.get("args", {}).items()
                if not k.startswith("cp_")
            }
        )
@@ -450,13 +452,11 @@ class Porter(object):
    def run(self):
        try:
            sqlite_db_pool = adbapi.ConnectionPool(
-                self.sqlite_config["name"],
-                **self.sqlite_config["args"]
+                self.sqlite_config["name"], **self.sqlite_config["args"]
            )

            postgres_db_pool = adbapi.ConnectionPool(
-                self.postgres_config["name"],
-                **self.postgres_config["args"]
+                self.postgres_config["name"], **self.postgres_config["args"]
            )

            sqlite_engine = create_engine(sqlite_config)
@@ -465,9 +465,7 @@ class Porter(object):
            self.sqlite_store = Store(sqlite_db_pool, sqlite_engine)
            self.postgres_store = Store(postgres_db_pool, postgres_engine)

-            yield self.postgres_store.execute(
-                postgres_engine.check_database
-            )
+            yield self.postgres_store.execute(postgres_engine.check_database)

            # Step 1. Set up databases.
            self.progress.set_state("Preparing SQLite3")
@@ -477,6 +475,7 @@ class Porter(object):
            self.setup_db(postgres_config, postgres_engine)

            self.progress.set_state("Creating port tables")
+
            def create_port_table(txn):
                txn.execute(
                    "CREATE TABLE IF NOT EXISTS port_from_sqlite3 ("
@@ -501,10 +500,9 @@ class Porter(object):
                )

            try:
-                yield self.postgres_store.runInteraction(
-                    "alter_table", alter_table
-                )
-            except Exception as e:
+                yield self.postgres_store.runInteraction("alter_table", alter_table)
+            except Exception:
+                # On Error Resume Next
                pass

            yield self.postgres_store.runInteraction(
@@ -514,11 +512,7 @@ class Porter(object):
            # Step 2. Get tables.
            self.progress.set_state("Fetching tables")
            sqlite_tables = yield self.sqlite_store._simple_select_onecol(
-                table="sqlite_master",
-                keyvalues={
-                    "type": "table",
-                },
-                retcol="name",
+                table="sqlite_master", keyvalues={"type": "table"}, retcol="name"
            )

            postgres_tables = yield self.postgres_store._simple_select_onecol(
@@ -545,18 +539,14 @@ class Porter(object):
            # Step 4. Do the copying.
            self.progress.set_state("Copying to postgres")
            yield defer.gatherResults(
-                [
-                    self.handle_table(*res)
-                    for res in setup_res
-                ],
-                consumeErrors=True,
+                [self.handle_table(*res) for res in setup_res], consumeErrors=True
            )

            # Step 5. Do final post-processing
            yield self._setup_state_group_id_seq()

            self.progress.done()
-        except:
+        except Exception:
            global end_error_exec_info
            end_error_exec_info = sys.exc_info()
            logger.exception("")
@@ -566,9 +556,7 @@ class Porter(object):
    def _convert_rows(self, table, headers, rows):
        bool_col_names = BOOLEAN_COLUMNS.get(table, [])

-        bool_cols = [
-            i for i, h in enumerate(headers) if h in bool_col_names
-        ]
+        bool_cols = [i for i, h in enumerate(headers) if h in bool_col_names]

        class BadValueException(Exception):
            pass
@@ -577,18 +565,21 @@ class Porter(object):
            if j in bool_cols:
                return bool(col)
            elif isinstance(col, string_types) and "\0" in col:
-                logger.warn("DROPPING ROW: NUL value in table %s col %s: %r", table, headers[j], col)
-                raise BadValueException();
+                logger.warn(
+                    "DROPPING ROW: NUL value in table %s col %s: %r",
+                    table,
+                    headers[j],
+                    col,
+                )
+                raise BadValueException()
            return col

        outrows = []
        for i, row in enumerate(rows):
            try:
-                outrows.append(tuple(
-                    conv(j, col)
-                    for j, col in enumerate(row)
-                    if j > 0
-                ))
+                outrows.append(
+                    tuple(conv(j, col) for j, col in enumerate(row) if j > 0)
+                )
            except BadValueException:
                pass

@@ -616,9 +607,7 @@ class Porter(object):

            return headers, [r for r in rows if r[ts_ind] < yesterday]

-        headers, rows = yield self.sqlite_store.runInteraction(
-            "select", r,
-        )
+        headers, rows = yield self.sqlite_store.runInteraction("select", r)

        rows = self._convert_rows("sent_transactions", headers, rows)

@@ -639,7 +628,7 @@ class Porter(object):
            txn.execute(
                "SELECT rowid FROM sent_transactions WHERE ts >= ?"
                " ORDER BY rowid ASC LIMIT 1",
-                (yesterday,)
+                (yesterday,),
            )

            rows = txn.fetchall()
@@ -657,21 +646,17 @@ class Porter(object):
                "table_name": "sent_transactions",
                "forward_rowid": next_chunk,
                "backward_rowid": 0,
-            }
+            },
        )

        def get_sent_table_size(txn):
            txn.execute(
-                "SELECT count(*) FROM sent_transactions"
-                " WHERE ts >= ?",
-                (yesterday,)
+                "SELECT count(*) FROM sent_transactions" " WHERE ts >= ?", (yesterday,)
            )
            size, = txn.fetchone()
            return int(size)

-        remaining_count = yield self.sqlite_store.execute(
-            get_sent_table_size
-        )
+        remaining_count = yield self.sqlite_store.execute(get_sent_table_size)

        total_count = remaining_count + inserted_rows

@@ -680,13 +665,11 @@ class Porter(object):
    @defer.inlineCallbacks
    def _get_remaining_count_to_port(self, table, forward_chunk, backward_chunk):
        frows = yield self.sqlite_store.execute_sql(
-            "SELECT count(*) FROM %s WHERE rowid >= ?" % (table,),
-            forward_chunk,
+            "SELECT count(*) FROM %s WHERE rowid >= ?" % (table,), forward_chunk
        )

        brows = yield self.sqlite_store.execute_sql(
-            "SELECT count(*) FROM %s WHERE rowid <= ?" % (table,),
-            backward_chunk,
+            "SELECT count(*) FROM %s WHERE rowid <= ?" % (table,), backward_chunk
        )

        defer.returnValue(frows[0][0] + brows[0][0])
@@ -694,7 +677,7 @@ class Porter(object):
    @defer.inlineCallbacks
    def _get_already_ported_count(self, table):
        rows = yield self.postgres_store.execute_sql(
-            "SELECT count(*) FROM %s" % (table,),
+            "SELECT count(*) FROM %s" % (table,)
        )

        defer.returnValue(rows[0][0])
@@ -717,22 +700,21 @@ class Porter(object):
    def _setup_state_group_id_seq(self):
        def r(txn):
            txn.execute("SELECT MAX(id) FROM state_groups")
-            next_id = txn.fetchone()[0]+1
-            txn.execute(
-                "ALTER SEQUENCE state_group_id_seq RESTART WITH %s",
-                (next_id,),
-            )
+            next_id = txn.fetchone()[0] + 1
+            txn.execute("ALTER SEQUENCE state_group_id_seq RESTART WITH %s", (next_id,))
+
        return self.postgres_store.runInteraction("setup_state_group_id_seq", r)


 ##############################################
-###### The following is simply UI stuff ######
+# The following is simply UI stuff
 ##############################################


 class Progress(object):
    """Used to report progress of the port
    """
+
    def __init__(self):
        self.tables = {}

@@ -758,6 +740,7 @@ class Progress(object):
 class CursesProgress(Progress):
    """Reports progress to a curses window
    """
+
    def __init__(self, stdscr):
        self.stdscr = stdscr

@@ -801,7 +784,7 @@ class CursesProgress(Progress):
        duration = int(now) - int(self.start_time)

        minutes, seconds = divmod(duration, 60)
-        duration_str = '%02dm %02ds' % (minutes, seconds,)
+        duration_str = '%02dm %02ds' % (minutes, seconds)

        if self.finished:
            status = "Time spent: %s (Done!)" % (duration_str,)
@@ -814,16 +797,12 @@ class CursesProgress(Progress):
                est_remaining_str = '%02dm %02ds remaining' % divmod(est_remaining, 60)
            else:
                est_remaining_str = "Unknown"
-            status = (
-                "Time spent: %s (est. remaining: %s)"
-                % (duration_str, est_remaining_str,)
+            status = "Time spent: %s (est. remaining: %s)" % (
+                duration_str,
+                est_remaining_str,
            )

-        self.stdscr.addstr(
-            0, 0,
-            status,
-            curses.A_BOLD,
-        )
+        self.stdscr.addstr(0, 0, status, curses.A_BOLD)

        max_len = max([len(t) for t in self.tables.keys()])

@@ -831,9 +810,7 @@ class CursesProgress(Progress):
        middle_space = 1

        items = self.tables.items()
-        items.sort(
-            key=lambda i: (i[1]["perc"], i[0]),
-        )
+        items.sort(key=lambda i: (i[1]["perc"], i[0]))

        for i, (table, data) in enumerate(items):
            if i + 2 >= rows:
@@ -844,9 +821,7 @@ class CursesProgress(Progress):
            color = curses.color_pair(2) if perc == 100 else curses.color_pair(1)

            self.stdscr.addstr(
-                i + 2, left_margin + max_len - len(table),
-                table,
-                curses.A_BOLD | color,
+                i + 2, left_margin + max_len - len(table), table, curses.A_BOLD | color
            )

            size = 20
@@ -857,15 +832,13 @@ class CursesProgress(Progress):
            )

            self.stdscr.addstr(
-                i + 2, left_margin + max_len + middle_space,
+                i + 2,
+                left_margin + max_len + middle_space,
                "%s %3d%% (%d/%d)" % (progress, perc, data["num_done"], data["total"]),
            )

        if self.finished:
-            self.stdscr.addstr(
-                rows - 1, 0,
-                "Press any key to exit...",
-            )
+            self.stdscr.addstr(rows - 1, 0, "Press any key to exit...")

        self.stdscr.refresh()
        self.last_update = time.time()
@@ -877,29 +850,25 @@ class CursesProgress(Progress):

    def set_state(self, state):
        self.stdscr.clear()
-        self.stdscr.addstr(
-            0, 0,
-            state + "...",
-            curses.A_BOLD,
-        )
+        self.stdscr.addstr(0, 0, state + "...", curses.A_BOLD)
        self.stdscr.refresh()


 class TerminalProgress(Progress):
    """Just prints progress to the terminal
    """
+
    def update(self, table, num_done):
        super(TerminalProgress, self).update(table, num_done)

        data = self.tables[table]

-        print "%s: %d%% (%d/%d)" % (
-            table, data["perc"],
-            data["num_done"], data["total"],
+        print(
+            "%s: %d%% (%d/%d)" % (table, data["perc"], data["num_done"], data["total"])
        )

    def set_state(self, state):
-        print state + "..."
+        print(state + "...")


 ##############################################
@@ -909,34 +878,38 @@ class TerminalProgress(Progress):
 if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="A script to port an existing synapse SQLite database to"
-                    " a new PostgreSQL database."
+        " a new PostgreSQL database."
    )
    parser.add_argument("-v", action='store_true')
    parser.add_argument(
-        "--sqlite-database", required=True,
+        "--sqlite-database",
+        required=True,
        help="The snapshot of the SQLite database file. This must not be"
-             " currently used by a running synapse server"
+        " currently used by a running synapse server",
    )
    parser.add_argument(
-        "--postgres-config", type=argparse.FileType('r'), required=True,
-        help="The database config file for the PostgreSQL database"
+        "--postgres-config",
+        type=argparse.FileType('r'),
+        required=True,
+        help="The database config file for the PostgreSQL database",
    )
    parser.add_argument(
-        "--curses", action='store_true',
-        help="display a curses based progress UI"
+        "--curses", action='store_true', help="display a curses based progress UI"
    )

    parser.add_argument(
-        "--batch-size", type=int, default=1000,
+        "--batch-size",
+        type=int,
+        default=1000,
        help="The number of rows to select from the SQLite table each"
-             " iteration [default=1000]",
+        " iteration [default=1000]",
    )

    args = parser.parse_args()

    logging_config = {
        "level": logging.DEBUG if args.v else logging.INFO,
-        "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s"
+        "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s",
    }

    if args.curses:
@@ -14,17 +14,16 @@ ignore =
    pylint.cfg
    tox.ini

-[pep8]
-max-line-length = 90
-#  W503 requires that binary operators be at the end, not start, of lines. Erik
-#  doesn't like it.  E203 is contrary to PEP8.  E731 is silly.
-ignore = W503,E203,E731
-
 [flake8]
-# note that flake8 inherits the "ignore" settings from "pep8" (because it uses
-# pep8 to do those checks), but not the "max-line-length" setting
 max-line-length = 90
-ignore=W503,E203,E731
+
+# see https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes
+# for error codes. The ones we ignore are:
+#  W503: line break before binary operator
+#  W504: line break after binary operator
+#  E203: whitespace before ':' (which is contrary to pep8?)
+#  E731: do not assign a lambda expression, use a def
+ignore=W503,W504,E203,E731

 [isort]
 line_length = 89
@@ -1,6 +1,8 @@
 #!/usr/bin/env python

-# Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2014-2017 OpenMarket Ltd
+# Copyright 2017 Vector Creations Ltd
+# Copyright 2017-2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -86,7 +88,7 @@ setup(
    name="matrix-synapse",
    version=version,
    packages=find_packages(exclude=["tests", "tests.*"]),
-    description="Reference Synapse Home Server",
+    description="Reference homeserver for the Matrix decentralised comms protocol",
    install_requires=dependencies['requirements'](include_conditional=True).keys(),
    dependency_links=dependencies["DEPENDENCY_LINKS"].values(),
    include_package_data=True,
@@ -27,4 +27,4 @@ try:
 except ImportError:
    pass

-__version__ = "0.33.7"
+__version__ = "0.33.9"
@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015, 2016 OpenMarket Ltd
+# Copyright 2018 New Vector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import argparse
+import getpass
+import hashlib
+import hmac
+import logging
+import sys
+
+from six.moves import input
+
+import requests as _requests
+import yaml
+
+
+def request_registration(
+    user,
+    password,
+    server_location,
+    shared_secret,
+    admin=False,
+    requests=_requests,
+    _print=print,
+    exit=sys.exit,
+):
+
+    url = "%s/_matrix/client/r0/admin/register" % (server_location,)
+
+    # Get the nonce
+    r = requests.get(url, verify=False)
+
+    if r.status_code is not 200:
+        _print("ERROR! Received %d %s" % (r.status_code, r.reason))
+        if 400 <= r.status_code < 500:
+            try:
+                _print(r.json()["error"])
+            except Exception:
+                pass
+        return exit(1)
+
+    nonce = r.json()["nonce"]
+
+    mac = hmac.new(key=shared_secret.encode('utf8'), digestmod=hashlib.sha1)
+
+    mac.update(nonce.encode('utf8'))
+    mac.update(b"\x00")
+    mac.update(user.encode('utf8'))
+    mac.update(b"\x00")
+    mac.update(password.encode('utf8'))
+    mac.update(b"\x00")
+    mac.update(b"admin" if admin else b"notadmin")
+
+    mac = mac.hexdigest()
+
+    data = {
+        "nonce": nonce,
+        "username": user,
+        "password": password,
+        "mac": mac,
+        "admin": admin,
+    }
+
+    _print("Sending registration request...")
+    r = requests.post(url, json=data, verify=False)
+
+    if r.status_code is not 200:
+        _print("ERROR! Received %d %s" % (r.status_code, r.reason))
+        if 400 <= r.status_code < 500:
+            try:
+                _print(r.json()["error"])
+            except Exception:
+                pass
+        return exit(1)
+
+    _print("Success!")
+
+
+def register_new_user(user, password, server_location, shared_secret, admin):
+    if not user:
+        try:
+            default_user = getpass.getuser()
+        except Exception:
+            default_user = None
+
+        if default_user:
+            user = input("New user localpart [%s]: " % (default_user,))
+            if not user:
+                user = default_user
+        else:
+            user = input("New user localpart: ")
+
+    if not user:
+        print("Invalid user name")
+        sys.exit(1)
+
+    if not password:
+        password = getpass.getpass("Password: ")
+
+        if not password:
+            print("Password cannot be blank.")
+            sys.exit(1)
+
+        confirm_password = getpass.getpass("Confirm password: ")
+
+        if password != confirm_password:
+            print("Passwords do not match")
+            sys.exit(1)
+
+    if admin is None:
+        admin = input("Make admin [no]: ")
+        if admin in ("y", "yes", "true"):
+            admin = True
+        else:
+            admin = False
+
+    request_registration(user, password, server_location, shared_secret, bool(admin))
+
+
+def main():
+
+    logging.captureWarnings(True)
+
+    parser = argparse.ArgumentParser(
+        description="Used to register new users with a given home server when"
+        " registration has been disabled. The home server must be"
+        " configured with the 'registration_shared_secret' option"
+        " set."
+    )
+    parser.add_argument(
+        "-u",
+        "--user",
+        default=None,
+        help="Local part of the new user. Will prompt if omitted.",
+    )
+    parser.add_argument(
+        "-p",
+        "--password",
+        default=None,
+        help="New password for user. Will prompt if omitted.",
+    )
+    admin_group = parser.add_mutually_exclusive_group()
+    admin_group.add_argument(
+        "-a",
+        "--admin",
+        action="store_true",
+        help=(
+            "Register new user as an admin. "
+            "Will prompt if --no-admin is not set either."
+        ),
+    )
+    admin_group.add_argument(
+        "--no-admin",
+        action="store_true",
+        help=(
+            "Register new user as a regular user. "
+            "Will prompt if --admin is not set either."
+        ),
+    )
+
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument(
+        "-c",
+        "--config",
+        type=argparse.FileType('r'),
+        help="Path to server config file. Used to read in shared secret.",
+    )
+
+    group.add_argument(
+        "-k", "--shared-secret", help="Shared secret as defined in server config file."
+    )
+
+    parser.add_argument(
+        "server_url",
+        default="https://localhost:8448",
+        nargs='?',
+        help="URL to use to talk to the home server. Defaults to "
+        " 'https://localhost:8448'.",
+    )
+
+    args = parser.parse_args()
+
+    if "config" in args and args.config:
+        config = yaml.safe_load(args.config)
+        secret = config.get("registration_shared_secret", None)
+        if not secret:
+            print("No 'registration_shared_secret' defined in config.")
+            sys.exit(1)
+    else:
+        secret = args.shared_secret
+
+    admin = None
+    if args.admin or args.no_admin:
+        admin = args.admin
+
+    register_new_user(args.user, args.password, args.server_url, secret, admin)
+
+
+if __name__ == "__main__":
+    main()
@@ -51,6 +51,7 @@ class LoginType(object):
    EMAIL_IDENTITY = u"m.login.email.identity"
    MSISDN = u"m.login.msisdn"
    RECAPTCHA = u"m.login.recaptcha"
+    TERMS = u"m.login.terms"
    DUMMY = u"m.login.dummy"

    # Only for C/S API v1
@@ -61,6 +62,7 @@ class LoginType(object):
 class EventTypes(object):
    Member = "m.room.member"
    Create = "m.room.create"
+    Tombstone = "m.room.tombstone"
    JoinRules = "m.room.join_rules"
    PowerLevels = "m.room.power_levels"
    Aliases = "m.room.aliases"
@@ -101,6 +103,7 @@ class ThirdPartyEntityKind(object):
 class RoomVersions(object):
    V1 = "1"
    VDH_TEST = "vdh-test-version"
+    STATE_V2_TEST = "state-v2-test"


 # the version we will give rooms which are created on this server
@@ -108,7 +111,11 @@ DEFAULT_ROOM_VERSION = RoomVersions.V1

 # vdh-test-version is a placeholder to get room versioning support working and tested
 # until we have a working v2.
-KNOWN_ROOM_VERSIONS = {RoomVersions.V1, RoomVersions.VDH_TEST}
+KNOWN_ROOM_VERSIONS = {
+    RoomVersions.V1,
+    RoomVersions.VDH_TEST,
+    RoomVersions.STATE_V2_TEST,
+}

 ServerNoticeMsgType = "m.server_notice"
 ServerNoticeLimitReached = "m.server_notice.usage_limit_reached"
@@ -172,7 +172,10 @@ USER_FILTER_SCHEMA = {
                # events a lot easier as we can then use a negative lookbehind
                # assertion to split '\.' If we allowed \\ then it would
                # incorrectly split '\\.' See synapse.events.utils.serialize_event
-                "pattern": "^((?!\\\).)*$"
+                #
+                # Note that because this is a regular expression, we have to escape
+                # each backslash in the pattern.
+                "pattern": r"^((?!\\\\).)*$"
            }
        }
    },
@@ -28,7 +28,6 @@ FEDERATION_PREFIX = "/_matrix/federation/v1"
 STATIC_PREFIX = "/_matrix/static"
 WEB_CLIENT_PREFIX = "/_matrix/client"
 CONTENT_REPO_PREFIX = "/_matrix/content"
-SERVER_KEY_PREFIX = "/_matrix/key/v1"
 SERVER_KEY_V2_PREFIX = "/_matrix/key/v2"
 MEDIA_PREFIX = "/_matrix/media/r0"
 LEGACY_MEDIA_PREFIX = "/_matrix/media/v1"
@@ -20,6 +20,7 @@ import sys

 from six import iteritems

+import psutil
 from prometheus_client import Gauge

 from twisted.application import service
@@ -36,7 +37,6 @@ from synapse.api.urls import (
    FEDERATION_PREFIX,
    LEGACY_MEDIA_PREFIX,
    MEDIA_PREFIX,
-    SERVER_KEY_PREFIX,
    SERVER_KEY_V2_PREFIX,
    STATIC_PREFIX,
    WEB_CLIENT_PREFIX,
@@ -58,7 +58,6 @@ from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, check_requirem
 from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
 from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
 from synapse.rest import ClientRestResource
-from synapse.rest.key.v1.server_key_resource import LocalKey
 from synapse.rest.key.v2 import KeyApiV2Resource
 from synapse.rest.media.v0.content_repository import ContentRepoResource
 from synapse.server import HomeServer
@@ -235,10 +234,7 @@ class SynapseHomeServer(HomeServer):
                )

        if name in ["keys", "federation"]:
-            resources.update({
-                SERVER_KEY_PREFIX: LocalKey(self),
-                SERVER_KEY_V2_PREFIX: KeyApiV2Resource(self),
-            })
+            resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self)

        if name == "webclient":
            resources[WEB_CLIENT_PREFIX] = build_resource_for_web_client(self)
@@ -502,7 +498,6 @@ def run(hs):

    def performance_stats_init():
        try:
-            import psutil
            process = psutil.Process()
            # Ensure we can fetch both, and make the initial request for cpu_percent
            # so the next request will use this as the initial point.
@@ -510,12 +505,9 @@ def run(hs):
            process.cpu_percent(interval=None)
            logger.info("report_stats can use psutil")
            stats_process.append(process)
-        except (ImportError, AttributeError):
-            logger.warn(
-                "report_stats enabled but psutil is not installed or incorrect version."
-                " Disabling reporting of memory/cpu stats."
-                " Ensuring psutil is available will help matrix.org track performance"
-                " changes across releases."
+        except (AttributeError):
+            logger.warning(
+                "Unable to read memory/cpu stats. Disabling reporting."
            )

    def generate_user_daily_visit_stats():
@@ -530,10 +522,13 @@ def run(hs):
    clock.looping_call(generate_user_daily_visit_stats, 5 * 60 * 1000)

    # monthly active user limiting functionality
-    clock.looping_call(
-        hs.get_datastore().reap_monthly_active_users, 1000 * 60 * 60
-    )
-    hs.get_datastore().reap_monthly_active_users()
+    def reap_monthly_active_users():
+        return run_as_background_process(
+            "reap_monthly_active_users",
+            hs.get_datastore().reap_monthly_active_users,
+        )
+    clock.looping_call(reap_monthly_active_users, 1000 * 60 * 60)
+    reap_monthly_active_users()

    @defer.inlineCallbacks
    def generate_monthly_active_users():
@@ -547,12 +542,15 @@ def run(hs):
        registered_reserved_users_mau_gauge.set(float(reserved_count))
        max_mau_gauge.set(float(hs.config.max_mau_value))

-    hs.get_datastore().initialise_reserved_users(
-        hs.config.mau_limits_reserved_threepids
-    )
-    generate_monthly_active_users()
+    def start_generate_monthly_active_users():
+        return run_as_background_process(
+            "generate_monthly_active_users",
+            generate_monthly_active_users,
+        )
+
+    start_generate_monthly_active_users()
    if hs.config.limit_usage_by_mau:
-        clock.looping_call(generate_monthly_active_users, 5 * 60 * 1000)
+        clock.looping_call(start_generate_monthly_active_users, 5 * 60 * 1000)
    # End of monthly active user settings

    if hs.config.report_stats:
@@ -568,7 +566,7 @@ def run(hs):
        clock.call_later(5 * 60, start_phone_stats_home)

    if hs.config.daemonize and hs.config.print_pidfile:
-        print (hs.config.pid_file)
+        print(hs.config.pid_file)

    _base.start_reactor(
        "synapse-homeserver",
@@ -161,11 +161,11 @@ class PusherReplicationHandler(ReplicationClientHandler):
                    else:
                        yield self.start_pusher(row.user_id, row.app_id, row.pushkey)
            elif stream_name == "events":
-                self.pusher_pool.on_new_notifications(
+                yield self.pusher_pool.on_new_notifications(
                    token, token,
                )
            elif stream_name == "receipts":
-                self.pusher_pool.on_new_receipts(
+                yield self.pusher_pool.on_new_receipts(
                    token, token, set(row.room_id for row in rows)
                )
        except Exception:
@@ -183,7 +183,7 @@ class PusherReplicationHandler(ReplicationClientHandler):
    def start_pusher(self, user_id, app_id, pushkey):
        key = "%s:%s" % (app_id, pushkey)
        logger.info("Starting pusher %r / %r", user_id, key)
-        return self.pusher_pool._refresh_pusher(app_id, pushkey, user_id)
+        return self.pusher_pool.start_pusher_by_id(app_id, pushkey, user_id)


 def start(config_options):
@@ -226,7 +226,15 @@ class SynchrotronPresence(object):
 class SynchrotronTyping(object):
    def __init__(self, hs):
        self._latest_room_serial = 0
+        self._reset()
+
+    def _reset(self):
+        """
+        Reset the typing handler's data caches.
+        """
+        # map room IDs to serial numbers
        self._room_serials = {}
+        # map room IDs to sets of users currently typing
        self._room_typing = {}

    def stream_positions(self):
@@ -236,6 +244,12 @@ class SynchrotronTyping(object):
        return {"typing": self._latest_room_serial}

    def process_replication_rows(self, token, rows):
+        if self._latest_room_serial > token:
+            # The master has gone backwards. To prevent inconsistent data, just
+            # clear everything.
+            self._reset()
+
+        # Set the latest serial token to whatever the server gave us.
        self._latest_room_serial = token

        for row in rows:
@@ -28,7 +28,7 @@ if __name__ == "__main__":
            sys.stderr.write("\n" + str(e) + "\n")
            sys.exit(1)

-        print (getattr(config, key))
+        print(getattr(config, key))
        sys.exit(0)
    else:
        sys.stderr.write("Unknown command %r\n" % (action,))
@@ -106,10 +106,7 @@ class Config(object):
    @classmethod
    def check_file(cls, file_path, config_name):
        if file_path is None:
-            raise ConfigError(
-                "Missing config for %s."
-                % (config_name,)
-            )
+            raise ConfigError("Missing config for %s." % (config_name,))
        try:
            os.stat(file_path)
        except OSError as e:
@@ -128,9 +125,7 @@ class Config(object):
            if e.errno != errno.EEXIST:
                raise
        if not os.path.isdir(dir_path):
-            raise ConfigError(
-                "%s is not a directory" % (dir_path,)
-            )
+            raise ConfigError("%s is not a directory" % (dir_path,))
        return dir_path

    @classmethod
@@ -156,21 +151,20 @@ class Config(object):
        return results

    def generate_config(
-            self,
-            config_dir_path,
-            server_name,
-            is_generating_file,
-            report_stats=None,
+        self, config_dir_path, server_name, is_generating_file, report_stats=None
    ):
        default_config = "# vim:ft=yaml\n"

-        default_config += "\n\n".join(dedent(conf) for conf in self.invoke_all(
-            "default_config",
-            config_dir_path=config_dir_path,
-            server_name=server_name,
-            is_generating_file=is_generating_file,
-            report_stats=report_stats,
-        ))
+        default_config += "\n\n".join(
+            dedent(conf)
+            for conf in self.invoke_all(
+                "default_config",
+                config_dir_path=config_dir_path,
+                server_name=server_name,
+                is_generating_file=is_generating_file,
+                report_stats=report_stats,
+            )
+        )

        config = yaml.load(default_config)

@@ -178,23 +172,22 @@ class Config(object):

    @classmethod
    def load_config(cls, description, argv):
-        config_parser = argparse.ArgumentParser(
-            description=description,
-        )
+        config_parser = argparse.ArgumentParser(description=description)
        config_parser.add_argument(
-            "-c", "--config-path",
+            "-c",
+            "--config-path",
            action="append",
            metavar="CONFIG_FILE",
            help="Specify config file. Can be given multiple times and"
-                 " may specify directories containing *.yaml files."
+            " may specify directories containing *.yaml files.",
        )

        config_parser.add_argument(
            "--keys-directory",
            metavar="DIRECTORY",
            help="Where files such as certs and signing keys are stored when"
-                 " their location is given explicitly in the config."
-                 " Defaults to the directory containing the last config file",
+            " their location is given explicitly in the config."
+            " Defaults to the directory containing the last config file",
        )

        config_args = config_parser.parse_args(argv)
@@ -203,9 +196,7 @@ class Config(object):

        obj = cls()
        obj.read_config_files(
-            config_files,
-            keys_directory=config_args.keys_directory,
-            generate_keys=False,
+            config_files, keys_directory=config_args.keys_directory, generate_keys=False
        )
        return obj

@@ -213,38 +204,38 @@ class Config(object):
    def load_or_generate_config(cls, description, argv):
        config_parser = argparse.ArgumentParser(add_help=False)
        config_parser.add_argument(
-            "-c", "--config-path",
+            "-c",
+            "--config-path",
            action="append",
            metavar="CONFIG_FILE",
            help="Specify config file. Can be given multiple times and"
-                 " may specify directories containing *.yaml files."
+            " may specify directories containing *.yaml files.",
        )
        config_parser.add_argument(
            "--generate-config",
            action="store_true",
-            help="Generate a config file for the server name"
+            help="Generate a config file for the server name",
        )
        config_parser.add_argument(
            "--report-stats",
            action="store",
            help="Whether the generated config reports anonymized usage statistics",
-            choices=["yes", "no"]
+            choices=["yes", "no"],
        )
        config_parser.add_argument(
            "--generate-keys",
            action="store_true",
-            help="Generate any missing key files then exit"
+            help="Generate any missing key files then exit",
        )
        config_parser.add_argument(
            "--keys-directory",
            metavar="DIRECTORY",
            help="Used with 'generate-*' options to specify where files such as"
-                 " certs and signing keys should be stored in, unless explicitly"
-                 " specified in the config."
+            " certs and signing keys should be stored in, unless explicitly"
+            " specified in the config.",
        )
        config_parser.add_argument(
-            "-H", "--server-name",
-            help="The server name to generate a config file for"
+            "-H", "--server-name", help="The server name to generate a config file for"
        )
        config_args, remaining_args = config_parser.parse_known_args(argv)

@@ -257,8 +248,8 @@ class Config(object):
        if config_args.generate_config:
            if config_args.report_stats is None:
                config_parser.error(
-                    "Please specify either --report-stats=yes or --report-stats=no\n\n" +
-                    MISSING_REPORT_STATS_SPIEL
+                    "Please specify either --report-stats=yes or --report-stats=no\n\n"
+                    + MISSING_REPORT_STATS_SPIEL
                )
            if not config_files:
                config_parser.error(
@@ -287,26 +278,32 @@ class Config(object):
                        config_dir_path=config_dir_path,
                        server_name=server_name,
                        report_stats=(config_args.report_stats == "yes"),
-                        is_generating_file=True
+                        is_generating_file=True,
                    )
                    obj.invoke_all("generate_files", config)
                    config_file.write(config_str)
-                print((
-                    "A config file has been generated in %r for server name"
-                    " %r with corresponding SSL keys and self-signed"
-                    " certificates. Please review this file and customise it"
-                    " to your needs."
-                ) % (config_path, server_name))
+                print(
+                    (
+                        "A config file has been generated in %r for server name"
+                        " %r with corresponding SSL keys and self-signed"
+                        " certificates. Please review this file and customise it"
+                        " to your needs."
+                    )
+                    % (config_path, server_name)
+                )
                print(
                    "If this server name is incorrect, you will need to"
                    " regenerate the SSL certificates"
                )
                return
            else:
-                print((
-                    "Config file %r already exists. Generating any missing key"
-                    " files."
-                ) % (config_path,))
+                print(
+                    (
+                        "Config file %r already exists. Generating any missing key"
+                        " files."
+                    )
+                    % (config_path,)
+                )
                generate_keys = True

        parser = argparse.ArgumentParser(
@@ -338,8 +335,7 @@ class Config(object):

        return obj

-    def read_config_files(self, config_files, keys_directory=None,
-                          generate_keys=False):
+    def read_config_files(self, config_files, keys_directory=None, generate_keys=False):
        if not keys_directory:
            keys_directory = os.path.dirname(config_files[-1])

@@ -364,8 +360,9 @@ class Config(object):

        if "report_stats" not in config:
            raise ConfigError(
-                MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS + "\n" +
-                MISSING_REPORT_STATS_SPIEL
+                MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS
+                + "\n"
+                + MISSING_REPORT_STATS_SPIEL
            )

        if generate_keys:
@@ -399,16 +396,16 @@ def find_config_files(search_paths):
                for entry in os.listdir(config_path):
                    entry_path = os.path.join(config_path, entry)
                    if not os.path.isfile(entry_path):
-                        print (
-                            "Found subdirectory in config directory: %r. IGNORING."
-                        ) % (entry_path, )
+                        err = "Found subdirectory in config directory: %r. IGNORING."
+                        print(err % (entry_path,))
                        continue

                    if not entry.endswith(".yaml"):
-                        print (
-                            "Found file in config directory that does not"
-                            " end in '.yaml': %r. IGNORING."
-                        ) % (entry_path, )
+                        err = (
+                            "Found file in config directory that does not end in "
+                            "'.yaml': %r. IGNORING."
+                        )
+                        print(err % (entry_path,))
                        continue

                    files.append(entry_path)
@@ -42,6 +42,14 @@ DEFAULT_CONFIG = """\
 # until the user consents to the privacy policy. The value of the setting is
 # used as the text of the error.
 #
+# 'require_at_registration', if enabled, will add a step to the registration
+# process, similar to how captcha works. Users will be required to accept the
+# policy before their account is created.
+#
+# 'policy_name' is the display name of the policy users will see when registering
+# for an account. Has no effect unless `require_at_registration` is enabled.
+# Defaults to "Privacy Policy".
+#
 # user_consent:
 #   template_dir: res/templates/privacy
 #   version: 1.0
@@ -54,6 +62,8 @@ DEFAULT_CONFIG = """\
 #   block_events_error: >-
 #     To continue using this homeserver you must review and agree to the
 #     terms and conditions at %(consent_uri)s
+#   require_at_registration: False
+#   policy_name: Privacy Policy
 #
 """

@@ -67,6 +77,8 @@ class ConsentConfig(Config):
        self.user_consent_server_notice_content = None
        self.user_consent_server_notice_to_guests = False
        self.block_events_without_consent_error = None
+        self.user_consent_at_registration = False
+        self.user_consent_policy_name = "Privacy Policy"

    def read_config(self, config):
        consent_config = config.get("user_consent")
@@ -83,6 +95,12 @@ class ConsentConfig(Config):
        self.user_consent_server_notice_to_guests = bool(consent_config.get(
            "send_server_notice_to_guests", False,
        ))
+        self.user_consent_at_registration = bool(consent_config.get(
+            "require_at_registration", False,
+        ))
+        self.user_consent_policy_name = consent_config.get(
+            "policy_name", "Privacy Policy",
+        )

    def default_config(self, **kwargs):
        return DEFAULT_CONFIG
@@ -19,19 +19,13 @@ from __future__ import print_function
 import email.utils
 import logging
 import os
-import sys
-import textwrap

-from ._base import Config
+import pkg_resources
+
+from ._base import Config, ConfigError

 logger = logging.getLogger(__name__)

-TEMPLATE_DIR_WARNING = """\
-WARNING: The email notifier is configured to look for templates in '%(template_dir)s',
-but no templates could be found there. We will fall back to using the example templates;
-to get rid of this warning, leave 'email.template_dir' unset.
-"""
-

 class EmailConfig(Config):
    def read_config(self, config):
@@ -78,20 +72,22 @@ class EmailConfig(Config):
            self.email_notif_template_html = email_config["notif_template_html"]
            self.email_notif_template_text = email_config["notif_template_text"]

-            self.email_template_dir = email_config.get("template_dir")
-
-            # backwards-compatibility hack
-            if (
-                self.email_template_dir == "res/templates"
-                and not os.path.isfile(
-                    os.path.join(self.email_template_dir, self.email_notif_template_text)
+            template_dir = email_config.get("template_dir")
+            # we need an absolute path, because we change directory after starting (and
+            # we don't yet know what auxilliary templates like mail.css we will need).
+            # (Note that loading as package_resources with jinja.PackageLoader doesn't
+            # work for the same reason.)
+            if not template_dir:
+                template_dir = pkg_resources.resource_filename(
+                    'synapse', 'res/templates'
                )
-            ):
-                t = TEMPLATE_DIR_WARNING % {
-                    "template_dir": self.email_template_dir,
-                }
-                print(textwrap.fill(t, width=80) + "\n", file=sys.stderr)
-                self.email_template_dir = None
+            template_dir = os.path.abspath(template_dir)
+
+            for f in self.email_notif_template_text, self.email_notif_template_html:
+                p = os.path.join(template_dir, f)
+                if not os.path.isfile(p):
+                    raise ConfigError("Unable to find email template file %s" % (p, ))
+            self.email_template_dir = template_dir

            self.email_notif_for_new_users = email_config.get(
                "notif_for_new_users", True
@@ -31,6 +31,7 @@ from .push import PushConfig
 from .ratelimiting import RatelimitConfig
 from .registration import RegistrationConfig
 from .repository import ContentRepositoryConfig
+from .room_directory import RoomDirectoryConfig
 from .saml2 import SAML2Config
 from .server import ServerConfig
 from .server_notices_config import ServerNoticesConfig
@@ -49,7 +50,7 @@ class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig,
                       WorkerConfig, PasswordAuthProviderConfig, PushConfig,
                       SpamCheckerConfig, GroupsConfig, UserDirectoryConfig,
                       ConsentConfig,
-                       ServerNoticesConfig,
+                       ServerNoticesConfig, RoomDirectoryConfig,
                       ):
    pass

@@ -50,6 +50,7 @@ handlers:
        maxBytes: 104857600
        backupCount: 10
        filters: [context]
+        encoding: utf8
    console:
        class: logging.StreamHandler
        formatter: precise
@@ -15,10 +15,10 @@

 from distutils.util import strtobool

+from synapse.config._base import Config, ConfigError
+from synapse.types import RoomAlias
 from synapse.util.stringutils import random_string_with_symbols

-from ._base import Config
-

 class RegistrationConfig(Config):

@@ -44,6 +44,10 @@ class RegistrationConfig(Config):
        )

        self.auto_join_rooms = config.get("auto_join_rooms", [])
+        for room_alias in self.auto_join_rooms:
+            if not RoomAlias.is_valid(room_alias):
+                raise ConfigError('Invalid auto_join_rooms entry %s' % (room_alias,))
+        self.autocreate_auto_join_rooms = config.get("autocreate_auto_join_rooms", True)

    def default_config(self, **kwargs):
        registration_shared_secret = random_string_with_symbols(50)
@@ -98,6 +102,13 @@ class RegistrationConfig(Config):
        # to these rooms
        #auto_join_rooms:
        #    - "#example:example.com"
+
+        # Where auto_join_rooms are specified, setting this flag ensures that the
+        # the rooms exist by creating them when the first user on the
+        # homeserver registers.
+        # Setting to false means that if the rooms are not manually created,
+        # users cannot be auto-joined since they do not exist.
+        autocreate_auto_join_rooms: true
        """ % locals()

    def add_arguments(self, parser):
@@ -178,7 +178,7 @@ class ContentRepositoryConfig(Config):
    def default_config(self, **kwargs):
        media_store = self.default_path("media_store")
        uploads_path = self.default_path("uploads")
-        return """
+        return r"""
        # Directory where uploaded images and attachments are stored.
        media_store_path: "%(media_store)s"

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.util import glob_to_regex
+
+from ._base import Config, ConfigError
+
+
+class RoomDirectoryConfig(Config):
+    def read_config(self, config):
+        alias_creation_rules = config["alias_creation_rules"]
+
+        self._alias_creation_rules = [
+            _AliasRule(rule)
+            for rule in alias_creation_rules
+        ]
+
+    def default_config(self, config_dir_path, server_name, **kwargs):
+        return """
+        # The `alias_creation` option controls who's allowed to create aliases
+        # on this server.
+        #
+        # The format of this option is a list of rules that contain globs that
+        # match against user_id and the new alias (fully qualified with server
+        # name). The action in the first rule that matches is taken, which can
+        # currently either be "allow" or "deny".
+        #
+        # If no rules match the request is denied.
+        alias_creation_rules:
+            - user_id: "*"
+              alias: "*"
+              action: allow
+        """
+
+    def is_alias_creation_allowed(self, user_id, alias):
+        """Checks if the given user is allowed to create the given alias
+
+        Args:
+            user_id (str)
+            alias (str)
+
+        Returns:
+            boolean: True if user is allowed to crate the alias
+        """
+        for rule in self._alias_creation_rules:
+            if rule.matches(user_id, alias):
+                return rule.action == "allow"
+
+        return False
+
+
+class _AliasRule(object):
+    def __init__(self, rule):
+        action = rule["action"]
+        user_id = rule["user_id"]
+        alias = rule["alias"]
+
+        if action in ("allow", "deny"):
+            self.action = action
+        else:
+            raise ConfigError(
+                "alias_creation_rules rules can only have action of 'allow'"
+                " or 'deny'"
+            )
+
+        try:
+            self._user_id_regex = glob_to_regex(user_id)
+            self._alias_regex = glob_to_regex(alias)
+        except Exception as e:
+            raise ConfigError("Failed to parse glob into regex: %s", e)
+
+    def matches(self, user_id, alias):
+        """Tests if this rule matches the given user_id and alias.
+
+        Args:
+            user_id (str)
+            alias (str)
+
+        Returns:
+            boolean
+        """
+
+        # Note: The regexes are anchored at both ends
+        if not self._user_id_regex.match(user_id):
+            return False
+
+        if not self._alias_regex.match(alias):
+            return False
+
+        return True
@@ -15,6 +15,8 @@

 import logging

+from six.moves import urllib
+
 from canonicaljson import json

 from twisted.internet import defer, reactor
@@ -28,15 +30,15 @@ from synapse.util import logcontext

 logger = logging.getLogger(__name__)

-KEY_API_V1 = b"/_matrix/key/v1/"
+KEY_API_V2 = "/_matrix/key/v2/server/%s"


@defer.inlineCallbacks
-def fetch_server_key(server_name, tls_client_options_factory, path=KEY_API_V1):
+def fetch_server_key(server_name, tls_client_options_factory, key_id):
    """Fetch the keys for a remote server."""

    factory = SynapseKeyClientFactory()
-    factory.path = path
+    factory.path = KEY_API_V2 % (urllib.parse.quote(key_id), )
    factory.host = server_name
    endpoint = matrix_federation_endpoint(
        reactor, server_name, tls_client_options_factory, timeout=30
@@ -55,7 +57,7 @@ def fetch_server_key(server_name, tls_client_options_factory, path=KEY_API_V1):
                raise IOError("Cannot get key for %r" % server_name)
        except (ConnectError, DomainError) as e:
            logger.warn("Error getting key for %r: %s", server_name, e)
-        except Exception as e:
+        except Exception:
            logger.exception("Error getting key for %r", server_name)
    raise IOError("Cannot get key for %r" % server_name)

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2017 New Vector Ltd.
+# Copyright 2017, 2018 New Vector Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,8 +18,6 @@ import hashlib
 import logging
 from collections import namedtuple

-from six.moves import urllib
-
 from signedjson.key import (
    decode_verify_key_bytes,
    encode_verify_key_base64,
@@ -395,32 +393,13 @@ class Keyring(object):

    @defer.inlineCallbacks
    def get_keys_from_server(self, server_name_and_key_ids):
-        @defer.inlineCallbacks
-        def get_key(server_name, key_ids):
-            keys = None
-            try:
-                keys = yield self.get_server_verify_key_v2_direct(
-                    server_name, key_ids
-                )
-            except Exception as e:
-                logger.info(
-                    "Unable to get key %r for %r directly: %s %s",
-                    key_ids, server_name,
-                    type(e).__name__, str(e),
-                )
-
-            if not keys:
-                keys = yield self.get_server_verify_key_v1_direct(
-                    server_name, key_ids
-                )
-
-                keys = {server_name: keys}
-
-            defer.returnValue(keys)
-
        results = yield logcontext.make_deferred_yieldable(defer.gatherResults(
            [
-                run_in_background(get_key, server_name, key_ids)
+                run_in_background(
+                    self.get_server_verify_key_v2_direct,
+                    server_name,
+                    key_ids,
+                )
                for server_name, key_ids in server_name_and_key_ids
            ],
            consumeErrors=True,
@@ -525,10 +504,7 @@ class Keyring(object):
                continue

            (response, tls_certificate) = yield fetch_server_key(
-                server_name, self.hs.tls_client_options_factory,
-                path=("/_matrix/key/v2/server/%s" % (
-                    urllib.parse.quote(requested_key_id),
-                )).encode("ascii"),
+                server_name, self.hs.tls_client_options_factory, requested_key_id
            )

            if (u"signatures" not in response
@@ -657,78 +633,6 @@ class Keyring(object):

        defer.returnValue(results)

-    @defer.inlineCallbacks
-    def get_server_verify_key_v1_direct(self, server_name, key_ids):
-        """Finds a verification key for the server with one of the key ids.
-        Args:
-            server_name (str): The name of the server to fetch a key for.
-            keys_ids (list of str): The key_ids to check for.
-        """
-
-        # Try to fetch the key from the remote server.
-
-        (response, tls_certificate) = yield fetch_server_key(
-            server_name, self.hs.tls_client_options_factory
-        )
-
-        # Check the response.
-
-        x509_certificate_bytes = crypto.dump_certificate(
-            crypto.FILETYPE_ASN1, tls_certificate
-        )
-
-        if ("signatures" not in response
-                or server_name not in response["signatures"]):
-            raise KeyLookupError("Key response not signed by remote server")
-
-        if "tls_certificate" not in response:
-            raise KeyLookupError("Key response missing TLS certificate")
-
-        tls_certificate_b64 = response["tls_certificate"]
-
-        if encode_base64(x509_certificate_bytes) != tls_certificate_b64:
-            raise KeyLookupError("TLS certificate doesn't match")
-
-        # Cache the result in the datastore.
-
-        time_now_ms = self.clock.time_msec()
-
-        verify_keys = {}
-        for key_id, key_base64 in response["verify_keys"].items():
-            if is_signing_algorithm_supported(key_id):
-                key_bytes = decode_base64(key_base64)
-                verify_key = decode_verify_key_bytes(key_id, key_bytes)
-                verify_key.time_added = time_now_ms
-                verify_keys[key_id] = verify_key
-
-        for key_id in response["signatures"][server_name]:
-            if key_id not in response["verify_keys"]:
-                raise KeyLookupError(
-                    "Key response must include verification keys for all"
-                    " signatures"
-                )
-            if key_id in verify_keys:
-                verify_signed_json(
-                    response,
-                    server_name,
-                    verify_keys[key_id]
-                )
-
-        yield self.store.store_server_certificate(
-            server_name,
-            server_name,
-            time_now_ms,
-            tls_certificate,
-        )
-
-        yield self.store_keys(
-            server_name=server_name,
-            from_server=server_name,
-            verify_keys=verify_keys,
-        )
-
-        defer.returnValue(verify_keys)
-
    def store_keys(self, server_name, from_server, verify_keys):
        """Store a collection of verify keys for a given server
        Args:
@@ -46,7 +46,7 @@ def check(event, auth_events, do_sig_check=True, do_size_check=True):
    if not hasattr(event, "room_id"):
        raise AuthError(500, "Event has no room_id: %s" % event)

-    if do_sig_check:
+    if False and do_sig_check:  # Disable all sig checks for meshsim
        sender_domain = get_domain_from_id(event.sender)
        event_id_domain = get_domain_from_id(event.event_id)

@@ -200,11 +200,11 @@ def _is_membership_change_allowed(event, auth_events):
    membership = event.content["membership"]

    # Check if this is the room creator joining:
-    if len(event.prev_events) == 1 and Membership.JOIN == membership:
+    if len(event.prev_event_ids()) == 1 and Membership.JOIN == membership:
        # Get room creation event:
        key = (EventTypes.Create, "", )
        create = auth_events.get(key)
-        if create and event.prev_events[0][0] == create.event_id:
+        if create and event.prev_event_ids()[0] == create.event_id:
            if create.content["creator"] == event.state_key:
                return

@@ -690,7 +690,7 @@ def auth_types_for_event(event):
    auth_types = []

    auth_types.append((EventTypes.PowerLevels, "", ))
-    auth_types.append((EventTypes.Member, event.user_id, ))
+    auth_types.append((EventTypes.Member, event.sender, ))
    auth_types.append((EventTypes.Create, "", ))

    if event.type == EventTypes.Member:
@@ -159,6 +159,24 @@ class EventBase(object):
    def keys(self):
        return six.iterkeys(self._event_dict)

+    def prev_event_ids(self):
+        """Returns the list of prev event IDs. The order matches the order
+        specified in the event, though there is no meaning to it.
+
+        Returns:
+            list[str]: The list of event IDs of this event's prev_events
+        """
+        return [e for e, _ in self.prev_events]
+
+    def auth_event_ids(self):
+        """Returns the list of auth event IDs. The order matches the order
+        specified in the event, though there is no meaning to it.
+
+        Returns:
+            list[str]: The list of event IDs of this event's auth_events
+        """
+        return [e for e, _ in self.auth_events]
+

 class FrozenEvent(EventBase):
    def __init__(self, event_dict, internal_metadata_dict={}, rejected_reason=None):
@@ -14,9 +14,9 @@
 # limitations under the License.

 import copy
+import string

 from synapse.types import EventID
-from synapse.util.stringutils import random_string

 from . import EventBase, FrozenEvent, _event_dict_property

@@ -49,10 +49,10 @@ class EventBuilderFactory(object):
        self.event_id_count = 0

    def create_event_id(self):
-        i = str(self.event_id_count)
+        i = self.event_id_count
        self.event_id_count += 1

-        local_part = str(int(self.clock.time())) + i + random_string(5)
+        local_part = _encode_id(i)

        e_id = EventID(local_part, self.hostname)

@@ -73,3 +73,19 @@ class EventBuilderFactory(object):
        key_values["signatures"] = {}

        return EventBuilder(key_values=key_values,)
+
+
+def _numberToBase(n, b):
+    if n == 0:
+        return [0]
+    digits = []
+    while n:
+        digits.append(int(n % b))
+        n //= b
+    return digits[::-1]
+
+
+def _encode_id(i):
+    digits = string.digits + string.ascii_letters
+    val_slice = _numberToBase(i, len(digits))
+    return "".join(digits[x] for x in val_slice)
@@ -26,7 +26,7 @@ from synapse.crypto.event_signing import check_event_content_hash
 from synapse.events import FrozenEvent
 from synapse.events.utils import prune_event
 from synapse.http.servlet import assert_params_in_dict
-from synapse.types import get_domain_from_id
+from synapse.types import get_domain_from_id, EventID
 from synapse.util import logcontext, unwrapFirstError

 logger = logging.getLogger(__name__)
@@ -136,6 +136,7 @@ class FederationBase(object):
              * throws a SynapseError if the signature check failed.
            The deferreds run their callbacks in the sentinel logcontext.
        """
+        return [defer.succeed(p) for p in pdus]
        deferreds = _check_sigs_on_pdus(self.keyring, pdus)

        ctx = logcontext.LoggingContext.current_context()
@@ -317,7 +318,7 @@ def event_from_pdu_json(pdu_json, outlier=False):

    depth = pdu_json['depth']
    if not isinstance(depth, six.integer_types):
-        raise SynapseError(400, "Depth %r not an intger" % (depth, ),
+        raise SynapseError(400, "Depth %r not an integer" % (depth, ),
                           Codes.BAD_JSON)

    if depth < 0:
@@ -325,6 +326,31 @@ def event_from_pdu_json(pdu_json, outlier=False):
    elif depth > MAX_DEPTH:
        raise SynapseError(400, "Depth too large", Codes.BAD_JSON)

+    event_id = pdu_json["event_id"]
+    if event_id[0] != "$":
+        pdu_json["event_id"] = EventID(
+            event_id,
+            get_domain_from_id(pdu_json["sender"]),
+        ).to_string()
+        event_id = pdu_json["event_id"]
+
+    if "auth_events" in pdu_json:
+        pdu_json["auth_events"] = [
+            (e, {}) if isinstance(e, six.string_types) else e
+            for e in pdu_json["auth_events"]
+        ]
+
+    if "prev_events" in pdu_json:
+        pdu_json["prev_events"] = [
+            (e, {}) if isinstance(e, six.string_types) else e
+            for e in pdu_json["prev_events"]
+        ]
+
+    if "origin" not in pdu_json:
+        pdu_json["origin"] = get_domain_from_id(pdu_json["sender"])
+
+    logger.info("Unmangled event to: %s", pdu_json)
+
    event = FrozenEvent(
        pdu_json
    )
@@ -39,6 +39,7 @@ from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 from synapse.util.logutils import log_function
 from synapse.util.retryutils import NotRetryingDestination
+from synapse.federation.units import _mangle_pdu

 logger = logging.getLogger(__name__)

@@ -696,7 +697,7 @@ class FederationClient(FederationBase):
                destination=destination,
                room_id=room_id,
                event_id=event_id,
-                content=pdu.get_pdu_json(time_now),
+                content=_mangle_pdu(pdu.get_pdu_json(time_now)),
            )
        except HttpResponseException as e:
            if e.code == 403:
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import re

 import six
 from six import iteritems
@@ -37,18 +36,20 @@ from synapse.api.errors import (
 from synapse.crypto.event_signing import compute_event_signature
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.federation.persistence import TransactionActions
-from synapse.federation.units import Edu, Transaction
+from synapse.federation.units import Edu, Transaction, _mangle_pdu
 from synapse.http.endpoint import parse_server_name
 from synapse.replication.http.federation import (
    ReplicationFederationSendEduRestServlet,
    ReplicationGetQueryRestServlet,
 )
 from synapse.types import get_domain_from_id
+from synapse.util import glob_to_regex
 from synapse.util.async_helpers import Linearizer, concurrently_execute
 from synapse.util.caches.response_cache import ResponseCache
 from synapse.util.logcontext import nested_logging_context
 from synapse.util.logutils import log_function

+
 # when processing incoming transactions, we try to handle multiple rooms in
 # parallel, up to this limit.
 TRANSACTION_CONCURRENCY_LIMIT = 10
@@ -162,8 +163,30 @@ class FederationServer(FederationBase):
                p["age_ts"] = request_time - int(p["age"])
                del p["age"]

+            # We try and pull out an event ID so that if later checks fail we
+            # can log something sensible. We don't mandate an event ID here in
+            # case future event formats get rid of the key.
+            possible_event_id = p.get("event_id", "<Unknown>")
+
+            # Now we get the room ID so that we can check that we know the
+            # version of the room.
+            room_id = p.get("room_id")
+            if not room_id:
+                logger.info(
+                    "Ignoring PDU as does not have a room_id. Event ID: %s",
+                    possible_event_id,
+                )
+                continue
+
+            try:
+                # In future we will actually use the room version to parse the
+                # PDU into an event.
+                yield self.store.get_room_version(room_id)
+            except NotFoundError:
+                logger.info("Ignoring PDU for unknown room_id: %s", room_id)
+                continue
+
            event = event_from_pdu_json(p)
-            room_id = event.room_id
            pdus_by_room.setdefault(room_id, []).append(event)

        pdu_results = {}
@@ -211,7 +234,16 @@ class FederationServer(FederationBase):
        )

        if hasattr(transaction, "edus"):
-            for edu in (Edu(**x) for x in transaction.edus):
+            logger.info("Got edus: %s", transaction.edus)
+
+            edus = []
+            for x in transaction.edus:
+                try:
+                    edus.append(Edu(**x))
+                except Exception:
+                    logger.exception("Failed to handle EDU: %s", x)
+
+            for edu in edus:
                yield self.received_edu(
                    origin,
                    edu.edu_type,
@@ -307,8 +339,8 @@ class FederationServer(FederationBase):
                )

        defer.returnValue({
-            "pdus": [pdu.get_pdu_json() for pdu in pdus],
-            "auth_chain": [pdu.get_pdu_json() for pdu in auth_chain],
+            "pdus": [_mangle_pdu(pdu.get_pdu_json()) for pdu in pdus],
+            "auth_chain": [_mangle_pdu(pdu.get_pdu_json()) for pdu in auth_chain],
        })

    @defer.inlineCallbacks
@@ -323,11 +355,6 @@ class FederationServer(FederationBase):
        else:
            defer.returnValue((404, ""))

-    @defer.inlineCallbacks
-    @log_function
-    def on_pull_request(self, origin, versions):
-        raise NotImplementedError("Pull transactions not implemented")
-
    @defer.inlineCallbacks
    def on_query_request(self, query_type, args):
        received_queries_counter.labels(query_type).inc()
@@ -358,7 +385,7 @@ class FederationServer(FederationBase):
        yield self.check_server_matches_acl(origin_host, pdu.room_id)
        ret_pdu = yield self.handler.on_invite_request(origin, pdu)
        time_now = self._clock.time_msec()
-        defer.returnValue((200, {"event": ret_pdu.get_pdu_json(time_now)}))
+        defer.returnValue((200, {"event": _mangle_pdu(ret_pdu.get_pdu_json(time_now))}))

    @defer.inlineCallbacks
    def on_send_join_request(self, origin, content):
@@ -372,9 +399,9 @@ class FederationServer(FederationBase):
        res_pdus = yield self.handler.on_send_join_request(origin, pdu)
        time_now = self._clock.time_msec()
        defer.returnValue((200, {
-            "state": [p.get_pdu_json(time_now) for p in res_pdus["state"]],
+            "state": [_mangle_pdu(p.get_pdu_json(time_now)) for p in res_pdus["state"]],
            "auth_chain": [
-                p.get_pdu_json(time_now) for p in res_pdus["auth_chain"]
+                _mangle_pdu(p.get_pdu_json(time_now)) for p in res_pdus["auth_chain"]
            ],
        }))

@@ -407,7 +434,7 @@ class FederationServer(FederationBase):
            time_now = self._clock.time_msec()
            auth_pdus = yield self.handler.on_event_auth(event_id)
            res = {
-                "auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus],
+                "auth_chain": [_mangle_pdu(a.get_pdu_json(time_now)) for a in auth_pdus],
            }
        defer.returnValue((200, res))

@@ -456,7 +483,7 @@ class FederationServer(FederationBase):
            time_now = self._clock.time_msec()
            send_content = {
                "auth_chain": [
-                    e.get_pdu_json(time_now)
+                    _mangle_pdu(e.get_pdu_json(time_now))
                    for e in ret["auth_chain"]
                ],
                "rejects": ret.get("rejects", []),
@@ -532,7 +559,7 @@ class FederationServer(FederationBase):
            time_now = self._clock.time_msec()

        defer.returnValue({
-            "events": [ev.get_pdu_json(time_now) for ev in missing_events],
+            "events": [_mangle_pdu(ev.get_pdu_json(time_now)) for ev in missing_events],
        })

    @log_function
@@ -729,22 +756,10 @@ def _acl_entry_matches(server_name, acl_entry):
    if not isinstance(acl_entry, six.string_types):
        logger.warn("Ignoring non-str ACL entry '%s' (is %s)", acl_entry, type(acl_entry))
        return False
-    regex = _glob_to_regex(acl_entry)
+    regex = glob_to_regex(acl_entry)
    return regex.match(server_name)


-def _glob_to_regex(glob):
-    res = ''
-    for c in glob:
-        if c == '*':
-            res = res + '.*'
-        elif c == '?':
-            res = res + '.'
-        else:
-            res = res + re.escape(c)
-    return re.compile(res + "\\Z", re.IGNORECASE)
-
-
 class FederationHandlerRegistry(object):
    """Allows classes to register themselves as handlers for a given EDU or
    query type for incoming federation traffic.
@@ -800,7 +815,7 @@ class FederationHandlerRegistry(object):
            yield handler(origin, content)
        except SynapseError as e:
            logger.info("Failed to handle edu %r: %r", edu_type, e)
-        except Exception as e:
+        except Exception:
            logger.exception("Failed to handle edu %r", edu_type)

    def on_query(self, query_type, args):
@@ -14,6 +14,8 @@
 # limitations under the License.
 import datetime
 import logging
+import string
+import json

 from six import itervalues

@@ -42,6 +44,8 @@ from .units import Edu, Transaction

 logger = logging.getLogger(__name__)

+pdu_logger = logging.getLogger("synapse.federation.pdu_destination_logger")
+
 sent_pdus_destination_dist_count = Counter(
    "synapse_federation_client_sent_pdu_destinations:count", ""
 )
@@ -128,7 +132,7 @@ class TransactionQueue(object):
        self.last_device_list_stream_id_by_dest = {}

        # HACK to get unique tx id
-        self._next_txn_id = int(self.clock.time_msec())
+        self._next_txn_id = 1

        self._order = 1

@@ -183,9 +187,7 @@ class TransactionQueue(object):
                        # banned then it won't receive the event because it won't
                        # be in the room after the ban.
                        destinations = yield self.state.get_current_hosts_in_room(
-                            event.room_id, latest_event_ids=[
-                                prev_id for prev_id, _ in event.prev_events
-                            ],
+                            event.room_id, latest_event_ids=event.prev_event_ids(),
                        )
                    except Exception:
                        logger.exception(
@@ -262,6 +264,15 @@ class TransactionQueue(object):
        destinations.discard(self.server_name)
        logger.debug("Sending to: %s", str(destinations))

+        pdu_logger.info(
+            "SendingPDU",
+            extra={
+                "event_id": pdu.event_id, "room_id": pdu.room_id,
+                "destinations": json.dumps(list(destinations)),
+                "server": self.server_name,
+            },
+        )
+
        if not destinations:
            return

@@ -441,16 +452,22 @@ class TransactionQueue(object):
                pending_pdus = self.pending_pdus_by_dest.pop(destination, [])

                # We can only include at most 50 PDUs per transactions
-                pending_pdus, leftover_pdus = pending_pdus[:50], pending_pdus[50:]
+                pending_pdus, leftover_pdus = pending_pdus[-5:], pending_pdus[:-5]
                if leftover_pdus:
-                    self.pending_pdus_by_dest[destination] = leftover_pdus
+                    # self.pending_pdus_by_dest[destination] = leftover_pdus
+                    for _, _, p_span in leftover_pdus:
+                        p_span.set_tag("success", False)
+                        p_span.log_kv({"result": "dropped"})
+                        p_span.finish()
+
+                logger.info("TX [%s] Sending PDUs: %s", destination, pending_pdus)

                pending_edus = self.pending_edus_by_dest.pop(destination, [])

                # We can only include at most 100 EDUs per transactions
-                pending_edus, leftover_edus = pending_edus[:100], pending_edus[100:]
-                if leftover_edus:
-                    self.pending_edus_by_dest[destination] = leftover_edus
+                pending_edus, leftover_edus = pending_edus[-5:], pending_edus[:-5]
+                # if leftover_edus:
+                #     self.pending_edus_by_dest[destination] = leftover_edus

                pending_presence = self.pending_presence_by_dest.pop(destination, {})

@@ -521,7 +538,7 @@ class TransactionQueue(object):
        except FederationDeniedError as e:
            logger.info(e)
        except Exception as e:
-            logger.warn(
+            logger.exception(
                "TX [%s] Failed to send transaction: %s",
                destination,
                e,
@@ -576,6 +593,7 @@ class TransactionQueue(object):

        success = True

+        logger.debug("TX [%s] _attempt_new_transaction", destination)
        logger.debug("TX [%s] _attempt_new_transaction", destination)

        txn_id = str(self._next_txn_id)
@@ -633,14 +651,6 @@ class TransactionQueue(object):
                transaction, json_data_cb
            )
            code = 200
-
-            if response:
-                for e_id, r in response.get("pdus", {}).items():
-                    if "error" in r:
-                        logger.warn(
-                            "Transaction returned error for %s: %s",
-                            e_id, r,
-                        )
        except HttpResponseException as e:
            code = e.code
            response = e.response
@@ -657,20 +667,41 @@ class TransactionQueue(object):
            destination, txn_id, code
        )

-        logger.debug("TX [%s] Sent transaction", destination)
-        logger.debug("TX [%s] Marking as delivered...", destination)
-
        yield self.transaction_actions.delivered(
            transaction, code, response
        )

-        logger.debug("TX [%s] Marked as delivered", destination)
+        logger.debug("TX [%s] {%s} Marked as delivered", destination, txn_id)

-        if code != 200:
+        if code == 200:
+            for e_id, r in response.get("pdus", {}).items():
+                if "error" in r:
+                    logger.warn(
+                        "TX [%s] {%s} Remote returned error for %s: %s",
+                        destination, txn_id, e_id, r,
+                    )
+        else:
            for p in pdus:
-                logger.info(
-                    "Failed to send event %s to %s", p.event_id, destination
+                logger.warn(
+                    "TX [%s] {%s} Failed to send event %s",
+                    destination, txn_id, p.event_id,
                )
            success = False

        defer.returnValue(success)
+
+
+def _numberToBase(n, b):
+    if n == 0:
+        return [0]
+    digits = []
+    while n:
+        digits.append(int(n % b))
+        n //= b
+    return digits[::-1]
+
+
+def _encode_id(i):
+    digits = string.digits + string.ascii_letters
+    val_slice = _numberToBase(i, len(digits))
+    return "".join(digits[x] for x in val_slice)
@@ -143,9 +143,17 @@ class TransportLayerClient(object):
            transaction (Transaction)

        Returns:
-            Deferred: Results of the deferred is a tuple in the form of
-            (response_code, response_body) where the response_body is a
-            python dict decoded from json
+            Deferred: Succeeds when we get a 2xx HTTP response. The result
+            will be the decoded JSON body.
+
+            Fails with ``HTTPRequestException`` if we get an HTTP response
+            code >= 300.
+
+            Fails with ``NotRetryingDestination`` if we are not yet ready
+            to retry this server.
+
+            Fails with ``FederationDeniedError`` if this destination
+            is not on our federation whitelist
        """
        logger.debug(
            "send_data dest=%s, txid=%s",
@@ -170,11 +178,6 @@ class TransportLayerClient(object):
            backoff_on_404=True,  # If we get a 404 the other side has gone
        )

-        logger.debug(
-            "send_data dest=%s, txid=%s, got response: 200",
-            transaction.destination, transaction.transaction_id,
-        )
-
        defer.returnValue(response)

    @defer.inlineCallbacks
@@ -125,7 +125,7 @@ class Authenticator(object):
                401, "Missing Authorization headers", Codes.UNAUTHORIZED,
            )

-        yield self.keyring.verify_json_for_server(origin, json_request)
+        # yield self.keyring.verify_json_for_server(origin, json_request)

        logger.info("Request from %s", origin)
        request.authenticated_entity = origin
@@ -362,14 +362,6 @@ class FederationSendServlet(BaseFederationServlet):
        defer.returnValue((code, response))


-class FederationPullServlet(BaseFederationServlet):
-    PATH = "/pull/"
-
-    # This is for when someone asks us for everything since version X
-    def on_GET(self, origin, content, query):
-        return self.handler.on_pull_request(query["origin"][0], query["v"])
-
-
 class FederationEventServlet(BaseFederationServlet):
    PATH = "/event/(?P<event_id>[^/]*)/"

@@ -1261,7 +1253,6 @@ class FederationGroupsSettingJoinPolicyServlet(BaseFederationServlet):

 FEDERATION_SERVLET_CLASSES = (
    FederationSendServlet,
-    FederationPullServlet,
    FederationEventServlet,
    FederationStateServlet,
    FederationStateIdsServlet,
@@ -19,6 +19,7 @@ server protocol.

 import logging

+from synapse.types import get_localpart_from_id, get_domain_from_id
 from synapse.util.jsonobject import JsonEncodedObject

 logger = logging.getLogger(__name__)
@@ -76,15 +77,14 @@ class Transaction(JsonEncodedObject):
    ]

    internal_keys = [
-        "transaction_id",
-        "destination",
-    ]
-
-    required_keys = [
        "transaction_id",
        "origin",
        "destination",
        "origin_server_ts",
+        "previous_ids",
+    ]
+
+    required_keys = [
        "pdus",
    ]

@@ -108,18 +108,33 @@ class Transaction(JsonEncodedObject):
        """ Used to create a new transaction. Will auto fill out
        transaction_id and origin_server_ts keys.
        """
-        if "origin_server_ts" not in kwargs:
-            raise KeyError(
-                "Require 'origin_server_ts' to construct a Transaction"
-            )
-        if "transaction_id" not in kwargs:
-            raise KeyError(
-                "Require 'transaction_id' to construct a Transaction"
-            )
-
-        for p in pdus:
-            p.transaction_id = kwargs["transaction_id"]
-
-        kwargs["pdus"] = [p.get_pdu_json() for p in pdus]
+        kwargs["pdus"] = [
+            _mangle_pdu(p.get_pdu_json())
+            for p in pdus
+        ]

        return Transaction(**kwargs)
+
+
+def _mangle_pdu(pdu_json):
+    pdu_json.pop("origin", None)
+    pdu_json.pop("hashes", None)
+    pdu_json.pop("signatures", None)
+    pdu_json.get("unsigned", {}).pop("age_ts", None)
+    pdu_json.get("unsigned", {}).pop("age", None)
+
+    pdu_json["auth_events"] = list(_strip_hashes(pdu_json["auth_events"]))
+    pdu_json["prev_events"] = list(_strip_hashes(pdu_json["prev_events"]))
+
+    if get_domain_from_id(pdu_json["event_id"]) == get_domain_from_id(pdu_json["sender"]):
+        pdu_json["event_id"] = get_localpart_from_id(pdu_json["event_id"])
+
+    logger.info("Mangled PDU: %s", pdu_json)
+
+    return pdu_json
+
+
+def _strip_hashes(iterable):
+    return (
+        e for e, hashes in iterable
+    )
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+import base64
+import secrets
 import logging
 import unicodedata

@@ -22,7 +24,7 @@ import bcrypt
 import pymacaroons
 from canonicaljson import json

-from twisted.internet import defer, threads
+from twisted.internet import defer
 from twisted.web.client import PartialDownloadError

 import synapse.util.stringutils as stringutils
@@ -37,8 +39,8 @@ from synapse.api.errors import (
 )
 from synapse.module_api import ModuleApi
 from synapse.types import UserID
+from synapse.util import logcontext
 from synapse.util.caches.expiringcache import ExpiringCache
-from synapse.util.logcontext import make_deferred_yieldable

 from ._base import BaseHandler

@@ -59,6 +61,7 @@ class AuthHandler(BaseHandler):
            LoginType.EMAIL_IDENTITY: self._check_email_identity,
            LoginType.MSISDN: self._check_msisdn,
            LoginType.DUMMY: self._check_dummy_auth,
+            LoginType.TERMS: self._check_terms_auth,
        }
        self.bcrypt_rounds = hs.config.bcrypt_rounds

@@ -431,6 +434,9 @@ class AuthHandler(BaseHandler):
    def _check_dummy_auth(self, authdict, _):
        return defer.succeed(True)

+    def _check_terms_auth(self, authdict, _):
+        return defer.succeed(True)
+
    @defer.inlineCallbacks
    def _check_threepid(self, medium, authdict):
        if 'threepid_creds' not in authdict:
@@ -462,6 +468,22 @@ class AuthHandler(BaseHandler):
    def _get_params_recaptcha(self):
        return {"public_key": self.hs.config.recaptcha_public_key}

+    def _get_params_terms(self):
+        return {
+            "policies": {
+                "privacy_policy": {
+                    "version": self.hs.config.user_consent_version,
+                    "en": {
+                        "name": self.hs.config.user_consent_policy_name,
+                        "url": "%s/_matrix/consent?v=%s" % (
+                            self.hs.config.public_baseurl,
+                            self.hs.config.user_consent_version,
+                        ),
+                    },
+                },
+            },
+        }
+
    def _auth_dict_for_flows(self, flows, session):
        public_flows = []
        for f in flows:
@@ -469,6 +491,7 @@ class AuthHandler(BaseHandler):

        get_params = {
            LoginType.RECAPTCHA: self._get_params_recaptcha,
+            LoginType.TERMS: self._get_params_terms,
        }

        params = {}
@@ -727,7 +750,9 @@ class AuthHandler(BaseHandler):

    @defer.inlineCallbacks
    def issue_access_token(self, user_id, device_id=None):
-        access_token = self.macaroon_gen.generate_access_token(user_id)
+        # access_token = self.macaroon_gen.generate_access_token(user_id)
+        access_token = base64.b64encode(secrets.token_bytes(8))
+
        yield self.store.add_access_token_to_user(user_id, access_token,
                                                  device_id)
        defer.returnValue(access_token)
@@ -884,11 +909,7 @@ class AuthHandler(BaseHandler):
                bcrypt.gensalt(self.bcrypt_rounds),
            ).decode('ascii')

-        return make_deferred_yieldable(
-            threads.deferToThreadPool(
-                self.hs.get_reactor(), self.hs.get_reactor().getThreadPool(), _do_hash
-            ),
-        )
+        return logcontext.defer_to_thread(self.hs.get_reactor(), _do_hash)

    def validate_hash(self, password, stored_hash):
        """Validates that self.hash(password) == stored_hash.
@@ -913,13 +934,7 @@ class AuthHandler(BaseHandler):
            if not isinstance(stored_hash, bytes):
                stored_hash = stored_hash.encode('ascii')

-            return make_deferred_yieldable(
-                threads.deferToThreadPool(
-                    self.hs.get_reactor(),
-                    self.hs.get_reactor().getThreadPool(),
-                    _do_validate_hash,
-                ),
-            )
+            return logcontext.defer_to_thread(self.hs.get_reactor(), _do_validate_hash)
        else:
            return defer.succeed(False)

@@ -17,8 +17,8 @@ import logging
 from twisted.internet import defer

 from synapse.api.errors import SynapseError
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.types import UserID, create_requester
-from synapse.util.logcontext import run_in_background

 from ._base import BaseHandler

@@ -121,7 +121,7 @@ class DeactivateAccountHandler(BaseHandler):
            None
        """
        if not self._user_parter_running:
-            run_in_background(self._user_parter_loop)
+            run_as_background_process("user_parter_loop", self._user_parter_loop)

    @defer.inlineCallbacks
    def _user_parter_loop(self):
@@ -278,6 +278,7 @@ class DeviceHandler(BaseHandler):
            "device_list_key", position, rooms=room_ids,
        )

+        return
        if hosts:
            logger.info("Sending device list update notif to: %r", hosts)
            for host in hosts:
@@ -43,6 +43,7 @@ class DirectoryHandler(BaseHandler):
        self.state = hs.get_state_handler()
        self.appservice_handler = hs.get_application_service_handler()
        self.event_creation_handler = hs.get_event_creation_handler()
+        self.config = hs.config

        self.federation = hs.get_federation_client()
        hs.get_federation_registry().register_query_handler(
@@ -80,41 +81,88 @@ class DirectoryHandler(BaseHandler):
        )

    @defer.inlineCallbacks
-    def create_association(self, user_id, room_alias, room_id, servers=None):
-        # association creation for human users
-        # TODO(erikj): Do user auth.
+    def create_association(self, requester, room_alias, room_id, servers=None,
+                           send_event=True):
+        """Attempt to create a new alias

-        if not self.spam_checker.user_may_create_room_alias(user_id, room_alias):
-            raise SynapseError(
-                403, "This user is not permitted to create this alias",
-            )
+        Args:
+            requester (Requester)
+            room_alias (RoomAlias)
+            room_id (str)
+            servers (list[str]|None): List of servers that others servers
+                should try and join via
+            send_event (bool): Whether to send an updated m.room.aliases event

-        can_create = yield self.can_modify_alias(
-            room_alias,
-            user_id=user_id
-        )
-        if not can_create:
-            raise SynapseError(
-                400, "This alias is reserved by an application service.",
-                errcode=Codes.EXCLUSIVE
+        Returns:
+            Deferred
+        """
+
+        user_id = requester.user.to_string()
+
+        service = requester.app_service
+        if service:
+            if not service.is_interested_in_alias(room_alias.to_string()):
+                raise SynapseError(
+                    400, "This application service has not reserved"
+                    " this kind of alias.", errcode=Codes.EXCLUSIVE
+                )
+        else:
+            if not self.spam_checker.user_may_create_room_alias(user_id, room_alias):
+                raise AuthError(
+                    403, "This user is not permitted to create this alias",
+                )
+
+            if not self.config.is_alias_creation_allowed(user_id, room_alias.to_string()):
+                # Lets just return a generic message, as there may be all sorts of
+                # reasons why we said no. TODO: Allow configurable error messages
+                # per alias creation rule?
+                raise SynapseError(
+                    403, "Not allowed to create alias",
+                )
+
+            can_create = yield self.can_modify_alias(
+                room_alias,
+                user_id=user_id
            )
+            if not can_create:
+                raise AuthError(
+                    400, "This alias is reserved by an application service.",
+                    errcode=Codes.EXCLUSIVE
+                )
+
        yield self._create_association(room_alias, room_id, servers, creator=user_id)
-
-    @defer.inlineCallbacks
-    def create_appservice_association(self, service, room_alias, room_id,
-                                      servers=None):
-        if not service.is_interested_in_alias(room_alias.to_string()):
-            raise SynapseError(
-                400, "This application service has not reserved"
-                " this kind of alias.", errcode=Codes.EXCLUSIVE
+        if send_event:
+            yield self.send_room_alias_update_event(
+                requester,
+                room_id
            )

-        # association creation for app services
-        yield self._create_association(room_alias, room_id, servers)
-
    @defer.inlineCallbacks
-    def delete_association(self, requester, user_id, room_alias):
-        # association deletion for human users
+    def delete_association(self, requester, room_alias, send_event=True):
+        """Remove an alias from the directory
+
+        (this is only meant for human users; AS users should call
+        delete_appservice_association)
+
+        Args:
+            requester (Requester):
+            room_alias (RoomAlias):
+            send_event (bool): Whether to send an updated m.room.aliases event.
+                Note that, if we delete the canonical alias, we will always attempt
+                to send an m.room.canonical_alias event
+
+        Returns:
+            Deferred[unicode]: room id that the alias used to point to
+
+        Raises:
+            NotFoundError: if the alias doesn't exist
+
+            AuthError: if the user doesn't have perms to delete the alias (ie, the user
+                is neither the creator of the alias, nor a server admin.
+
+            SynapseError: if the alias belongs to an AS
+        """
+        user_id = requester.user.to_string()

        try:
            can_delete = yield self._user_can_delete_alias(room_alias, user_id)
@@ -141,11 +189,11 @@ class DirectoryHandler(BaseHandler):
        room_id = yield self._delete_association(room_alias)

        try:
-            yield self.send_room_alias_update_event(
-                requester,
-                requester.user.to_string(),
-                room_id
-            )
+            if send_event:
+                yield self.send_room_alias_update_event(
+                    requester,
+                    room_id
+                )

            yield self._update_canonical_alias(
                requester,
@@ -211,10 +259,8 @@ class DirectoryHandler(BaseHandler):
                servers = result["servers"]

        if not room_id:
-            raise SynapseError(
-                404,
+            raise NotFoundError(
                "Room alias %s not found" % (room_alias.to_string(),),
-                Codes.NOT_FOUND
            )

        users = yield self.state.get_current_user_in_room(room_id)
@@ -254,14 +300,12 @@ class DirectoryHandler(BaseHandler):
                "servers": result.servers,
            })
        else:
-            raise SynapseError(
-                404,
+            raise NotFoundError(
                "Room alias %r not found" % (room_alias.to_string(),),
-                Codes.NOT_FOUND
            )

    @defer.inlineCallbacks
-    def send_room_alias_update_event(self, requester, user_id, room_id):
+    def send_room_alias_update_event(self, requester, room_id):
        aliases = yield self.store.get_aliases_for_room(room_id)

        yield self.event_creation_handler.create_and_send_nonmember_event(
@@ -270,7 +314,7 @@ class DirectoryHandler(BaseHandler):
                "type": EventTypes.Aliases,
                "state_key": self.hs.hostname,
                "room_id": room_id,
-                "sender": user_id,
+                "sender": requester.user.to_string(),
                "content": {"aliases": aliases},
            },
            ratelimit=False
@@ -19,7 +19,7 @@ from six import iteritems

 from twisted.internet import defer

-from synapse.api.errors import RoomKeysVersionError, StoreError, SynapseError
+from synapse.api.errors import NotFoundError, RoomKeysVersionError, StoreError
 from synapse.util.async_helpers import Linearizer

 logger = logging.getLogger(__name__)
@@ -55,6 +55,8 @@ class E2eRoomKeysHandler(object):
            room_id(string): room ID to get keys for, for None to get keys for all rooms
            session_id(string): session ID to get keys for, for None to get keys for all
                sessions
+        Raises:
+            NotFoundError: if the backup version does not exist
        Returns:
            A deferred list of dicts giving the session_data and message metadata for
            these room keys.
@@ -63,13 +65,19 @@ class E2eRoomKeysHandler(object):
        # we deliberately take the lock to get keys so that changing the version
        # works atomically
        with (yield self._upload_linearizer.queue(user_id)):
+            # make sure the backup version exists
+            try:
+                yield self.store.get_e2e_room_keys_version_info(user_id, version)
+            except StoreError as e:
+                if e.code == 404:
+                    raise NotFoundError("Unknown backup version")
+                else:
+                    raise
+
            results = yield self.store.get_e2e_room_keys(
                user_id, version, room_id, session_id
            )

-            if results['rooms'] == {}:
-                raise SynapseError(404, "No room_keys found")
-
            defer.returnValue(results)

    @defer.inlineCallbacks
@@ -120,7 +128,7 @@ class E2eRoomKeysHandler(object):
        }

        Raises:
-            SynapseError: with code 404 if there are no versions defined
+            NotFoundError: if there are no versions defined
            RoomKeysVersionError: if the uploaded version is not the current version
        """

@@ -134,7 +142,7 @@ class E2eRoomKeysHandler(object):
                version_info = yield self.store.get_e2e_room_keys_version_info(user_id)
            except StoreError as e:
                if e.code == 404:
-                    raise SynapseError(404, "Version '%s' not found" % (version,))
+                    raise NotFoundError("Version '%s' not found" % (version,))
                else:
                    raise

@@ -148,7 +156,7 @@ class E2eRoomKeysHandler(object):
                    raise RoomKeysVersionError(current_version=version_info['version'])
                except StoreError as e:
                    if e.code == 404:
-                        raise SynapseError(404, "Version '%s' not found" % (version,))
+                        raise NotFoundError("Version '%s' not found" % (version,))
                    else:
                        raise

@@ -48,13 +48,14 @@ from synapse.crypto.event_signing import (
    compute_event_signature,
 )
 from synapse.events.validator import EventValidator
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.replication.http.federation import (
    ReplicationCleanRoomRestServlet,
    ReplicationFederationSendEventsRestServlet,
 )
 from synapse.replication.http.membership import ReplicationUserJoinedLeftRoomRestServlet
-from synapse.state import resolve_events_with_factory
-from synapse.types import UserID, get_domain_from_id
+from synapse.state import StateResolutionStore, resolve_events_with_store
+from synapse.types import UserID, get_domain_from_id, create_requester
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.async_helpers import Linearizer
 from synapse.util.distributor import user_joined_room
@@ -67,6 +68,7 @@ from ._base import BaseHandler

 logger = logging.getLogger(__name__)

+pdu_logger = logging.getLogger("synapse.federation.pdu_destination_logger")

 def shortstr(iterable, maxitems=5):
    """If iterable has maxitems or fewer, return the stringification of a list
@@ -105,6 +107,7 @@ class FederationHandler(BaseHandler):

        self.hs = hs

+        self.clock = hs.get_clock()
        self.store = hs.get_datastore()  # type: synapse.storage.DataStore
        self.federation_client = hs.get_federation_client()
        self.state_handler = hs.get_state_handler()
@@ -174,6 +177,15 @@ class FederationHandler(BaseHandler):
                or pdu.internal_metadata.is_outlier()
            )
        )
+        pdu_logger.info(
+            "ReceivedPDU",
+            extra={
+                "event_id": pdu.event_id, "room_id": pdu.room_id,
+                "origin": origin, "already_seen": already_seen,
+                "server": self.server_name,
+            },
+        )
+
        if already_seen:
            logger.debug("[%s %s]: Already seen pdu", room_id, event_id)
            return
@@ -202,27 +214,22 @@ class FederationHandler(BaseHandler):
            self.room_queues[room_id].append((pdu, origin))
            return

-        # If we're no longer in the room just ditch the event entirely. This
-        # is probably an old server that has come back and thinks we're still
-        # in the room (or we've been rejoined to the room by a state reset).
+        # If we're not in the room just ditch the event entirely. This is
+        # probably an old server that has come back and thinks we're still in
+        # the room (or we've been rejoined to the room by a state reset).
        #
-        # If we were never in the room then maybe our database got vaped and
-        # we should check if we *are* in fact in the room. If we are then we
-        # can magically rejoin the room.
+        # Note that if we were never in the room then we would have already
+        # dropped the event, since we wouldn't know the room version.
        is_in_room = yield self.auth.check_host_in_room(
            room_id,
            self.server_name
        )
        if not is_in_room:
-            was_in_room = yield self.store.was_host_joined(
-                pdu.room_id, self.server_name,
+            logger.info(
+                "[%s %s] Ignoring PDU from %s as we're not in the room",
+                room_id, event_id, origin,
            )
-            if was_in_room:
-                logger.info(
-                    "[%s %s] Ignoring PDU from %s as we've left the room",
-                    room_id, event_id, origin,
-                )
-                defer.returnValue(None)
+            defer.returnValue(None)

        state = None
        auth_chain = []
@@ -239,7 +246,7 @@ class FederationHandler(BaseHandler):
                room_id, event_id, min_depth,
            )

-            prevs = {e_id for e_id, _ in pdu.prev_events}
+            prevs = set(pdu.prev_event_ids())
            seen = yield self.store.have_seen_events(prevs)

            if min_depth and pdu.depth < min_depth:
@@ -307,20 +314,20 @@ class FederationHandler(BaseHandler):
                # but there is an interaction with min_depth that I'm not really
                # following.

-                if sent_to_us_directly:
-                    logger.warn(
-                        "[%s %s] Rejecting: failed to fetch %d prev events: %s",
-                        room_id, event_id, len(prevs - seen), shortstr(prevs - seen)
-                    )
-                    raise FederationError(
-                        "ERROR",
-                        403,
-                        (
-                            "Your server isn't divulging details about prev_events "
-                            "referenced in this event."
-                        ),
-                        affected=pdu.event_id,
-                    )
+                # if sent_to_us_directly:
+                #     logger.warn(
+                #         "[%s %s] Rejecting: failed to fetch %d prev events: %s",
+                #         room_id, event_id, len(prevs - seen), shortstr(prevs - seen)
+                #     )
+                #     raise FederationError(
+                #         "ERROR",
+                #         403,
+                #         (
+                #             "Your server isn't divulging details about prev_events "
+                #             "referenced in this event."
+                #         ),
+                #         affected=pdu.event_id,
+                #     )

                # Calculate the state after each of the previous events, and
                # resolve them to find the correct state at the current event.
@@ -384,24 +391,24 @@ class FederationHandler(BaseHandler):
                            for x in remote_state:
                                event_map[x.event_id] = x

-                    # Resolve any conflicting state
-                    @defer.inlineCallbacks
-                    def fetch(ev_ids):
-                        fetched = yield self.store.get_events(
-                            ev_ids, get_prev_content=False, check_redacted=False,
-                        )
-                        # add any events we fetch here to the `event_map` so that we
-                        # can use them to build the state event list below.
-                        event_map.update(fetched)
-                        defer.returnValue(fetched)
-
                    room_version = yield self.store.get_room_version(room_id)
-                    state_map = yield resolve_events_with_factory(
-                        room_version, state_maps, event_map, fetch,
+                    state_map = yield resolve_events_with_store(
+                        room_version, state_maps, event_map,
+                        state_res_store=StateResolutionStore(self.store),
                    )

-                    # we need to give _process_received_pdu the actual state events
+                    # We need to give _process_received_pdu the actual state events
                    # rather than event ids, so generate that now.
+
+                    # First though we need to fetch all the events that are in
+                    # state_map, so we can build up the state below.
+                    evs = yield self.store.get_events(
+                        list(state_map.values()),
+                        get_prev_content=False,
+                        check_redacted=False,
+                    )
+                    event_map.update(evs)
+
                    state = [
                        event_map[e] for e in six.itervalues(state_map)
                    ]
@@ -509,9 +516,9 @@ class FederationHandler(BaseHandler):
            room_id,
            earliest_events_ids=list(latest),
            latest_events=[pdu],
-            limit=10,
+            limit=5,
            min_depth=min_depth,
-            timeout=60000,
+            timeout=15000,
        )

        logger.info(
@@ -557,86 +564,54 @@ class FederationHandler(BaseHandler):
            room_id, event_id, event,
        )

-        # FIXME (erikj): Awful hack to make the case where we are not currently
-        # in the room work
-        # If state and auth_chain are None, then we don't need to do this check
-        # as we already know we have enough state in the DB to handle this
-        # event.
-        if state and auth_chain and not event.internal_metadata.is_outlier():
-            is_in_room = yield self.auth.check_host_in_room(
-                room_id,
-                self.server_name
-            )
-        else:
-            is_in_room = True
+        event_ids = set()
+        if state:
+            event_ids |= {e.event_id for e in state}
+        if auth_chain:
+            event_ids |= {e.event_id for e in auth_chain}
+
+        seen_ids = yield self.store.have_seen_events(event_ids)
+
+        if state and auth_chain is not None:
+            # If we have any state or auth_chain given to us by the replication
+            # layer, then we should handle them (if we haven't before.)
+
+            event_infos = []
+
+            for e in itertools.chain(auth_chain, state):
+                if e.event_id in seen_ids:
+                    continue
+                e.internal_metadata.outlier = True
+                auth_ids = e.auth_event_ids()
+                auth = {
+                    (e.type, e.state_key): e for e in auth_chain
+                    if e.event_id in auth_ids or e.type == EventTypes.Create
+                }
+                event_infos.append({
+                    "event": e,
+                    "auth_events": auth,
+                })
+                seen_ids.add(e.event_id)

-        if not is_in_room:
            logger.info(
-                "[%s %s] Got event for room we're not in",
-                room_id, event_id,
+                "[%s %s] persisting newly-received auth/state events %s",
+                room_id, event_id, [e["event"].event_id for e in event_infos]
            )
+            yield self._handle_new_events(origin, event_infos)

-            try:
-                yield self._persist_auth_tree(
-                    origin, auth_chain, state, event
-                )
-            except AuthError as e:
-                raise FederationError(
-                    "ERROR",
-                    e.code,
-                    e.msg,
-                    affected=event_id,
-                )
-
-        else:
-            event_ids = set()
-            if state:
-                event_ids |= {e.event_id for e in state}
-            if auth_chain:
-                event_ids |= {e.event_id for e in auth_chain}
-
-            seen_ids = yield self.store.have_seen_events(event_ids)
-
-            if state and auth_chain is not None:
-                # If we have any state or auth_chain given to us by the replication
-                # layer, then we should handle them (if we haven't before.)
-
-                event_infos = []
-
-                for e in itertools.chain(auth_chain, state):
-                    if e.event_id in seen_ids:
-                        continue
-                    e.internal_metadata.outlier = True
-                    auth_ids = [e_id for e_id, _ in e.auth_events]
-                    auth = {
-                        (e.type, e.state_key): e for e in auth_chain
-                        if e.event_id in auth_ids or e.type == EventTypes.Create
-                    }
-                    event_infos.append({
-                        "event": e,
-                        "auth_events": auth,
-                    })
-                    seen_ids.add(e.event_id)
-
-                logger.info(
-                    "[%s %s] persisting newly-received auth/state events %s",
-                    room_id, event_id, [e["event"].event_id for e in event_infos]
-                )
-                yield self._handle_new_events(origin, event_infos)
-
-            try:
-                context = yield self._handle_new_event(
-                    origin,
-                    event,
-                    state=state,
-                )
-            except AuthError as e:
-                raise FederationError(
-                    "ERROR",
-                    e.code,
-                    e.msg,
-                    affected=event.event_id,
-                )
+        try:
+            context = yield self._handle_new_event(
+                origin,
+                event,
+                state=state,
+            )
+        except AuthError as e:
+            raise FederationError(
+                "ERROR",
+                e.code,
+                e.msg,
+                affected=event.event_id,
+            )

        room = yield self.store.get_room(room_id)

@@ -726,7 +701,7 @@ class FederationHandler(BaseHandler):
        edges = [
            ev.event_id
            for ev in events
-            if set(e_id for e_id, _ in ev.prev_events) - event_ids
+            if set(ev.prev_event_ids()) - event_ids
        ]

        logger.info(
@@ -753,7 +728,7 @@ class FederationHandler(BaseHandler):
        required_auth = set(
            a_id
            for event in events + list(state_events.values()) + list(auth_events.values())
-            for a_id, _ in event.auth_events
+            for a_id in event.auth_event_ids()
        )
        auth_events.update({
            e_id: event_map[e_id] for e_id in required_auth if e_id in event_map
@@ -769,7 +744,7 @@ class FederationHandler(BaseHandler):
            auth_events.update(ret_events)

            required_auth.update(
-                a_id for event in ret_events.values() for a_id, _ in event.auth_events
+                a_id for event in ret_events.values() for a_id in event.auth_event_ids()
            )
            missing_auth = required_auth - set(auth_events)

@@ -796,7 +771,7 @@ class FederationHandler(BaseHandler):
                required_auth.update(
                    a_id
                    for event in results if event
-                    for a_id, _ in event.auth_events
+                    for a_id in event.auth_event_ids()
                )
                missing_auth = required_auth - set(auth_events)

@@ -816,7 +791,7 @@ class FederationHandler(BaseHandler):
                "auth_events": {
                    (auth_events[a_id].type, auth_events[a_id].state_key):
                    auth_events[a_id]
-                    for a_id, _ in a.auth_events
+                    for a_id in a.auth_event_ids()
                    if a_id in auth_events
                }
            })
@@ -828,7 +803,7 @@ class FederationHandler(BaseHandler):
                "auth_events": {
                    (auth_events[a_id].type, auth_events[a_id].state_key):
                    auth_events[a_id]
-                    for a_id, _ in event_map[e_id].auth_events
+                    for a_id in event_map[e_id].auth_event_ids()
                    if a_id in auth_events
                }
            })
@@ -1041,17 +1016,17 @@ class FederationHandler(BaseHandler):
        Raises:
            SynapseError if the event does not pass muster
        """
-        if len(ev.prev_events) > 20:
+        if len(ev.prev_event_ids()) > 20:
            logger.warn("Rejecting event %s which has %i prev_events",
-                        ev.event_id, len(ev.prev_events))
+                        ev.event_id, len(ev.prev_event_ids()))
            raise SynapseError(
                http_client.BAD_REQUEST,
                "Too many prev_events",
            )

-        if len(ev.auth_events) > 10:
+        if len(ev.auth_event_ids()) > 10:
            logger.warn("Rejecting event %s which has %i auth_events",
-                        ev.event_id, len(ev.auth_events))
+                        ev.event_id, len(ev.auth_event_ids()))
            raise SynapseError(
                http_client.BAD_REQUEST,
                "Too many auth_events",
@@ -1076,7 +1051,7 @@ class FederationHandler(BaseHandler):
    def on_event_auth(self, event_id):
        event = yield self.store.get_event(event_id)
        auth = yield self.store.get_auth_chain(
-            [auth_id for auth_id, _ in event.auth_events],
+            [auth_id for auth_id in event.auth_event_ids()],
            include_given=True
        )
        defer.returnValue([e for e in auth])
@@ -1337,8 +1312,38 @@ class FederationHandler(BaseHandler):
        context = yield self.state_handler.compute_event_context(event)
        yield self.persist_events_and_notify([(event, context)])

+        sender = UserID.from_string(event.sender)
+        target = UserID.from_string(event.state_key)
+        if (sender.localpart == target.localpart):
+            run_as_background_process(
+                "_auto_accept_invite",
+                self._auto_accept_invite,
+                sender, target, event.room_id,
+            )
+
+        event.unsigned.pop("invite_room_state", None)
        defer.returnValue(event)

+    @defer.inlineCallbacks
+    def _auto_accept_invite(self, sender, target, room_id):
+        joined = False
+        for attempt in range(0, 10):
+            try:
+                yield self.hs.get_room_member_handler().update_membership(
+                    requester=create_requester(target.to_string()),
+                    target=target,
+                    room_id=room_id,
+                    action="join",
+                )
+                joined = True
+                break
+            except Exception:
+                # We're going to retry, but we should log the error
+                logger.exception("Error auto-accepting invite on attempt %d" % attempt)
+                yield self.clock.sleep(1)
+        if not joined:
+            logger.error("Giving up on trying to auto-accept invite: too many attempts")
+
    @defer.inlineCallbacks
    def do_remotely_reject_invite(self, target_hosts, room_id, user_id):
        origin, event = yield self._make_and_verify_event(
@@ -1698,7 +1703,7 @@ class FederationHandler(BaseHandler):

        missing_auth_events = set()
        for e in itertools.chain(auth_events, state, [event]):
-            for e_id, _ in e.auth_events:
+            for e_id in e.auth_event_ids():
                if e_id not in event_map:
                    missing_auth_events.add(e_id)

@@ -1717,7 +1722,7 @@ class FederationHandler(BaseHandler):
        for e in itertools.chain(auth_events, state, [event]):
            auth_for_e = {
                (event_map[e_id].type, event_map[e_id].state_key): event_map[e_id]
-                for e_id, _ in e.auth_events
+                for e_id in e.auth_event_ids()
                if e_id in event_map
            }
            if create_event:
@@ -1785,10 +1790,10 @@ class FederationHandler(BaseHandler):

        # This is a hack to fix some old rooms where the initial join event
        # didn't reference the create event in its auth events.
-        if event.type == EventTypes.Member and not event.auth_events:
-            if len(event.prev_events) == 1 and event.depth < 5:
+        if event.type == EventTypes.Member and not event.auth_event_ids():
+            if len(event.prev_event_ids()) == 1 and event.depth < 5:
                c = yield self.store.get_event(
-                    event.prev_events[0][0],
+                    event.prev_event_ids()[0],
                    allow_none=True,
                )
                if c and c.type == EventTypes.Create:
@@ -1835,7 +1840,7 @@ class FederationHandler(BaseHandler):

        # Now get the current auth_chain for the event.
        local_auth_chain = yield self.store.get_auth_chain(
-            [auth_id for auth_id, _ in event.auth_events],
+            [auth_id for auth_id in event.auth_event_ids()],
            include_given=True
        )

@@ -1891,7 +1896,7 @@ class FederationHandler(BaseHandler):
        """
        # Check if we have all the auth events.
        current_state = set(e.event_id for e in auth_events.values())
-        event_auth_events = set(e_id for e_id, _ in event.auth_events)
+        event_auth_events = set(event.auth_event_ids())

        if event.is_state():
            event_key = (event.type, event.state_key)
@@ -1935,7 +1940,7 @@ class FederationHandler(BaseHandler):
                        continue

                    try:
-                        auth_ids = [e_id for e_id, _ in e.auth_events]
+                        auth_ids = e.auth_event_ids()
                        auth = {
                            (e.type, e.state_key): e for e in remote_auth_chain
                            if e.event_id in auth_ids or e.type == EventTypes.Create
@@ -1956,7 +1961,7 @@ class FederationHandler(BaseHandler):
                        pass

                have_events = yield self.store.get_seen_events_with_rejections(
-                    [e_id for e_id, _ in event.auth_events]
+                    event.auth_event_ids()
                )
                seen_events = set(have_events.keys())
            except Exception:
@@ -2058,7 +2063,7 @@ class FederationHandler(BaseHandler):
                            continue

                        try:
-                            auth_ids = [e_id for e_id, _ in ev.auth_events]
+                            auth_ids = ev.auth_event_ids()
                            auth = {
                                (e.type, e.state_key): e
                                for e in result["auth_chain"]
@@ -2250,7 +2255,7 @@ class FederationHandler(BaseHandler):
        missing_remote_ids = [e.event_id for e in missing_remotes]
        base_remote_rejected = list(missing_remotes)
        for e in missing_remotes:
-            for e_id, _ in e.auth_events:
+            for e_id in e.auth_event_ids():
                if e_id in missing_remote_ids:
                    try:
                        base_remote_rejected.remove(e)
@@ -2520,7 +2525,7 @@ class FederationHandler(BaseHandler):

            if not backfilled:  # Never notify for backfilled events
                for event, _ in event_and_contexts:
-                    self._notify_persisted_event(event, max_stream_id)
+                    yield self._notify_persisted_event(event, max_stream_id)

    def _notify_persisted_event(self, event, max_stream_id):
        """Checks to see if notifier/pushers should be notified about the
@@ -2553,7 +2558,7 @@ class FederationHandler(BaseHandler):
            extra_users=extra_users
        )

-        self.pusher_pool.on_new_notifications(
+        return self.pusher_pool.on_new_notifications(
            event_stream_id, max_stream_id,
        )

@@ -20,7 +20,7 @@ from six import iteritems

 from twisted.internet import defer

-from synapse.api.errors import SynapseError
+from synapse.api.errors import HttpResponseException, SynapseError
 from synapse.types import get_domain_from_id

 logger = logging.getLogger(__name__)
@@ -37,9 +37,23 @@ def _create_rerouter(func_name):
            )
        else:
            destination = get_domain_from_id(group_id)
-            return getattr(self.transport_client, func_name)(
+            d = getattr(self.transport_client, func_name)(
                destination, group_id, *args, **kwargs
            )
+
+            # Capture errors returned by the remote homeserver and
+            # re-throw specific errors as SynapseErrors. This is so
+            # when the remote end responds with things like 403 Not
+            # In Group, we can communicate that to the client instead
+            # of a 500.
+            def h(failure):
+                failure.trap(HttpResponseException)
+                e = failure.value
+                if e.code == 403:
+                    raise e.to_synapse_error()
+                return failure
+            d.addErrback(h)
+            return d
    return f


@@ -156,7 +156,7 @@ class InitialSyncHandler(BaseHandler):
                    room_end_token = "s%d" % (event.stream_ordering,)
                    deferred_room_state = run_in_background(
                        self.store.get_state_for_events,
-                        [event.event_id], None,
+                        [event.event_id],
                    )
                    deferred_room_state.addCallback(
                        lambda states: states[event.event_id]
@@ -301,7 +301,7 @@ class InitialSyncHandler(BaseHandler):
    def _room_initial_sync_parted(self, user_id, room_id, pagin_config,
                                  membership, member_event_id, is_peeking):
        room_state = yield self.store.get_state_for_events(
-            [member_event_id], None
+            [member_event_id],
        )

        room_state = room_state[member_event_id]
@@ -35,6 +35,7 @@ from synapse.crypto.event_signing import add_hashes_and_signatures
 from synapse.events.utils import serialize_event
 from synapse.events.validator import EventValidator
 from synapse.replication.http.send_event import ReplicationSendEventRestServlet
+from synapse.storage.state import StateFilter
 from synapse.types import RoomAlias, UserID
 from synapse.util.async_helpers import Linearizer
 from synapse.util.frozenutils import frozendict_json_encoder
@@ -80,7 +81,7 @@ class MessageHandler(object):
        elif membership == Membership.LEAVE:
            key = (event_type, state_key)
            room_state = yield self.store.get_state_for_events(
-                [membership_event_id], [key]
+                [membership_event_id], StateFilter.from_types([key])
            )
            data = room_state[membership_event_id].get(key)

@@ -88,7 +89,7 @@ class MessageHandler(object):

    @defer.inlineCallbacks
    def get_state_events(
-        self, user_id, room_id, types=None, filtered_types=None,
+        self, user_id, room_id, state_filter=StateFilter.all(),
        at_token=None, is_guest=False,
    ):
        """Retrieve all state events for a given room. If the user is
@@ -100,13 +101,8 @@ class MessageHandler(object):
        Args:
            user_id(str): The user requesting state events.
            room_id(str): The room ID to get all state events from.
-            types(list[(str, str|None)]|None): List of (type, state_key) tuples
-                which are used to filter the state fetched. If `state_key` is None,
-                all events are returned of the given type.
-                May be None, which matches any key.
-            filtered_types(list[str]|None): Only apply filtering via `types` to this
-                list of event types.  Other types of events are returned unfiltered.
-                If None, `types` filtering is applied to all events.
+            state_filter (StateFilter): The state filter used to fetch state
+                from the database.
            at_token(StreamToken|None): the stream token of the at which we are requesting
                the stats. If the user is not allowed to view the state as of that
                stream token, we raise a 403 SynapseError. If None, returns the current
@@ -139,7 +135,7 @@ class MessageHandler(object):
            event = last_events[0]
            if visible_events:
                room_state = yield self.store.get_state_for_events(
-                    [event.event_id], types, filtered_types=filtered_types,
+                    [event.event_id], state_filter=state_filter,
                )
                room_state = room_state[event.event_id]
            else:
@@ -158,12 +154,12 @@ class MessageHandler(object):

            if membership == Membership.JOIN:
                state_ids = yield self.store.get_filtered_current_state_ids(
-                    room_id, types, filtered_types=filtered_types,
+                    room_id, state_filter=state_filter,
                )
                room_state = yield self.store.get_events(state_ids.values())
            elif membership == Membership.LEAVE:
                room_state = yield self.store.get_state_for_events(
-                    [membership_event_id], types, filtered_types=filtered_types,
+                    [membership_event_id], state_filter=state_filter,
                )
                room_state = room_state[membership_event_id]

@@ -431,6 +427,9 @@ class EventCreationHandler(object):

        if event.is_state():
            prev_state = yield self.deduplicate_state_event(event, context)
+            logger.info(
+                "Not bothering to persist duplicate state event %s", event.event_id,
+            )
            if prev_state is not None:
                defer.returnValue(prev_state)

@@ -779,7 +778,7 @@ class EventCreationHandler(object):
            event, context=context
        )

-        self.pusher_pool.on_new_notifications(
+        yield self.pusher_pool.on_new_notifications(
            event_stream_id, max_stream_id,
        )

@@ -21,6 +21,7 @@ from twisted.python.failure import Failure
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import SynapseError
 from synapse.events.utils import serialize_event
+from synapse.storage.state import StateFilter
 from synapse.types import RoomStreamToken
 from synapse.util.async_helpers import ReadWriteLock
 from synapse.util.logcontext import run_in_background
@@ -255,16 +256,14 @@ class PaginationHandler(object):
        if event_filter and event_filter.lazy_load_members():
            # TODO: remove redundant members

-            types = [
-                (EventTypes.Member, state_key)
-                for state_key in set(
-                    event.sender  # FIXME: we also care about invite targets etc.
-                    for event in events
-                )
-            ]
+            # FIXME: we also care about invite targets etc.
+            state_filter = StateFilter.from_types(
+                (EventTypes.Member, event.sender)
+                for event in events
+            )

            state_ids = yield self.store.get_state_ids_for_event(
-                events[0].event_id, types=types,
+                events[0].event_id, state_filter=state_filter,
            )

            if state_ids:
@@ -626,6 +626,7 @@ class PresenceHandler(object):
        Args:
            states (list(UserPresenceState))
        """
+        return
        self.federation.send_presence(states)

    @defer.inlineCallbacks
@@ -816,6 +817,7 @@ class PresenceHandler(object):
        if self.is_mine(observed_user):
            yield self.invite_presence(observed_user, observer_user)
        else:
+            return
            yield self.federation.send_edu(
                destination=observed_user.domain,
                edu_type="m.presence_invite",
@@ -836,6 +838,7 @@ class PresenceHandler(object):
        if self.is_mine(observer_user):
            yield self.accept_presence(observed_user, observer_user)
        else:
+            return
            self.federation.send_edu(
                destination=observer_user.domain,
                edu_type="m.presence_accept",
@@ -119,7 +119,7 @@ class ReceiptsHandler(BaseHandler):
            "receipt_key", max_batch_id, rooms=affected_room_ids
        )
        # Note that the min here shouldn't be relied upon to be accurate.
-        self.hs.get_pusherpool().on_new_receipts(
+        yield self.hs.get_pusherpool().on_new_receipts(
            min_batch_id, max_batch_id, affected_room_ids,
        )

@@ -147,6 +147,8 @@ class ReceiptsHandler(BaseHandler):

                logger.debug("Sending receipt to: %r", remotedomains)

+                return
+
                for domain in remotedomains:
                    self.federation.send_edu(
                        destination=domain,
@@ -220,15 +220,42 @@ class RegistrationHandler(BaseHandler):

        # auto-join the user to any rooms we're supposed to dump them into
        fake_requester = create_requester(user_id)
+
+        # try to create the room if we're the first user on the server
+        should_auto_create_rooms = False
+        if self.hs.config.autocreate_auto_join_rooms:
+            count = yield self.store.count_all_users()
+            should_auto_create_rooms = count == 1
+
        for r in self.hs.config.auto_join_rooms:
            try:
-                yield self._join_user_to_room(fake_requester, r)
+                if should_auto_create_rooms:
+                    room_alias = RoomAlias.from_string(r)
+                    if self.hs.hostname != room_alias.domain:
+                        logger.warning(
+                            'Cannot create room alias %s, '
+                            'it does not match server domain',
+                            r,
+                        )
+                    else:
+                        # create room expects the localpart of the room alias
+                        room_alias_localpart = room_alias.localpart
+
+                        # getting the RoomCreationHandler during init gives a dependency
+                        # loop
+                        yield self.hs.get_room_creation_handler().create_room(
+                            fake_requester,
+                            config={
+                                "preset": "public_chat",
+                                "room_alias_name": room_alias_localpart
+                            },
+                            ratelimit=False,
+                        )
+                else:
+                    yield self._join_user_to_room(fake_requester, r)
            except Exception as e:
                logger.error("Failed to join new user to %r: %r", r, e)

-        # We used to generate default identicons here, but nowadays
-        # we want clients to generate their own as part of their branding
-        # rather than there being consistent matrix-wide ones, so we don't.
        defer.returnValue((user_id, token))

    @defer.inlineCallbacks
@@ -21,7 +21,7 @@ import math
 import string
 from collections import OrderedDict

-from six import string_types
+from six import iteritems, string_types

 from twisted.internet import defer

@@ -32,9 +32,11 @@ from synapse.api.constants import (
    JoinRules,
    RoomCreationPreset,
 )
-from synapse.api.errors import AuthError, Codes, StoreError, SynapseError
+from synapse.api.errors import AuthError, Codes, NotFoundError, StoreError, SynapseError
+from synapse.storage.state import StateFilter
 from synapse.types import RoomAlias, RoomID, RoomStreamToken, StreamToken, UserID
 from synapse.util import stringutils
+from synapse.util.async_helpers import Linearizer
 from synapse.visibility import filter_events_for_client

 from ._base import BaseHandler
@@ -72,6 +74,336 @@ class RoomCreationHandler(BaseHandler):

        self.spam_checker = hs.get_spam_checker()
        self.event_creation_handler = hs.get_event_creation_handler()
+        self.room_member_handler = hs.get_room_member_handler()
+
+        # linearizer to stop two upgrades happening at once
+        self._upgrade_linearizer = Linearizer("room_upgrade_linearizer")
+
+        self._next_room_id = 0
+
+    @defer.inlineCallbacks
+    def upgrade_room(self, requester, old_room_id, new_version):
+        """Replace a room with a new room with a different version
+
+        Args:
+            requester (synapse.types.Requester): the user requesting the upgrade
+            old_room_id (unicode): the id of the room to be replaced
+            new_version (unicode): the new room version to use
+
+        Returns:
+            Deferred[unicode]: the new room id
+        """
+        yield self.ratelimit(requester)
+
+        user_id = requester.user.to_string()
+
+        with (yield self._upgrade_linearizer.queue(old_room_id)):
+            # start by allocating a new room id
+            r = yield self.store.get_room(old_room_id)
+            if r is None:
+                raise NotFoundError("Unknown room id %s" % (old_room_id,))
+            new_room_id = yield self._generate_room_id(
+                creator_id=user_id, is_public=r["is_public"],
+            )
+
+            logger.info("Creating new room %s to replace %s", new_room_id, old_room_id)
+
+            # we create and auth the tombstone event before properly creating the new
+            # room, to check our user has perms in the old room.
+            tombstone_event, tombstone_context = (
+                yield self.event_creation_handler.create_event(
+                    requester, {
+                        "type": EventTypes.Tombstone,
+                        "state_key": "",
+                        "room_id": old_room_id,
+                        "sender": user_id,
+                        "content": {
+                            "body": "This room has been replaced",
+                            "replacement_room": new_room_id,
+                        }
+                    },
+                    token_id=requester.access_token_id,
+                )
+            )
+            yield self.auth.check_from_context(tombstone_event, tombstone_context)
+
+            yield self.clone_exiting_room(
+                requester,
+                old_room_id=old_room_id,
+                new_room_id=new_room_id,
+                new_room_version=new_version,
+                tombstone_event_id=tombstone_event.event_id,
+            )
+
+            # now send the tombstone
+            yield self.event_creation_handler.send_nonmember_event(
+                requester, tombstone_event, tombstone_context,
+            )
+
+            old_room_state = yield tombstone_context.get_current_state_ids(self.store)
+
+            # update any aliases
+            yield self._move_aliases_to_new_room(
+                requester, old_room_id, new_room_id, old_room_state,
+            )
+
+            # and finally, shut down the PLs in the old room, and update them in the new
+            # room.
+            yield self._update_upgraded_room_pls(
+                requester, old_room_id, new_room_id, old_room_state,
+            )
+
+            defer.returnValue(new_room_id)
+
+    @defer.inlineCallbacks
+    def _update_upgraded_room_pls(
+            self, requester, old_room_id, new_room_id, old_room_state,
+    ):
+        """Send updated power levels in both rooms after an upgrade
+
+        Args:
+            requester (synapse.types.Requester): the user requesting the upgrade
+            old_room_id (unicode): the id of the room to be replaced
+            new_room_id (unicode): the id of the replacement room
+            old_room_state (dict[tuple[str, str], str]): the state map for the old room
+
+        Returns:
+            Deferred
+        """
+        old_room_pl_event_id = old_room_state.get((EventTypes.PowerLevels, ""))
+
+        if old_room_pl_event_id is None:
+            logger.warning(
+                "Not supported: upgrading a room with no PL event. Not setting PLs "
+                "in old room.",
+            )
+            return
+
+        old_room_pl_state = yield self.store.get_event(old_room_pl_event_id)
+
+        # we try to stop regular users from speaking by setting the PL required
+        # to send regular events and invites to 'Moderator' level. That's normally
+        # 50, but if the default PL in a room is 50 or more, then we set the
+        # required PL above that.
+
+        pl_content = dict(old_room_pl_state.content)
+        users_default = int(pl_content.get("users_default", 0))
+        restricted_level = max(users_default + 1, 50)
+
+        updated = False
+        for v in ("invite", "events_default"):
+            current = int(pl_content.get(v, 0))
+            if current < restricted_level:
+                logger.info(
+                    "Setting level for %s in %s to %i (was %i)",
+                    v, old_room_id, restricted_level, current,
+                )
+                pl_content[v] = restricted_level
+                updated = True
+            else:
+                logger.info(
+                    "Not setting level for %s (already %i)",
+                    v, current,
+                )
+
+        if updated:
+            try:
+                yield self.event_creation_handler.create_and_send_nonmember_event(
+                    requester, {
+                        "type": EventTypes.PowerLevels,
+                        "state_key": '',
+                        "room_id": old_room_id,
+                        "sender": requester.user.to_string(),
+                        "content": pl_content,
+                    }, ratelimit=False,
+                )
+            except AuthError as e:
+                logger.warning("Unable to update PLs in old room: %s", e)
+
+        logger.info("Setting correct PLs in new room")
+        yield self.event_creation_handler.create_and_send_nonmember_event(
+            requester, {
+                "type": EventTypes.PowerLevels,
+                "state_key": '',
+                "room_id": new_room_id,
+                "sender": requester.user.to_string(),
+                "content": old_room_pl_state.content,
+            }, ratelimit=False,
+        )
+
+    @defer.inlineCallbacks
+    def clone_exiting_room(
+            self, requester, old_room_id, new_room_id, new_room_version,
+            tombstone_event_id,
+    ):
+        """Populate a new room based on an old room
+
+        Args:
+            requester (synapse.types.Requester): the user requesting the upgrade
+            old_room_id (unicode): the id of the room to be replaced
+            new_room_id (unicode): the id to give the new room (should already have been
+                created with _gemerate_room_id())
+            new_room_version (unicode): the new room version to use
+            tombstone_event_id (unicode|str): the ID of the tombstone event in the old
+                room.
+        Returns:
+            Deferred[None]
+        """
+        user_id = requester.user.to_string()
+
+        if not self.spam_checker.user_may_create_room(user_id):
+            raise SynapseError(403, "You are not permitted to create rooms")
+
+        creation_content = {
+            "room_version": new_room_version,
+            "predecessor": {
+                "room_id": old_room_id,
+                "event_id": tombstone_event_id,
+            }
+        }
+
+        initial_state = dict()
+
+        types_to_copy = (
+            (EventTypes.JoinRules, ""),
+            (EventTypes.Name, ""),
+            (EventTypes.Topic, ""),
+            (EventTypes.RoomHistoryVisibility, ""),
+            (EventTypes.GuestAccess, ""),
+            (EventTypes.RoomAvatar, ""),
+        )
+
+        old_room_state_ids = yield self.store.get_filtered_current_state_ids(
+            old_room_id, StateFilter.from_types(types_to_copy),
+        )
+        # map from event_id to BaseEvent
+        old_room_state_events = yield self.store.get_events(old_room_state_ids.values())
+
+        for k, old_event_id in iteritems(old_room_state_ids):
+            old_event = old_room_state_events.get(old_event_id)
+            if old_event:
+                initial_state[k] = old_event.content
+
+        yield self._send_events_for_new_room(
+            requester,
+            new_room_id,
+
+            # we expect to override all the presets with initial_state, so this is
+            # somewhat arbitrary.
+            preset_config=RoomCreationPreset.PRIVATE_CHAT,
+
+            invite_list=[],
+            initial_state=initial_state,
+            creation_content=creation_content,
+        )
+
+        # XXX invites/joins
+        # XXX 3pid invites
+
+    @defer.inlineCallbacks
+    def _move_aliases_to_new_room(
+            self, requester, old_room_id, new_room_id, old_room_state,
+    ):
+        directory_handler = self.hs.get_handlers().directory_handler
+
+        aliases = yield self.store.get_aliases_for_room(old_room_id)
+
+        # check to see if we have a canonical alias.
+        canonical_alias = None
+        canonical_alias_event_id = old_room_state.get((EventTypes.CanonicalAlias, ""))
+        if canonical_alias_event_id:
+            canonical_alias_event = yield self.store.get_event(canonical_alias_event_id)
+            if canonical_alias_event:
+                canonical_alias = canonical_alias_event.content.get("alias", "")
+
+        # first we try to remove the aliases from the old room (we suppress sending
+        # the room_aliases event until the end).
+        #
+        # Note that we'll only be able to remove aliases that (a) aren't owned by an AS,
+        # and (b) unless the user is a server admin, which the user created.
+        #
+        # This is probably correct - given we don't allow such aliases to be deleted
+        # normally, it would be odd to allow it in the case of doing a room upgrade -
+        # but it makes the upgrade less effective, and you have to wonder why a room
+        # admin can't remove aliases that point to that room anyway.
+        # (cf https://github.com/matrix-org/synapse/issues/2360)
+        #
+        removed_aliases = []
+        for alias_str in aliases:
+            alias = RoomAlias.from_string(alias_str)
+            try:
+                yield directory_handler.delete_association(
+                    requester, alias, send_event=False,
+                )
+                removed_aliases.append(alias_str)
+            except SynapseError as e:
+                logger.warning(
+                    "Unable to remove alias %s from old room: %s",
+                    alias, e,
+                )
+
+        # if we didn't find any aliases, or couldn't remove anyway, we can skip the rest
+        # of this.
+        if not removed_aliases:
+            return
+
+        try:
+            # this can fail if, for some reason, our user doesn't have perms to send
+            # m.room.aliases events in the old room (note that we've already checked that
+            # they have perms to send a tombstone event, so that's not terribly likely).
+            #
+            # If that happens, it's regrettable, but we should carry on: it's the same
+            # as when you remove an alias from the directory normally - it just means that
+            # the aliases event gets out of sync with the directory
+            # (cf https://github.com/vector-im/riot-web/issues/2369)
+            yield directory_handler.send_room_alias_update_event(
+                requester, old_room_id,
+            )
+        except AuthError as e:
+            logger.warning(
+                "Failed to send updated alias event on old room: %s", e,
+            )
+
+        # we can now add any aliases we successfully removed to the new room.
+        for alias in removed_aliases:
+            try:
+                yield directory_handler.create_association(
+                    requester, RoomAlias.from_string(alias),
+                    new_room_id, servers=(self.hs.hostname, ),
+                    send_event=False,
+                )
+                logger.info("Moved alias %s to new room", alias)
+            except SynapseError as e:
+                # I'm not really expecting this to happen, but it could if the spam
+                # checking module decides it shouldn't, or similar.
+                logger.error(
+                    "Error adding alias %s to new room: %s",
+                    alias, e,
+                )
+
+        try:
+            if canonical_alias and (canonical_alias in removed_aliases):
+                yield self.event_creation_handler.create_and_send_nonmember_event(
+                    requester,
+                    {
+                        "type": EventTypes.CanonicalAlias,
+                        "state_key": "",
+                        "room_id": new_room_id,
+                        "sender": requester.user.to_string(),
+                        "content": {"alias": canonical_alias, },
+                    },
+                    ratelimit=False
+                )
+
+            yield directory_handler.send_room_alias_update_event(
+                requester, new_room_id,
+            )
+        except SynapseError as e:
+            # again I'm not really expecting this to fail, but if it does, I'd rather
+            # we returned the new room to the client at this point.
+            logger.error(
+                "Unable to send updated alias events in new room: %s", e,
+            )

    @defer.inlineCallbacks
    def create_room(self, requester, config, ratelimit=True,
@@ -164,36 +496,16 @@ class RoomCreationHandler(BaseHandler):
        visibility = config.get("visibility", None)
        is_public = visibility == "public"

-        # autogen room IDs and try to create it. We may clash, so just
-        # try a few times till one goes through, giving up eventually.
-        attempts = 0
-        room_id = None
-        while attempts < 5:
-            try:
-                random_string = stringutils.random_string(18)
-                gen_room_id = RoomID(
-                    random_string,
-                    self.hs.hostname,
-                )
-                yield self.store.store_room(
-                    room_id=gen_room_id.to_string(),
-                    room_creator_user_id=user_id,
-                    is_public=is_public
-                )
-                room_id = gen_room_id.to_string()
-                break
-            except StoreError:
-                attempts += 1
-        if not room_id:
-            raise StoreError(500, "Couldn't generate a room ID.")
+        room_id = yield self._generate_room_id(creator_id=user_id, is_public=is_public)

        if room_alias:
            directory_handler = self.hs.get_handlers().directory_handler
            yield directory_handler.create_association(
-                user_id=user_id,
+                requester=requester,
                room_id=room_id,
                room_alias=room_alias,
                servers=[self.hs.hostname],
+                send_event=False,
            )

        preset_config = config.get(
@@ -214,18 +526,15 @@ class RoomCreationHandler(BaseHandler):
        # override any attempt to set room versions via the creation_content
        creation_content["room_version"] = room_version

-        room_member_handler = self.hs.get_room_member_handler()
-
        yield self._send_events_for_new_room(
            requester,
            room_id,
-            room_member_handler,
            preset_config=preset_config,
            invite_list=invite_list,
            initial_state=initial_state,
            creation_content=creation_content,
            room_alias=room_alias,
-            power_level_content_override=config.get("power_level_content_override", {}),
+            power_level_content_override=config.get("power_level_content_override"),
            creator_join_profile=creator_join_profile,
        )

@@ -261,7 +570,7 @@ class RoomCreationHandler(BaseHandler):
            if is_direct:
                content["is_direct"] = is_direct

-            yield room_member_handler.update_membership(
+            yield self.room_member_handler.update_membership(
                requester,
                UserID.from_string(invitee),
                room_id,
@@ -289,7 +598,7 @@ class RoomCreationHandler(BaseHandler):
        if room_alias:
            result["room_alias"] = room_alias.to_string()
            yield directory_handler.send_room_alias_update_event(
-                requester, user_id, room_id
+                requester, room_id
            )

        defer.returnValue(result)
@@ -299,14 +608,13 @@ class RoomCreationHandler(BaseHandler):
            self,
            creator,  # A Requester object.
            room_id,
-            room_member_handler,
            preset_config,
            invite_list,
            initial_state,
            creation_content,
-            room_alias,
-            power_level_content_override,
-            creator_join_profile,
+            room_alias=None,
+            power_level_content_override=None,
+            creator_join_profile=None,
    ):
        def create(etype, content, **kwargs):
            e = {
@@ -322,6 +630,7 @@ class RoomCreationHandler(BaseHandler):
        @defer.inlineCallbacks
        def send(etype, content, **kwargs):
            event = create(etype, content, **kwargs)
+            logger.info("Sending %s in new room", etype)
            yield self.event_creation_handler.create_and_send_nonmember_event(
                creator,
                event,
@@ -344,7 +653,8 @@ class RoomCreationHandler(BaseHandler):
            content=creation_content,
        )

-        yield room_member_handler.update_membership(
+        logger.info("Sending %s in new room", EventTypes.Member)
+        yield self.room_member_handler.update_membership(
            creator,
            creator.user,
            room_id,
@@ -386,7 +696,8 @@ class RoomCreationHandler(BaseHandler):
                for invitee in invite_list:
                    power_level_content["users"][invitee] = 100

-            power_level_content.update(power_level_content_override)
+            if power_level_content_override:
+                power_level_content.update(power_level_content_override)

            yield send(
                etype=EventTypes.PowerLevels,
@@ -425,6 +736,32 @@ class RoomCreationHandler(BaseHandler):
                content=content,
            )

+    @defer.inlineCallbacks
+    def _generate_room_id(self, creator_id, is_public):
+        # autogen room IDs and try to create it. We may clash, so just
+        # try a few times till one goes through, giving up eventually.
+        attempts = 0
+        while attempts < 5:
+            try:
+                i = self._next_room_id
+                self._next_room_id += 1
+                random_string = stringutils.random_string(3) + str(i)
+                gen_room_id = RoomID(
+                    random_string,
+                    self.hs.hostname,
+                ).to_string()
+                if isinstance(gen_room_id, bytes):
+                    gen_room_id = gen_room_id.decode('utf-8')
+                yield self.store.store_room(
+                    room_id=gen_room_id,
+                    room_creator_user_id=creator_id,
+                    is_public=is_public,
+                )
+                defer.returnValue(gen_room_id)
+            except StoreError:
+                attempts += 1
+        raise StoreError(500, "Couldn't generate a room ID.")
+

 class RoomContextHandler(object):
    def __init__(self, hs):
@@ -488,23 +825,24 @@ class RoomContextHandler(object):
        else:
            last_event_id = event_id

-        types = None
-        filtered_types = None
        if event_filter and event_filter.lazy_load_members():
-            members = set(ev.sender for ev in itertools.chain(
-                results["events_before"],
-                (results["event"],),
-                results["events_after"],
-            ))
-            filtered_types = [EventTypes.Member]
-            types = [(EventTypes.Member, member) for member in members]
+            state_filter = StateFilter.from_lazy_load_member_list(
+                ev.sender
+                for ev in itertools.chain(
+                    results["events_before"],
+                    (results["event"],),
+                    results["events_after"],
+                )
+            )
+        else:
+            state_filter = StateFilter.all()

        # XXX: why do we return the state as of the last event rather than the
        # first? Shouldn't we be consistent with /sync?
        # https://github.com/matrix-org/matrix-doc/issues/687

        state = yield self.store.get_state_for_events(
-            [last_event_id], types, filtered_types=filtered_types,
+            [last_event_id], state_filter=state_filter,
        )
        results["state"] = list(state[last_event_id].values())

@@ -16,7 +16,7 @@
 import logging
 from collections import namedtuple

-from six import iteritems
+from six import PY3, iteritems
 from six.moves import range

 import msgpack
@@ -444,9 +444,16 @@ class RoomListNextBatch(namedtuple("RoomListNextBatch", (

    @classmethod
    def from_token(cls, token):
+        if PY3:
+            # The argument raw=False is only available on new versions of
+            # msgpack, and only really needed on Python 3. Gate it behind
+            # a PY3 check to avoid causing issues on Debian-packaged versions.
+            decoded = msgpack.loads(decode_base64(token), raw=False)
+        else:
+            decoded = msgpack.loads(decode_base64(token))
        return RoomListNextBatch(**{
            cls.REVERSE_KEY_DICT[key]: val
-            for key, val in msgpack.loads(decode_base64(token)).items()
+            for key, val in decoded.items()
        })

    def to_token(self):
@@ -29,7 +29,8 @@ import synapse.server
 import synapse.types
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import AuthError, Codes, SynapseError
-from synapse.types import RoomID, UserID
+from synapse.types import RoomID, UserID, RoomAlias
+from synapse.util import logcontext
 from synapse.util.async_helpers import Linearizer
 from synapse.util.distributor import user_joined_room, user_left_room

@@ -416,6 +417,10 @@ class RoomMemberHandler(object):
                ret = yield self._remote_join(
                    requester, remote_room_hosts, room_id, target, content
                )
+                logcontext.run_in_background(
+                    self._send_merged_user_invites,
+                    requester, room_id,
+                )
                defer.returnValue(ret)

        elif effective_membership_state == Membership.LEAVE:
@@ -450,8 +455,44 @@ class RoomMemberHandler(object):
            prev_events_and_hashes=prev_events_and_hashes,
            content=content,
        )
+        if effective_membership_state == Membership.JOIN:
+            logcontext.run_in_background(
+                self._send_merged_user_invites,
+                requester, room_id,
+            )
        defer.returnValue(res)

+    @defer.inlineCallbacks
+    def _send_merged_user_invites(self, requester, room_id):
+        try:
+            profile_alias = "#_profile_" + requester.user.localpart + ":" + self.hs.hostname
+            profile_alias = RoomAlias.from_string(profile_alias)
+            profile_room_id, remote_room_hosts = yield self.lookup_room_alias(profile_alias)
+            if profile_room_id:
+                linked_accounts = yield self.state_handler.get_current_state(
+                    room_id=profile_room_id.to_string(),
+                    event_type="m.linked_accounts",
+                    state_key="",
+                )
+                if not linked_accounts or not linked_accounts.content['all_children']:
+                    return
+                for child_id in linked_accounts.content['all_children']:
+                    child = UserID.from_string(child_id)
+                    if self.hs.is_mine(child) or child_id == requester.user.to_string():
+                        # TODO: Handle auto-invite for local users (not a priority)
+                        continue
+                    try:
+                        yield self.update_membership(
+                            requester=requester,
+                            target=child,
+                            room_id=room_id,
+                            action="invite",
+                        )
+                    except Exception:
+                        logger.exception("Failed to invite %s to %s" % (child_id, room_id))
+        except Exception:
+            logger.exception("Failed to send invites to children of %s in %s" % (requester.user.to_string(), room_id))
+
    @defer.inlineCallbacks
    def send_membership_event(
            self,
@@ -24,6 +24,7 @@ from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import SynapseError
 from synapse.api.filtering import Filter
 from synapse.events.utils import serialize_event
+from synapse.storage.state import StateFilter
 from synapse.visibility import filter_events_for_client

 from ._base import BaseHandler
@@ -324,9 +325,12 @@ class SearchHandler(BaseHandler):
                    else:
                        last_event_id = event.event_id

+                    state_filter = StateFilter.from_types(
+                        [(EventTypes.Member, sender) for sender in senders]
+                    )
+
                    state = yield self.store.get_state_for_event(
-                        last_event_id,
-                        types=[(EventTypes.Member, sender) for sender in senders]
+                        last_event_id, state_filter
                    )

                    res["profile_info"] = {
@@ -27,6 +27,7 @@ from twisted.internet import defer
 from synapse.api.constants import EventTypes, Membership
 from synapse.push.clientformat import format_push_rules_for_user
 from synapse.storage.roommember import MemberSummary
+from synapse.storage.state import StateFilter
 from synapse.types import RoomStreamToken
 from synapse.util.async_helpers import concurrently_execute
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -469,25 +470,20 @@ class SyncHandler(object):
        ))

    @defer.inlineCallbacks
-    def get_state_after_event(self, event, types=None, filtered_types=None):
+    def get_state_after_event(self, event, state_filter=StateFilter.all()):
        """
        Get the room state after the given event

        Args:
            event(synapse.events.EventBase): event of interest
-            types(list[(str, str|None)]|None): List of (type, state_key) tuples
-                which are used to filter the state fetched. If `state_key` is None,
-                all events are returned of the given type.
-                May be None, which matches any key.
-            filtered_types(list[str]|None): Only apply filtering via `types` to this
-                list of event types.  Other types of events are returned unfiltered.
-                If None, `types` filtering is applied to all events.
+            state_filter (StateFilter): The state filter used to fetch state
+                from the database.

        Returns:
            A Deferred map from ((type, state_key)->Event)
        """
        state_ids = yield self.store.get_state_ids_for_event(
-            event.event_id, types, filtered_types=filtered_types,
+            event.event_id, state_filter=state_filter,
        )
        if event.is_state():
            state_ids = state_ids.copy()
@@ -495,18 +491,14 @@ class SyncHandler(object):
        defer.returnValue(state_ids)

    @defer.inlineCallbacks
-    def get_state_at(self, room_id, stream_position, types=None, filtered_types=None):
+    def get_state_at(self, room_id, stream_position, state_filter=StateFilter.all()):
        """ Get the room state at a particular stream position

        Args:
            room_id(str): room for which to get state
            stream_position(StreamToken): point at which to get state
-            types(list[(str, str|None)]|None): List of (type, state_key) tuples
-                which are used to filter the state fetched. If `state_key` is None,
-                all events are returned of the given type.
-            filtered_types(list[str]|None): Only apply filtering via `types` to this
-                list of event types.  Other types of events are returned unfiltered.
-                If None, `types` filtering is applied to all events.
+            state_filter (StateFilter): The state filter used to fetch state
+                from the database.

        Returns:
            A Deferred map from ((type, state_key)->Event)
@@ -522,7 +514,7 @@ class SyncHandler(object):
        if last_events:
            last_event = last_events[-1]
            state = yield self.get_state_after_event(
-                last_event, types, filtered_types=filtered_types,
+                last_event, state_filter=state_filter,
            )

        else:
@@ -563,10 +555,11 @@ class SyncHandler(object):

        last_event = last_events[-1]
        state_ids = yield self.store.get_state_ids_for_event(
-            last_event.event_id, [
+            last_event.event_id,
+            state_filter=StateFilter.from_types([
                (EventTypes.Name, ''),
                (EventTypes.CanonicalAlias, ''),
-            ]
+            ]),
        )

        # this is heavily cached, thus: fast.
@@ -717,8 +710,7 @@ class SyncHandler(object):

        with Measure(self.clock, "compute_state_delta"):

-            types = None
-            filtered_types = None
+            members_to_fetch = None

            lazy_load_members = sync_config.filter_collection.lazy_load_members()
            include_redundant_members = (
@@ -729,16 +721,21 @@ class SyncHandler(object):
                # We only request state for the members needed to display the
                # timeline:

-                types = [
-                    (EventTypes.Member, state_key)
-                    for state_key in set(
-                        event.sender  # FIXME: we also care about invite targets etc.
-                        for event in batch.events
-                    )
-                ]
+                members_to_fetch = set(
+                    event.sender  # FIXME: we also care about invite targets etc.
+                    for event in batch.events
+                )

-                # only apply the filtering to room members
-                filtered_types = [EventTypes.Member]
+                if full_state:
+                    # always make sure we LL ourselves so we know we're in the room
+                    # (if we are) to fix https://github.com/vector-im/riot-web/issues/7209
+                    # We only need apply this on full state syncs given we disabled
+                    # LL for incr syncs in #3840.
+                    members_to_fetch.add(sync_config.user.to_string())
+
+                state_filter = StateFilter.from_lazy_load_member_list(members_to_fetch)
+            else:
+                state_filter = StateFilter.all()

            timeline_state = {
                (event.type, event.state_key): event.event_id
@@ -746,28 +743,19 @@ class SyncHandler(object):
            }

            if full_state:
-                if lazy_load_members:
-                    # always make sure we LL ourselves so we know we're in the room
-                    # (if we are) to fix https://github.com/vector-im/riot-web/issues/7209
-                    # We only need apply this on full state syncs given we disabled
-                    # LL for incr syncs in #3840.
-                    types.append((EventTypes.Member, sync_config.user.to_string()))
-
                if batch:
                    current_state_ids = yield self.store.get_state_ids_for_event(
-                        batch.events[-1].event_id, types=types,
-                        filtered_types=filtered_types,
+                        batch.events[-1].event_id, state_filter=state_filter,
                    )

                    state_ids = yield self.store.get_state_ids_for_event(
-                        batch.events[0].event_id, types=types,
-                        filtered_types=filtered_types,
+                        batch.events[0].event_id, state_filter=state_filter,
                    )

                else:
                    current_state_ids = yield self.get_state_at(
-                        room_id, stream_position=now_token, types=types,
-                        filtered_types=filtered_types,
+                        room_id, stream_position=now_token,
+                        state_filter=state_filter,
                    )

                    state_ids = current_state_ids
@@ -781,8 +769,7 @@ class SyncHandler(object):
                )
            elif batch.limited:
                state_at_timeline_start = yield self.store.get_state_ids_for_event(
-                    batch.events[0].event_id, types=types,
-                    filtered_types=filtered_types,
+                    batch.events[0].event_id, state_filter=state_filter,
                )

                # for now, we disable LL for gappy syncs - see
@@ -797,17 +784,15 @@ class SyncHandler(object):
                # members to just be ones which were timeline senders, which then ensures
                # all of the rest get included in the state block (if we need to know
                # about them).
-                types = None
-                filtered_types = None
+                state_filter = StateFilter.all()

                state_at_previous_sync = yield self.get_state_at(
-                    room_id, stream_position=since_token, types=types,
-                    filtered_types=filtered_types,
+                    room_id, stream_position=since_token,
+                    state_filter=state_filter,
                )

                current_state_ids = yield self.store.get_state_ids_for_event(
-                    batch.events[-1].event_id, types=types,
-                    filtered_types=filtered_types,
+                    batch.events[-1].event_id, state_filter=state_filter,
                )

                state_ids = _calculate_state(
@@ -821,7 +806,7 @@ class SyncHandler(object):
            else:
                state_ids = {}
                if lazy_load_members:
-                    if types and batch.events:
+                    if members_to_fetch and batch.events:
                        # We're returning an incremental sync, with no
                        # "gap" since the previous sync, so normally there would be
                        # no state to return.
@@ -831,8 +816,12 @@ class SyncHandler(object):
                        # timeline here, and then dedupe any redundant ones below.

                        state_ids = yield self.store.get_state_ids_for_event(
-                            batch.events[0].event_id, types=types,
-                            filtered_types=None,  # we only want members!
+                            batch.events[0].event_id,
+                            # we only want members!
+                            state_filter=StateFilter.from_types(
+                                (EventTypes.Member, member)
+                                for member in members_to_fetch
+                            ),
                        )

            if lazy_load_members and not include_redundant_members:
@@ -942,10 +931,7 @@ class SyncHandler(object):
        newly_joined_rooms, newly_joined_users, _, _ = res
        _, _, newly_left_rooms, newly_left_users = res

-        block_all_presence_data = (
-            since_token is None and
-            sync_config.filter_collection.blocks_all_presence()
-        )
+        block_all_presence_data = True
        if self.hs_config.use_presence and not block_all_presence_data:
            yield self._generate_sync_entry_for_presence(
                sync_result_builder, newly_joined_rooms, newly_joined_users
@@ -1242,10 +1228,7 @@ class SyncHandler(object):
            `(newly_joined_rooms, newly_joined_users, newly_left_rooms, newly_left_users)`
        """
        user_id = sync_result_builder.sync_config.user.to_string()
-        block_all_room_ephemeral = (
-            sync_result_builder.since_token is None and
-            sync_result_builder.sync_config.filter_collection.blocks_all_room_ephemeral()
-        )
+        block_all_room_ephemeral = True

        if block_all_room_ephemeral:
            ephemeral_by_room = {}
@@ -63,11 +63,8 @@ class TypingHandler(object):
        self._member_typing_until = {}  # clock time we expect to stop
        self._member_last_federation_poke = {}

-        # map room IDs to serial numbers
-        self._room_serials = {}
        self._latest_room_serial = 0
-        # map room IDs to sets of users currently typing
-        self._room_typing = {}
+        self._reset()

        # caches which room_ids changed at which serials
        self._typing_stream_change_cache = StreamChangeCache(
@@ -79,6 +76,15 @@ class TypingHandler(object):
            5000,
        )

+    def _reset(self):
+        """
+        Reset the typing handler's data caches.
+        """
+        # map room IDs to serial numbers
+        self._room_serials = {}
+        # map room IDs to sets of users currently typing
+        self._room_typing = {}
+
    def _handle_timeouts(self):
        logger.info("Checking for typing timeouts")

@@ -225,6 +231,7 @@ class TypingHandler(object):
            for domain in set(get_domain_from_id(u) for u in users):
                if domain != self.server_name:
                    logger.debug("sending typing update to %s", domain)
+                    return
                    self.federation.send_edu(
                        destination=domain,
                        edu_type="m.typing",
@@ -20,6 +20,7 @@ from six import iteritems
 from twisted.internet import defer

 from synapse.api.constants import EventTypes, JoinRules, Membership
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.storage.roommember import ProfileInfo
 from synapse.types import get_localpart_from_id
 from synapse.util.metrics import Measure
@@ -98,7 +99,6 @@ class UserDirectoryHandler(object):
        """
        return self.store.search_user_dir(user_id, search_term, limit)

-    @defer.inlineCallbacks
    def notify_new_event(self):
        """Called when there may be more deltas to process
        """
@@ -108,11 +108,15 @@ class UserDirectoryHandler(object):
        if self._is_processing:
            return

+        @defer.inlineCallbacks
+        def process():
+            try:
+                yield self._unsafe_process()
+            finally:
+                self._is_processing = False
+
        self._is_processing = True
-        try:
-            yield self._unsafe_process()
-        finally:
-            self._is_processing = False
+        run_as_background_process("user_directory.notify_new_event", process)

    @defer.inlineCallbacks
    def handle_local_profile_change(self, user_id, profile):
@@ -194,8 +194,11 @@ class _WrappedConnection(object):
            # In Twisted >18.4; the TLS connection will be None if it has closed
            # which will make abortConnection() throw. Check that the TLS connection
            # is not None before trying to close it.
-            if self.transport.getHandle() is not None:
-                self.transport.abortConnection()
+            try:
+                if self.transport.getHandle() is not None:
+                    self.transport.abortConnection()
+            except:
+                logger.warning("Failed to abort connection")

    def request(self, request):
        self.last_request = time.time()
@@ -18,6 +18,7 @@ import logging
 import random
 import sys
 from io import BytesIO
+import os

 from six import PY3, string_types
 from six.moves import urllib
@@ -55,6 +56,7 @@ outgoing_requests_counter = Counter("synapse_http_matrixfederationclient_request
 incoming_responses_counter = Counter("synapse_http_matrixfederationclient_responses",
                                     "", ["method", "code"])

+USE_PROXY = "SYNAPSE_USE_PROXY" in os.environ

 MAX_LONG_RETRIES = 10
 MAX_SHORT_RETRIES = 3
@@ -65,6 +67,18 @@ else:
    MAXINT = sys.maxint


+class ProxyMatrixFederationEndpointFactory(object):
+    def __init__(self, hs):
+        self.reactor = hs.get_reactor()
+        self.tls_client_options_factory = hs.tls_client_options_factory
+
+    def endpointForURI(self, uri):
+        return matrix_federation_endpoint(
+            self.reactor, "localhost:8888", timeout=10,
+            tls_client_options_factory=None
+        )
+
+
 class MatrixFederationEndpointFactory(object):
    def __init__(self, hs):
        self.reactor = hs.get_reactor()
@@ -190,12 +204,27 @@ class MatrixFederationHttpClient(object):
        pool.retryAutomatically = False
        pool.maxPersistentPerHost = 5
        pool.cachedConnectionTimeout = 2 * 60
-        self.agent = Agent.usingEndpointFactory(
-            reactor, MatrixFederationEndpointFactory(hs), pool=pool
+
+        if USE_PROXY:
+            self.agent = Agent.usingEndpointFactory(
+                reactor, ProxyMatrixFederationEndpointFactory(hs), pool=pool
+            )
+        else:
+            self.agent = Agent.usingEndpointFactory(
+                reactor, MatrixFederationEndpointFactory(hs), pool=pool
+            )
+
+        file_pool = HTTPConnectionPool(reactor)
+        file_pool.retryAutomatically = False
+        file_pool.maxPersistentPerHost = 5
+        file_pool.cachedConnectionTimeout = 10
+
+        self.file_agent = Agent.usingEndpointFactory(
+            reactor, MatrixFederationEndpointFactory(hs), pool=file_pool
        )
        self.clock = hs.get_clock()
        self._store = hs.get_datastore()
-        self.version_string = hs.version_string.encode('ascii')
+        self.version_string_bytes = hs.version_string.encode('ascii')
        self.default_timeout = 60

        def schedule(x):
@@ -211,7 +240,8 @@ class MatrixFederationHttpClient(object):
        timeout=None,
        long_retries=False,
        ignore_backoff=False,
-        backoff_on_404=False
+        backoff_on_404=False,
+        agent=None,
    ):
        """
        Sends a request to the given server.
@@ -230,7 +260,7 @@ class MatrixFederationHttpClient(object):
        Returns:
            Deferred: resolves with the http response object on success.

-            Fails with ``HTTPRequestException``: if we get an HTTP response
+            Fails with ``HttpResponseException``: if we get an HTTP response
                code >= 300.

            Fails with ``NotRetryingDestination`` if we are not yet ready
@@ -261,8 +291,8 @@ class MatrixFederationHttpClient(object):
            ignore_backoff=ignore_backoff,
        )

-        method = request.method
-        destination = request.destination
+        method_bytes = request.method.encode("ascii")
+        destination_bytes = request.destination.encode("ascii")
        path_bytes = request.path.encode("ascii")
        if request.query:
            query_bytes = encode_query_args(request.query)
@@ -270,8 +300,8 @@ class MatrixFederationHttpClient(object):
            query_bytes = b""

        headers_dict = {
-            "User-Agent": [self.version_string],
-            "Host": [request.destination],
+            b"User-Agent": [self.version_string_bytes],
+            b"Host": [destination_bytes],
        }

        with limiter:
@@ -282,50 +312,51 @@ class MatrixFederationHttpClient(object):
            else:
                retries_left = MAX_SHORT_RETRIES

-            url = urllib.parse.urlunparse((
-                b"matrix", destination.encode("ascii"),
+            url_bytes = urllib.parse.urlunparse((
+                b"matrix", destination_bytes,
                path_bytes, None, query_bytes, b"",
-            )).decode('ascii')
+            ))
+            url_str = url_bytes.decode('ascii')

-            http_url = urllib.parse.urlunparse((
+            url_to_sign_bytes = urllib.parse.urlunparse((
                b"", b"",
                path_bytes, None, query_bytes, b"",
-            )).decode('ascii')
+            ))

            while True:
                try:
                    json = request.get_json()
                    if json:
-                        data = encode_canonical_json(json)
-                        headers_dict["Content-Type"] = ["application/json"]
+                        headers_dict[b"Content-Type"] = [b"application/json"]
                        self.sign_request(
-                            destination, method, http_url, headers_dict, json
+                            destination_bytes, method_bytes, url_to_sign_bytes,
+                            headers_dict, json,
                        )
-                    else:
-                        data = None
-                        self.sign_request(destination, method, http_url, headers_dict)
-
-                    logger.info(
-                        "{%s} [%s] Sending request: %s %s",
-                        request.txn_id, destination, method, url
-                    )
-
-                    if data:
+                        data = encode_canonical_json(json)
                        producer = FileBodyProducer(
                            BytesIO(data),
-                            cooperator=self._cooperator
+                            cooperator=self._cooperator,
                        )
                    else:
                        producer = None
+                        self.sign_request(
+                            destination_bytes, method_bytes, url_to_sign_bytes,
+                            headers_dict,
+                        )

-                    request_deferred = treq.request(
-                        method,
-                        url,
+                    logger.info(
+                        "{%s} [%s] Sending request: %s %s",
+                        request.txn_id, request.destination, request.method,
+                        url_str,
+                    )
+
+                    # we don't want all the fancy cookie and redirect handling that
+                    # treq.request gives: just use the raw Agent.
+                    request_deferred = self.agent.request(
+                        method_bytes,
+                        url_bytes,
                        headers=Headers(headers_dict),
-                        data=producer,
-                        agent=self.agent,
-                        reactor=self.hs.get_reactor(),
-                        unbuffered=True
+                        bodyProducer=producer,
                    )

                    request_deferred = timeout_deferred(
@@ -344,9 +375,9 @@ class MatrixFederationHttpClient(object):
                    logger.warn(
                        "{%s} [%s] Request failed: %s %s: %s",
                        request.txn_id,
-                        destination,
-                        method,
-                        url,
+                        request.destination,
+                        request.method,
+                        url_str,
                        _flatten_response_never_received(e),
                    )

@@ -366,7 +397,7 @@ class MatrixFederationHttpClient(object):
                        logger.debug(
                            "{%s} [%s] Waiting %ss before re-sending...",
                            request.txn_id,
-                            destination,
+                            request.destination,
                            delay,
                        )

@@ -378,7 +409,7 @@ class MatrixFederationHttpClient(object):
            logger.info(
                "{%s} [%s] Got response headers: %d %s",
                request.txn_id,
-                destination,
+                request.destination,
                response.code,
                response.phrase.decode('ascii', errors='replace'),
            )
@@ -411,8 +442,9 @@ class MatrixFederationHttpClient(object):
                destination_is must be non-None.
            method (bytes): The HTTP method of the request
            url_bytes (bytes): The URI path of the request
-            headers_dict (dict): Dictionary of request headers to append to
-            content (bytes): The body of the request
+            headers_dict (dict[bytes, list[bytes]]): Dictionary of request headers to
+                append to
+            content (object): The body of the request
            destination_is (bytes): As 'destination', but if the destination is an
                identity server

@@ -478,7 +510,7 @@ class MatrixFederationHttpClient(object):
            Deferred: Succeeds when we get a 2xx HTTP response. The result
            will be the decoded JSON body.

-            Fails with ``HTTPRequestException`` if we get an HTTP response
+            Fails with ``HttpResponseException`` if we get an HTTP response
            code >= 300.

            Fails with ``NotRetryingDestination`` if we are not yet ready
@@ -532,7 +564,7 @@ class MatrixFederationHttpClient(object):
            Deferred: Succeeds when we get a 2xx HTTP response. The result
            will be the decoded JSON body.

-            Fails with ``HTTPRequestException`` if we get an HTTP response
+            Fails with ``HttpResponseException`` if we get an HTTP response
            code >= 300.

            Fails with ``NotRetryingDestination`` if we are not yet ready
@@ -587,7 +619,7 @@ class MatrixFederationHttpClient(object):
            Deferred: Succeeds when we get a 2xx HTTP response. The result
            will be the decoded JSON body.

-            Fails with ``HTTPRequestException`` if we get an HTTP response
+            Fails with ``HttpResponseException`` if we get an HTTP response
            code >= 300.

            Fails with ``NotRetryingDestination`` if we are not yet ready
@@ -638,7 +670,7 @@ class MatrixFederationHttpClient(object):
            Deferred: Succeeds when we get a 2xx HTTP response. The result
            will be the decoded JSON body.

-            Fails with ``HTTPRequestException`` if we get an HTTP response
+            Fails with ``HttpResponseException`` if we get an HTTP response
            code >= 300.

            Fails with ``NotRetryingDestination`` if we are not yet ready
@@ -682,7 +714,7 @@ class MatrixFederationHttpClient(object):
            Deferred: resolves with an (int,dict) tuple of the file length and
            a dict of the response headers.

-            Fails with ``HTTPRequestException`` if we get an HTTP response code
+            Fails with ``HttpResponseException`` if we get an HTTP response code
            >= 300

            Fails with ``NotRetryingDestination`` if we are not yet ready
@@ -702,6 +734,7 @@ class MatrixFederationHttpClient(object):
            request,
            retry_on_dns_fail=retry_on_dns_fail,
            ignore_backoff=ignore_backoff,
+            agent=self.file_agent,
        )

        headers = dict(response.headers.getAllRawHeaders())
@@ -39,7 +39,8 @@ outgoing_responses_counter = Counter(
 )

 response_timer = Histogram(
-    "synapse_http_server_response_time_seconds", "sec",
+    "synapse_http_server_response_time_seconds",
+    "sec",
    ["method", "servlet", "tag", "code"],
 )

@@ -79,15 +80,11 @@ response_size = Counter(
 # than when the response was written.

 in_flight_requests_ru_utime = Counter(
-    "synapse_http_server_in_flight_requests_ru_utime_seconds",
-    "",
-    ["method", "servlet"],
+    "synapse_http_server_in_flight_requests_ru_utime_seconds", "", ["method", "servlet"]
 )

 in_flight_requests_ru_stime = Counter(
-    "synapse_http_server_in_flight_requests_ru_stime_seconds",
-    "",
-    ["method", "servlet"],
+    "synapse_http_server_in_flight_requests_ru_stime_seconds", "", ["method", "servlet"]
 )

 in_flight_requests_db_txn_count = Counter(
@@ -134,7 +131,7 @@ def _get_in_flight_counts():
    # type
    counts = {}
    for rm in reqs:
-        key = (rm.method, rm.name,)
+        key = (rm.method, rm.name)
        counts[key] = counts.get(key, 0) + 1

    return counts
@@ -175,7 +172,8 @@ class RequestMetrics(object):
            if context != self.start_context:
                logger.warn(
                    "Context have unexpectedly changed %r, %r",
-                    context, self.start_context
+                    context,
+                    self.start_context,
                )
                return

@@ -192,10 +190,10 @@ class RequestMetrics(object):
        resource_usage = context.get_resource_usage()

        response_ru_utime.labels(self.method, self.name, tag).inc(
-            resource_usage.ru_utime,
+            resource_usage.ru_utime
        )
        response_ru_stime.labels(self.method, self.name, tag).inc(
-            resource_usage.ru_stime,
+            resource_usage.ru_stime
        )
        response_db_txn_count.labels(self.method, self.name, tag).inc(
            resource_usage.db_txn_count
@@ -222,8 +220,15 @@ class RequestMetrics(object):
        diff = new_stats - self._request_stats
        self._request_stats = new_stats

-        in_flight_requests_ru_utime.labels(self.method, self.name).inc(diff.ru_utime)
-        in_flight_requests_ru_stime.labels(self.method, self.name).inc(diff.ru_stime)
+        # max() is used since rapid use of ru_stime/ru_utime can end up with the
+        # count going backwards due to NTP, time smearing, fine-grained
+        # correction, or floating points. Who knows, really?
+        in_flight_requests_ru_utime.labels(self.method, self.name).inc(
+            max(diff.ru_utime, 0)
+        )
+        in_flight_requests_ru_stime.labels(self.method, self.name).inc(
+            max(diff.ru_stime, 0)
+        )

        in_flight_requests_db_txn_count.labels(self.method, self.name).inc(
            diff.db_txn_count
@@ -468,13 +468,13 @@ def set_cors_headers(request):
    Args:
        request (twisted.web.http.Request): The http request to add CORs to.
    """
-    request.setHeader("Access-Control-Allow-Origin", "*")
+    request.setHeader(b"Access-Control-Allow-Origin", b"*")
    request.setHeader(
-        "Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS"
+        b"Access-Control-Allow-Methods", b"GET, POST, PUT, DELETE, OPTIONS"
    )
    request.setHeader(
-        "Access-Control-Allow-Headers",
-        "Origin, X-Requested-With, Content-Type, Accept, Authorization"
+        b"Access-Control-Allow-Headers",
+        b"Origin, X-Requested-With, Content-Type, Accept, Authorization"
    )


@@ -121,16 +121,15 @@ def parse_string(request, name, default=None, required=False,

    Args:
        request: the twisted HTTP request.
-        name (bytes/unicode): the name of the query parameter.
-        default (bytes/unicode|None): value to use if the parameter is absent,
+        name (bytes|unicode): the name of the query parameter.
+        default (bytes|unicode|None): value to use if the parameter is absent,
            defaults to None. Must be bytes if encoding is None.
        required (bool): whether to raise a 400 SynapseError if the
            parameter is absent, defaults to False.
-        allowed_values (list[bytes/unicode]): List of allowed values for the
+        allowed_values (list[bytes|unicode]): List of allowed values for the
            string, or None if any value is allowed, defaults to None. Must be
            the same type as name, if given.
-        encoding: The encoding to decode the name to, and decode the string
-            content with.
+        encoding (str|None): The encoding to decode the string content with.

    Returns:
        bytes/unicode|None: A string value or the default. Unicode if encoding
@@ -186,9 +186,9 @@ class Notifier(object):
        def count_listeners():
            all_user_streams = set()

-            for x in self.room_to_user_streams.values():
+            for x in list(self.room_to_user_streams.values()):
                all_user_streams |= x
-            for x in self.user_to_user_stream.values():
+            for x in list(self.user_to_user_stream.values()):
                all_user_streams.add(x)

            return sum(stream.count_listeners() for stream in all_user_streams)
@@ -196,7 +196,7 @@ class Notifier(object):

        LaterGauge(
            "synapse_notifier_rooms", "", [],
-            lambda: count(bool, self.room_to_user_streams.values()),
+            lambda: count(bool, list(self.room_to_user_streams.values())),
        )
        LaterGauge(
            "synapse_notifier_users", "", [],
@@ -18,8 +18,7 @@ import logging
 from twisted.internet import defer
 from twisted.internet.error import AlreadyCalled, AlreadyCancelled

-from synapse.util.logcontext import LoggingContext
-from synapse.util.metrics import Measure
+from synapse.metrics.background_process_metrics import run_as_background_process

 logger = logging.getLogger(__name__)

@@ -71,18 +70,11 @@ class EmailPusher(object):
        # See httppusher
        self.max_stream_ordering = None

-        self.processing = False
+        self._is_processing = False

-    @defer.inlineCallbacks
    def on_started(self):
        if self.mailer is not None:
-            try:
-                self.throttle_params = yield self.store.get_throttle_params_by_room(
-                    self.pusher_id
-                )
-                yield self._process()
-            except Exception:
-                logger.exception("Error starting email pusher")
+            self._start_processing()

    def on_stop(self):
        if self.timed_call:
@@ -92,43 +84,55 @@ class EmailPusher(object):
                pass
            self.timed_call = None

-    @defer.inlineCallbacks
    def on_new_notifications(self, min_stream_ordering, max_stream_ordering):
-        self.max_stream_ordering = max(max_stream_ordering, self.max_stream_ordering)
-        yield self._process()
+        if self.max_stream_ordering:
+            self.max_stream_ordering = max(max_stream_ordering, self.max_stream_ordering)
+        else:
+            self.max_stream_ordering = max_stream_ordering
+        self._start_processing()

    def on_new_receipts(self, min_stream_id, max_stream_id):
        # We could wake up and cancel the timer but there tend to be quite a
        # lot of read receipts so it's probably less work to just let the
        # timer fire
-        return defer.succeed(None)
+        pass

-    @defer.inlineCallbacks
    def on_timer(self):
        self.timed_call = None
-        yield self._process()
+        self._start_processing()
+
+    def _start_processing(self):
+        if self._is_processing:
+            return
+
+        run_as_background_process("emailpush.process", self._process)

    @defer.inlineCallbacks
    def _process(self):
-        if self.processing:
-            return
+        # we should never get here if we are already processing
+        assert not self._is_processing

-        with LoggingContext("emailpush._process"):
-            with Measure(self.clock, "emailpush._process"):
+        try:
+            self._is_processing = True
+
+            if self.throttle_params is None:
+                # this is our first loop: load up the throttle params
+                self.throttle_params = yield self.store.get_throttle_params_by_room(
+                    self.pusher_id
+                )
+
+            # if the max ordering changes while we're running _unsafe_process,
+            # call it again, and so on until we've caught up.
+            while True:
+                starting_max_ordering = self.max_stream_ordering
                try:
-                    self.processing = True
-                    # if the max ordering changes while we're running _unsafe_process,
-                    # call it again, and so on until we've caught up.
-                    while True:
-                        starting_max_ordering = self.max_stream_ordering
-                        try:
-                            yield self._unsafe_process()
-                        except Exception:
-                            logger.exception("Exception processing notifs")
-                        if self.max_stream_ordering == starting_max_ordering:
-                            break
-                finally:
-                    self.processing = False
+                    yield self._unsafe_process()
+                except Exception:
+                    logger.exception("Exception processing notifs")
+                if self.max_stream_ordering == starting_max_ordering:
+                    break
+        finally:
+            self._is_processing = False

    @defer.inlineCallbacks
    def _unsafe_process(self):
@@ -22,9 +22,8 @@ from prometheus_client import Counter
 from twisted.internet import defer
 from twisted.internet.error import AlreadyCalled, AlreadyCancelled

+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.push import PusherConfigException
-from synapse.util.logcontext import LoggingContext
-from synapse.util.metrics import Measure

 from . import push_rule_evaluator, push_tools

@@ -61,7 +60,7 @@ class HttpPusher(object):
        self.backoff_delay = HttpPusher.INITIAL_BACKOFF_SEC
        self.failing_since = pusherdict['failing_since']
        self.timed_call = None
-        self.processing = False
+        self._is_processing = False

        # This is the highest stream ordering we know it's safe to process.
        # When new events arrive, we'll be given a window of new events: we
@@ -92,34 +91,27 @@ class HttpPusher(object):
        self.data_minus_url.update(self.data)
        del self.data_minus_url['url']

-    @defer.inlineCallbacks
    def on_started(self):
-        try:
-            yield self._process()
-        except Exception:
-            logger.exception("Error starting http pusher")
+        self._start_processing()

-    @defer.inlineCallbacks
    def on_new_notifications(self, min_stream_ordering, max_stream_ordering):
        self.max_stream_ordering = max(max_stream_ordering, self.max_stream_ordering or 0)
-        yield self._process()
+        self._start_processing()

-    @defer.inlineCallbacks
    def on_new_receipts(self, min_stream_id, max_stream_id):
        # Note that the min here shouldn't be relied upon to be accurate.

        # We could check the receipts are actually m.read receipts here,
        # but currently that's the only type of receipt anyway...
-        with LoggingContext("push.on_new_receipts"):
-            with Measure(self.clock, "push.on_new_receipts"):
-                badge = yield push_tools.get_badge_count(
-                    self.hs.get_datastore(), self.user_id
-                )
-            yield self._send_badge(badge)
+        run_as_background_process("http_pusher.on_new_receipts", self._update_badge)

    @defer.inlineCallbacks
+    def _update_badge(self):
+        badge = yield push_tools.get_badge_count(self.hs.get_datastore(), self.user_id)
+        yield self._send_badge(badge)
+
    def on_timer(self):
-        yield self._process()
+        self._start_processing()

    def on_stop(self):
        if self.timed_call:
@@ -129,27 +121,31 @@ class HttpPusher(object):
                pass
            self.timed_call = None

-    @defer.inlineCallbacks
-    def _process(self):
-        if self.processing:
+    def _start_processing(self):
+        if self._is_processing:
            return

-        with LoggingContext("push._process"):
-            with Measure(self.clock, "push._process"):
+        run_as_background_process("httppush.process", self._process)
+
+    @defer.inlineCallbacks
+    def _process(self):
+        # we should never get here if we are already processing
+        assert not self._is_processing
+
+        try:
+            self._is_processing = True
+            # if the max ordering changes while we're running _unsafe_process,
+            # call it again, and so on until we've caught up.
+            while True:
+                starting_max_ordering = self.max_stream_ordering
                try:
-                    self.processing = True
-                    # if the max ordering changes while we're running _unsafe_process,
-                    # call it again, and so on until we've caught up.
-                    while True:
-                        starting_max_ordering = self.max_stream_ordering
-                        try:
-                            yield self._unsafe_process()
-                        except Exception:
-                            logger.exception("Exception processing notifs")
-                        if self.max_stream_ordering == starting_max_ordering:
-                            break
-                finally:
-                    self.processing = False
+                    yield self._unsafe_process()
+                except Exception:
+                    logger.exception("Exception processing notifs")
+                if self.max_stream_ordering == starting_max_ordering:
+                    break
+        finally:
+            self._is_processing = False

    @defer.inlineCallbacks
    def _unsafe_process(self):
@@ -315,10 +311,10 @@ class HttpPusher(object):
                ]
            }
        }
-        if event.type == 'm.room.member':
+        if event.type == 'm.room.member' and event.is_state():
            d['notification']['membership'] = event.content['membership']
            d['notification']['user_is_target'] = event.state_key == self.user_id
-        if self.hs.config.push_include_content and 'content' in event:
+        if self.hs.config.push_include_content and event.content:
            d['notification']['content'] = event.content

        # We no longer send aliases separately, instead, we send the human
@@ -26,7 +26,6 @@ import bleach
 import jinja2

 from twisted.internet import defer
-from twisted.mail.smtp import sendmail

 from synapse.api.constants import EventTypes
 from synapse.api.errors import StoreError
@@ -85,6 +84,7 @@ class Mailer(object):
        self.notif_template_html = notif_template_html
        self.notif_template_text = notif_template_text

+        self.sendmail = self.hs.get_sendmail()
        self.store = self.hs.get_datastore()
        self.macaroon_gen = self.hs.get_macaroon_generator()
        self.state_handler = self.hs.get_state_handler()
@@ -191,11 +191,11 @@ class Mailer(object):
        multipart_msg.attach(html_part)

        logger.info("Sending email push notification to %s" % email_address)
-        # logger.debug(html_text)

-        yield sendmail(
+        yield self.sendmail(
            self.hs.config.email_smtp_host,
-            raw_from, raw_to, multipart_msg.as_string(),
+            raw_from, raw_to, multipart_msg.as_string().encode('utf8'),
+            reactor=self.hs.get_reactor(),
            port=self.hs.config.email_smtp_port,
            requireAuthentication=self.hs.config.email_smtp_user is not None,
            username=self.hs.config.email_smtp_user,
@@ -333,7 +333,7 @@ class Mailer(object):
                          notif_events, user_id, reason):
        if len(notifs_by_room) == 1:
            # Only one room has new stuff
-            room_id = notifs_by_room.keys()[0]
+            room_id = list(notifs_by_room.keys())[0]

            # If the room has some kind of name, use it, but we don't
            # want the generated-from-names one here otherwise we'll
@@ -526,12 +526,8 @@ def load_jinja2_templates(config):
    Returns:
        (notif_template_html, notif_template_text)
    """
-    logger.info("loading jinja2")
-
-    if config.email_template_dir:
-        loader = jinja2.FileSystemLoader(config.email_template_dir)
-    else:
-        loader = jinja2.PackageLoader('synapse', 'res/templates')
+    logger.info("loading email templates from '%s'", config.email_template_dir)
+    loader = jinja2.FileSystemLoader(config.email_template_dir)
    env = jinja2.Environment(loader=loader)
    env.filters["format_ts"] = format_ts_filter
    env.filters["mxc_to_http"] = _create_mxc_to_http_filter(config)
@@ -124,7 +124,7 @@ class PushRuleEvaluatorForEvent(object):

        # XXX: optimisation: cache our pattern regexps
        if condition['key'] == 'content.body':
-            body = self._event["content"].get("body", None)
+            body = self._event.content.get("body", None)
            if not body:
                return False

@@ -140,7 +140,7 @@ class PushRuleEvaluatorForEvent(object):
        if not display_name:
            return False

-        body = self._event["content"].get("body", None)
+        body = self._event.content.get("body", None)
        if not body:
            return False

@@ -20,24 +20,39 @@ from twisted.internet import defer

 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.push.pusher import PusherFactory
-from synapse.util.logcontext import make_deferred_yieldable, run_in_background

 logger = logging.getLogger(__name__)


 class PusherPool:
+    """
+    The pusher pool. This is responsible for dispatching notifications of new events to
+    the http and email pushers.
+
+    It provides three methods which are designed to be called by the rest of the
+    application: `start`, `on_new_notifications`, and `on_new_receipts`: each of these
+    delegates to each of the relevant pushers.
+
+    Note that it is expected that each pusher will have its own 'processing' loop which
+    will send out the notifications in the background, rather than blocking until the
+    notifications are sent; accordingly Pusher.on_started, Pusher.on_new_notifications and
+    Pusher.on_new_receipts are not expected to return deferreds.
+    """
    def __init__(self, _hs):
        self.hs = _hs
        self.pusher_factory = PusherFactory(_hs)
-        self.start_pushers = _hs.config.start_pushers
+        self._should_start_pushers = _hs.config.start_pushers
        self.store = self.hs.get_datastore()
        self.clock = self.hs.get_clock()
        self.pushers = {}

-    @defer.inlineCallbacks
    def start(self):
-        pushers = yield self.store.get_all_pushers()
-        self._start_pushers(pushers)
+        """Starts the pushers off in a background process.
+        """
+        if not self._should_start_pushers:
+            logger.info("Not starting pushers because they are disabled in the config")
+            return
+        run_as_background_process("start_pushers", self._start_pushers)

    @defer.inlineCallbacks
    def add_pusher(self, user_id, access_token, kind, app_id,
@@ -86,7 +101,7 @@ class PusherPool:
            last_stream_ordering=last_stream_ordering,
            profile_tag=profile_tag,
        )
-        yield self._refresh_pusher(app_id, pushkey, user_id)
+        yield self.start_pusher_by_id(app_id, pushkey, user_id)

    @defer.inlineCallbacks
    def remove_pushers_by_app_id_and_pushkey_not_user(self, app_id, pushkey,
@@ -123,45 +138,23 @@ class PusherPool:
                    p['app_id'], p['pushkey'], p['user_name'],
                )

-    def on_new_notifications(self, min_stream_id, max_stream_id):
-        run_as_background_process(
-            "on_new_notifications",
-            self._on_new_notifications, min_stream_id, max_stream_id,
-        )
-
    @defer.inlineCallbacks
-    def _on_new_notifications(self, min_stream_id, max_stream_id):
+    def on_new_notifications(self, min_stream_id, max_stream_id):
        try:
            users_affected = yield self.store.get_push_action_users_in_range(
                min_stream_id, max_stream_id
            )

-            deferreds = []
-
            for u in users_affected:
                if u in self.pushers:
                    for p in self.pushers[u].values():
-                        deferreds.append(
-                            run_in_background(
-                                p.on_new_notifications,
-                                min_stream_id, max_stream_id,
-                            )
-                        )
+                        p.on_new_notifications(min_stream_id, max_stream_id)

-            yield make_deferred_yieldable(
-                defer.gatherResults(deferreds, consumeErrors=True),
-            )
        except Exception:
            logger.exception("Exception in pusher on_new_notifications")

-    def on_new_receipts(self, min_stream_id, max_stream_id, affected_room_ids):
-        run_as_background_process(
-            "on_new_receipts",
-            self._on_new_receipts, min_stream_id, max_stream_id, affected_room_ids,
-        )
-
    @defer.inlineCallbacks
-    def _on_new_receipts(self, min_stream_id, max_stream_id, affected_room_ids):
+    def on_new_receipts(self, min_stream_id, max_stream_id, affected_room_ids):
        try:
            # Need to subtract 1 from the minimum because the lower bound here
            # is not inclusive
@@ -171,26 +164,20 @@ class PusherPool:
            # This returns a tuple, user_id is at index 3
            users_affected = set([r[3] for r in updated_receipts])

-            deferreds = []
-
            for u in users_affected:
                if u in self.pushers:
                    for p in self.pushers[u].values():
-                        deferreds.append(
-                            run_in_background(
-                                p.on_new_receipts,
-                                min_stream_id, max_stream_id,
-                            )
-                        )
+                        p.on_new_receipts(min_stream_id, max_stream_id)

-            yield make_deferred_yieldable(
-                defer.gatherResults(deferreds, consumeErrors=True),
-            )
        except Exception:
            logger.exception("Exception in pusher on_new_receipts")

    @defer.inlineCallbacks
-    def _refresh_pusher(self, app_id, pushkey, user_id):
+    def start_pusher_by_id(self, app_id, pushkey, user_id):
+        """Look up the details for the given pusher, and start it"""
+        if not self._should_start_pushers:
+            return
+
        resultlist = yield self.store.get_pushers_by_app_id_and_pushkey(
            app_id, pushkey
        )
@@ -201,34 +188,50 @@ class PusherPool:
                p = r

        if p:
+            self._start_pusher(p)

-            self._start_pushers([p])
+    @defer.inlineCallbacks
+    def _start_pushers(self):
+        """Start all the pushers

-    def _start_pushers(self, pushers):
-        if not self.start_pushers:
-            logger.info("Not starting pushers because they are disabled in the config")
-            return
+        Returns:
+            Deferred
+        """
+        pushers = yield self.store.get_all_pushers()
        logger.info("Starting %d pushers", len(pushers))
        for pusherdict in pushers:
-            try:
-                p = self.pusher_factory.create_pusher(pusherdict)
-            except Exception:
-                logger.exception("Couldn't start a pusher: caught Exception")
-                continue
-            if p:
-                appid_pushkey = "%s:%s" % (
-                    pusherdict['app_id'],
-                    pusherdict['pushkey'],
-                )
-                byuser = self.pushers.setdefault(pusherdict['user_name'], {})
-
-                if appid_pushkey in byuser:
-                    byuser[appid_pushkey].on_stop()
-                byuser[appid_pushkey] = p
-                run_in_background(p.on_started)
-
+            self._start_pusher(pusherdict)
        logger.info("Started pushers")

+    def _start_pusher(self, pusherdict):
+        """Start the given pusher
+
+        Args:
+            pusherdict (dict):
+
+        Returns:
+            None
+        """
+        try:
+            p = self.pusher_factory.create_pusher(pusherdict)
+        except Exception:
+            logger.exception("Couldn't start a pusher: caught Exception")
+            return
+
+        if not p:
+            return
+
+        appid_pushkey = "%s:%s" % (
+            pusherdict['app_id'],
+            pusherdict['pushkey'],
+        )
+        byuser = self.pushers.setdefault(pusherdict['user_name'], {})
+
+        if appid_pushkey in byuser:
+            byuser[appid_pushkey].on_stop()
+        byuser[appid_pushkey] = p
+        p.on_started()
+
    @defer.inlineCallbacks
    def remove_pusher(self, app_id, pushkey, user_id):
        appid_pushkey = "%s:%s" % (app_id, pushkey)
@@ -51,11 +51,11 @@ REQUIREMENTS = {
    "daemonize>=2.3.1": ["daemonize"],
    "bcrypt>=3.1.0": ["bcrypt>=3.1.0"],
    "pillow>=3.1.2": ["PIL"],
-    "pydenticon>=0.2": ["pydenticon"],
    "sortedcontainers>=1.4.4": ["sortedcontainers"],
+    "psutil>=2.0.0": ["psutil>=2.0.0"],
    "pysaml2>=3.0.0": ["saml2"],
    "pymacaroons-pynacl>=0.9.3": ["pymacaroons"],
-    "msgpack-python>=0.3.0": ["msgpack"],
+    "msgpack-python>=0.4.2": ["msgpack"],
    "phonenumbers>=8.2.0": ["phonenumbers"],
    "six>=1.10": ["six"],

@@ -79,9 +79,6 @@ CONDITIONAL_REQUIREMENTS = {
    "matrix-synapse-ldap3": {
        "matrix-synapse-ldap3>=0.1": ["ldap_auth_provider"],
    },
-    "psutil": {
-        "psutil>=2.0.0": ["psutil>=2.0.0"],
-    },
    "postgres": {
        "psycopg2>=2.6": ["psycopg2"]
    }
@@ -106,7 +106,7 @@ class ReplicationClientHandler(object):

        Can be overriden in subclasses to handle more.
        """
-        logger.info("Received rdata %s -> %s", stream_name, token)
+        logger.debug("Received rdata %s -> %s", stream_name, token)
        return self.store.process_replication_rows(stream_name, token, rows)

    def on_position(self, stream_name, token):
@@ -656,7 +656,7 @@ tcp_inbound_commands = LaterGauge(
    "",
    ["command", "name"],
    lambda: {
-        (k[0], p.name,): count
+        (k, p.name,): count
        for p in connected_connections
        for k, count in iteritems(p.inbound_commands_counter)
    },
@@ -667,7 +667,7 @@ tcp_outbound_commands = LaterGauge(
    "",
    ["command", "name"],
    lambda: {
-        (k[0], p.name,): count
+        (k, p.name,): count
        for p in connected_connections
        for k, count in iteritems(p.outbound_commands_counter)
    },
@@ -47,6 +47,7 @@ from synapse.rest.client.v2_alpha import (
    register,
    report_event,
    room_keys,
+    room_upgrade_rest_servlet,
    sendtodevice,
    sync,
    tags,
@@ -116,3 +117,4 @@ class ClientRestResource(JsonResource):
        sendtodevice.register_servlets(hs, client_resource)
        user_directory.register_servlets(hs, client_resource)
        groups.register_servlets(hs, client_resource)
+        room_upgrade_rest_servlet.register_servlets(hs, client_resource)
@@ -74,38 +74,11 @@ class ClientDirectoryServer(ClientV1RestServlet):
        if room is None:
            raise SynapseError(400, "Room does not exist")

-        dir_handler = self.handlers.directory_handler
+        requester = yield self.auth.get_user_by_req(request)

-        try:
-            # try to auth as a user
-            requester = yield self.auth.get_user_by_req(request)
-            try:
-                user_id = requester.user.to_string()
-                yield dir_handler.create_association(
-                    user_id, room_alias, room_id, servers
-                )
-                yield dir_handler.send_room_alias_update_event(
-                    requester,
-                    user_id,
-                    room_id
-                )
-            except SynapseError as e:
-                raise e
-            except Exception:
-                logger.exception("Failed to create association")
-                raise
-        except AuthError:
-            # try to auth as an application service
-            service = yield self.auth.get_appservice_by_req(request)
-            yield dir_handler.create_appservice_association(
-                service, room_alias, room_id, servers
-            )
-            logger.info(
-                "Application service at %s created alias %s pointing to %s",
-                service.url,
-                room_alias.to_string(),
-                room_id
-            )
+        yield self.handlers.directory_handler.create_association(
+            requester, room_alias, room_id, servers
+        )

        defer.returnValue((200, {}))

@@ -135,7 +108,7 @@ class ClientDirectoryServer(ClientV1RestServlet):
        room_alias = RoomAlias.from_string(room_alias)

        yield dir_handler.delete_association(
-            requester, user.to_string(), room_alias
+            requester, room_alias
        )

        logger.info(
@@ -33,6 +33,7 @@ from synapse.http.servlet import (
    parse_json_object_from_request,
    parse_string,
 )
+from synapse.storage.state import StateFilter
 from synapse.streams.config import PaginationConfig
 from synapse.types import RoomAlias, RoomID, StreamToken, ThirdPartyInstanceID, UserID

@@ -409,7 +410,7 @@ class RoomMemberListRestServlet(ClientV1RestServlet):
            room_id=room_id,
            user_id=requester.user.to_string(),
            at_token=at_token,
-            types=[(EventTypes.Member, None)],
+            state_filter=StateFilter.from_types([(EventTypes.Member, None)]),
        )

        chunk = []
@@ -68,6 +68,29 @@ function captchaDone() {
 </html>
 """

+TERMS_TEMPLATE = """
+<html>
+<head>
+<title>Authentication</title>
+<meta name='viewport' content='width=device-width, initial-scale=1,
+    user-scalable=no, minimum-scale=1.0, maximum-scale=1.0'>
+<link rel="stylesheet" href="/_matrix/static/client/register/style.css">
+</head>
+<body>
+<form id="registrationForm" method="post" action="%(myurl)s">
+    <div>
+        <p>
+            Please click the button below if you agree to the
+            <a href="%(terms_url)s">privacy policy of this homeserver.</a>
+        </p>
+        <input type="hidden" name="session" value="%(session)s" />
+        <input type="submit" value="Agree" />
+    </div>
+</form>
+</body>
+</html>
+"""
+
 SUCCESS_TEMPLATE = """
 <html>
 <head>
@@ -99,7 +122,7 @@ class AuthRestServlet(RestServlet):
    cannot be handled in the normal flow (with requests to the same endpoint).
    Current use is for web fallback auth.
    """
-    PATTERNS = client_v2_patterns("/auth/(?P<stagetype>[\w\.]*)/fallback/web")
+    PATTERNS = client_v2_patterns(r"/auth/(?P<stagetype>[\w\.]*)/fallback/web")

    def __init__(self, hs):
        super(AuthRestServlet, self).__init__()
@@ -130,6 +153,27 @@ class AuthRestServlet(RestServlet):
            request.setHeader(b"Content-Type", b"text/html; charset=utf-8")
            request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),))

+            request.write(html_bytes)
+            finish_request(request)
+            defer.returnValue(None)
+        elif stagetype == LoginType.TERMS:
+            session = request.args['session'][0]
+
+            html = TERMS_TEMPLATE % {
+                'session': session,
+                'terms_url': "%s/_matrix/consent?v=%s" % (
+                    self.hs.config.public_baseurl,
+                    self.hs.config.user_consent_version,
+                ),
+                'myurl': "%s/auth/%s/fallback/web" % (
+                    CLIENT_V2_ALPHA_PREFIX, LoginType.TERMS
+                ),
+            }
+            html_bytes = html.encode("utf8")
+            request.setResponseCode(200)
+            request.setHeader(b"Content-Type", b"text/html; charset=utf-8")
+            request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),))
+
            request.write(html_bytes)
            finish_request(request)
            defer.returnValue(None)
@@ -139,7 +183,7 @@ class AuthRestServlet(RestServlet):
    @defer.inlineCallbacks
    def on_POST(self, request, stagetype):
        yield
-        if stagetype == "m.login.recaptcha":
+        if stagetype == LoginType.RECAPTCHA:
            if ('g-recaptcha-response' not in request.args or
                    len(request.args['g-recaptcha-response'])) == 0:
                raise SynapseError(400, "No captcha response supplied")
@@ -178,6 +222,41 @@ class AuthRestServlet(RestServlet):
            request.write(html_bytes)
            finish_request(request)

+            defer.returnValue(None)
+        elif stagetype == LoginType.TERMS:
+            if ('session' not in request.args or
+                    len(request.args['session'])) == 0:
+                raise SynapseError(400, "No session supplied")
+
+            session = request.args['session'][0]
+            authdict = {'session': session}
+
+            success = yield self.auth_handler.add_oob_auth(
+                LoginType.TERMS,
+                authdict,
+                self.hs.get_ip_from_request(request)
+            )
+
+            if success:
+                html = SUCCESS_TEMPLATE
+            else:
+                html = TERMS_TEMPLATE % {
+                    'session': session,
+                    'terms_url': "%s/_matrix/consent?v=%s" % (
+                        self.hs.config.public_baseurl,
+                        self.hs.config.user_consent_version,
+                    ),
+                    'myurl': "%s/auth/%s/fallback/web" % (
+                        CLIENT_V2_ALPHA_PREFIX, LoginType.TERMS
+                    ),
+                }
+            html_bytes = html.encode("utf8")
+            request.setResponseCode(200)
+            request.setHeader(b"Content-Type", b"text/html; charset=utf-8")
+            request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),))
+
+            request.write(html_bytes)
+            finish_request(request)
            defer.returnValue(None)
        else:
            raise SynapseError(404, "Unknown auth stage type")
@@ -359,6 +359,13 @@ class RegisterRestServlet(RestServlet):
                    [LoginType.MSISDN, LoginType.EMAIL_IDENTITY]
                ])

+        # Append m.login.terms to all flows if we're requiring consent
+        if self.hs.config.user_consent_at_registration:
+            new_flows = []
+            for flow in flows:
+                flow.append(LoginType.TERMS)
+            flows.extend(new_flows)
+
        auth_result, params, session_id = yield self.auth_handler.check_auth(
            flows, body, self.hs.get_ip_from_request(request)
        )
@@ -445,6 +452,12 @@ class RegisterRestServlet(RestServlet):
                params.get("bind_msisdn")
            )

+        if auth_result and LoginType.TERMS in auth_result:
+            logger.info("%s has consented to the privacy policy" % registered_user_id)
+            yield self.store.user_set_consent_version(
+                registered_user_id, self.hs.config.user_consent_version,
+            )
+
        defer.returnValue((200, return_dict))

    def on_OPTIONS(self, _):
--- a/Show More
+++ b/Show More