Merge pull request #3856 from matrix-org/erikj/speed_up_purge

Make purge history slightly faster
comment
2018-09-13 16:14:46 +01:00 · 2018-09-13 16:10:56 +01:00 · 2018-09-14 01:06:44 +10:00 · 2018-09-13 15:46:45 +01:00 · 2018-09-14 00:44:31 +10:00 · 2018-09-13 15:44:12 +01:00
133 changed files with 1812 additions and 1026 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -9,6 +9,8 @@ jobs:
      - store_artifacts:
          path: ~/project/logs
          destination: logs
+      - store_test_results:
+          path: logs
  sytestpy2postgres:
    machine: true
    steps:
@@ -18,6 +20,34 @@ jobs:
      - store_artifacts:
          path: ~/project/logs
          destination: logs
+      - store_test_results:
+          path: logs
+  sytestpy2merged:
+    machine: true
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy2
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs
+
+  sytestpy2postgresmerged:
+    machine: true
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy2
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs
+
  sytestpy3:
    machine: true
    steps:
@@ -27,6 +57,8 @@ jobs:
      - store_artifacts:
          path: ~/project/logs
          destination: logs
+      - store_test_results:
+          path: logs
  sytestpy3postgres:
    machine: true
    steps:
@@ -36,6 +68,32 @@ jobs:
      - store_artifacts:
          path: ~/project/logs
          destination: logs
+      - store_test_results:
+          path: logs
+  sytestpy3merged:
+    machine: true
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs hawkowl/sytestpy3
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs
+  sytestpy3postgresmerged:
+    machine: true
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs

 workflows:
  version: 2
@@ -43,6 +101,14 @@ workflows:
    jobs:
      - sytestpy2
      - sytestpy2postgres
+      - sytestpy2merged:
+          filters:
+            branches:
+              ignore: /develop|master/
+      - sytestpy2postgresmerged:
+          filters:
+            branches:
+              ignore: /develop|master/
 # Currently broken while the Python 3 port is incomplete
 #      - sytestpy3
 #      - sytestpy3postgres
--- a/.circleci/merge_base_branch.sh
+++ b/.circleci/merge_base_branch.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -e
+
+# CircleCI doesn't give CIRCLE_PR_NUMBER in the environment for non-forked PRs. Wonderful.
+# In this case, we just need to do some ~shell magic~ to strip it out of the PULL_REQUEST URL.
+echo 'export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}"' >> $BASH_ENV
+source $BASH_ENV
+
+if [[ -z "${CIRCLE_PR_NUMBER}" ]]
+then
+    echo "Can't figure out what the PR number is!"
+    exit 1
+fi
+
+# Get the reference, using the GitHub API
+GITBASE=`curl -q https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
+
+# Show what we are before
+git show -s
+
+# Set up username so it can do a merge
+git config --global user.email bot@matrix.org
+git config --global user.name "A robot"
+
+# Fetch and merge. If it doesn't work, it will raise due to set -e.
+git fetch -u origin $GITBASE
+git merge --no-edit origin/$GITBASE
+
+# Show what we are after.
+git show -s
--- a/.dockerignore
+++ b/.dockerignore
@@ -3,6 +3,5 @@ Dockerfile
 .gitignore
 demo/etc
 tox.ini
-synctl
 .git/*
 .tox/*
--- a/.gitignore
+++ b/.gitignore
@@ -44,6 +44,7 @@ media_store/
 build/
 venv/
 venv*/
+*venv/

 localhost-800*/
 static/client/register/register_config.js
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,9 +8,6 @@ before_script:
  - git remote set-branches --add origin develop
  - git fetch origin develop

-services:
-  - postgresql
-
 matrix:
  fast_finish: true
  include:
@@ -25,6 +22,8 @@ matrix:

  - python: 2.7
    env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
+    services:
+      - postgresql

  - python: 3.6
    env: TOX_ENV=py36
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,77 @@
+Synapse 0.33.4 (2018-09-07)
+===========================
+
+Internal Changes
+----------------
+
+- Unignore synctl in .dockerignore to fix docker builds ([\#3802](https://github.com/matrix-org/synapse/issues/3802))
+
+
+Synapse 0.33.4rc2 (2018-09-06)
+==============================
+
+Pull in security fixes from v0.33.3.1
+
+
+Synapse 0.33.3.1 (2018-09-06)
+=============================
+
+SECURITY FIXES
+--------------
+
+- Fix an issue where event signatures were not always correctly validated ([\#3796](https://github.com/matrix-org/synapse/issues/3796))
+- Fix an issue where server_acls could be circumvented for incoming events ([\#3796](https://github.com/matrix-org/synapse/issues/3796))
+
+
+Internal Changes
+----------------
+
+- Unignore synctl in .dockerignore to fix docker builds ([\#3802](https://github.com/matrix-org/synapse/issues/3802))
+
+
+Synapse 0.33.4rc1 (2018-09-04)
+==============================
+
+Features
+--------
+
+- Support profile API endpoints on workers ([\#3659](https://github.com/matrix-org/synapse/issues/3659))
+- Server notices for resource limit blocking ([\#3680](https://github.com/matrix-org/synapse/issues/3680))
+- Allow guests to use /rooms/:roomId/event/:eventId ([\#3724](https://github.com/matrix-org/synapse/issues/3724))
+- Add mau_trial_days config param, so that users only get counted as MAU after N days. ([\#3749](https://github.com/matrix-org/synapse/issues/3749))
+- Require twisted 17.1 or later (fixes [#3741](https://github.com/matrix-org/synapse/issues/3741)). ([\#3751](https://github.com/matrix-org/synapse/issues/3751))
+
+
+Bugfixes
+--------
+
+- Fix error collecting prometheus metrics when run on dedicated thread due to threading concurrency issues ([\#3722](https://github.com/matrix-org/synapse/issues/3722))
+- Fix bug where we resent "limit exceeded" server notices repeatedly ([\#3747](https://github.com/matrix-org/synapse/issues/3747))
+- Fix bug where we broke sync when using limit_usage_by_mau but hadn't configured server notices ([\#3753](https://github.com/matrix-org/synapse/issues/3753))
+- Fix 'federation_domain_whitelist' such that an empty list correctly blocks all outbound federation traffic ([\#3754](https://github.com/matrix-org/synapse/issues/3754))
+- Fix tagging of server notice rooms ([\#3755](https://github.com/matrix-org/synapse/issues/3755), [\#3756](https://github.com/matrix-org/synapse/issues/3756))
+- Fix 'admin_uri' config variable and error parameter to be 'admin_contact' to match the spec. ([\#3758](https://github.com/matrix-org/synapse/issues/3758))
+- Don't return non-LL-member state in incremental sync state blocks ([\#3760](https://github.com/matrix-org/synapse/issues/3760))
+- Fix bug in sending presence over federation ([\#3768](https://github.com/matrix-org/synapse/issues/3768))
+- Fix bug where preserved threepid user comes to sign up and server is mau blocked ([\#3777](https://github.com/matrix-org/synapse/issues/3777))
+
+Internal Changes
+----------------
+
+- Removed the link to the unmaintained matrix-synapse-auto-deploy project from the readme. ([\#3378](https://github.com/matrix-org/synapse/issues/3378))
+- Refactor state module to support multiple room versions ([\#3673](https://github.com/matrix-org/synapse/issues/3673))
+- The synapse.storage module has been ported to Python 3. ([\#3725](https://github.com/matrix-org/synapse/issues/3725))
+- Split the state_group_cache into member and non-member state events (and so speed up LL /sync) ([\#3726](https://github.com/matrix-org/synapse/issues/3726))
+- Log failure to authenticate remote servers as warnings (without stack traces) ([\#3727](https://github.com/matrix-org/synapse/issues/3727))
+- The CONTRIBUTING guidelines have been updated to mention our use of Markdown and that .misc files have content. ([\#3730](https://github.com/matrix-org/synapse/issues/3730))
+- Reference the need for an HTTP replication port when using the federation_reader worker ([\#3734](https://github.com/matrix-org/synapse/issues/3734))
+- Fix minor spelling error in federation client documentation. ([\#3735](https://github.com/matrix-org/synapse/issues/3735))
+- Remove redundant state resolution function ([\#3737](https://github.com/matrix-org/synapse/issues/3737))
+- The test suite now passes on PostgreSQL. ([\#3740](https://github.com/matrix-org/synapse/issues/3740))
+- Fix MAU cache invalidation due to missing yield ([\#3746](https://github.com/matrix-org/synapse/issues/3746))
+- Make sure that we close db connections opened during init ([\#3764](https://github.com/matrix-org/synapse/issues/3764))
+
+
 Synapse 0.33.3 (2018-08-22)
 ===========================

--- a/README.rst
+++ b/README.rst
@@ -742,6 +742,18 @@ so an example nginx configuration might look like::
      }
  }

+and an example apache configuration may look like::
+
+    <VirtualHost *:443>
+        SSLEngine on
+        ServerName matrix.example.com;
+
+        <Location /_matrix>
+            ProxyPass http://127.0.0.1:8008/_matrix nocanon
+            ProxyPassReverse http://127.0.0.1:8008/_matrix
+        </Location>
+    </VirtualHost>
+
 You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true``
 for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are
 recorded correctly.
--- a/changelog.d/3378.misc
+++ b/changelog.d/3378.misc
@@ -1 +0,0 @@
-Removed the link to the unmaintained matrix-synapse-auto-deploy project from the readme.
--- a/changelog.d/3659.feature
+++ b/changelog.d/3659.feature
@@ -1 +0,0 @@
-Support profile API endpoints on workers
--- a/changelog.d/3673.misc
+++ b/changelog.d/3673.misc
@@ -1 +0,0 @@
-Refactor state module to support multiple room versions
--- a/changelog.d/3680.feature
+++ b/changelog.d/3680.feature
@@ -1 +0,0 @@
-Server notices for resource limit blocking
--- a/changelog.d/3704.misc
+++ b/changelog.d/3704.misc
@@ -0,0 +1 @@
+CircleCI tests now run on the potential merge of a PR.
--- a/changelog.d/3722.bugfix
+++ b/changelog.d/3722.bugfix
@@ -1 +0,0 @@
-Fix error collecting prometheus metrics when run on dedicated thread due to threading concurrency issues
--- a/changelog.d/3724.feature
+++ b/changelog.d/3724.feature
@@ -1 +0,0 @@
-Allow guests to use /rooms/:roomId/event/:eventId
--- a/changelog.d/3725.misc
+++ b/changelog.d/3725.misc
@@ -1 +0,0 @@
-The synapse.storage module has been ported to Python 3.
--- a/changelog.d/3726.misc
+++ b/changelog.d/3726.misc
@@ -1 +0,0 @@
-Split the state_group_cache into member and non-member state events (and so speed up LL /sync)
--- a/changelog.d/3727.misc
+++ b/changelog.d/3727.misc
@@ -1 +0,0 @@
-Log failure to authenticate remote servers as warnings (without stack traces)
--- a/changelog.d/3730.misc
+++ b/changelog.d/3730.misc
@@ -1 +0,0 @@
-The CONTRIBUTING guidelines have been updated to mention our use of Markdown and that .misc files have content.
--- a/changelog.d/3734.misc
+++ b/changelog.d/3734.misc
@@ -1 +0,0 @@
-Reference the need for an HTTP replication port when using the federation_reader worker
--- a/changelog.d/3735.misc
+++ b/changelog.d/3735.misc
@@ -1 +0,0 @@
-Fix minor spelling error in federation client documentation.
--- a/changelog.d/3737.misc
+++ b/changelog.d/3737.misc
@@ -1 +0,0 @@
-Remove redundant state resolution function
--- a/changelog.d/3740.misc
+++ b/changelog.d/3740.misc
@@ -1 +0,0 @@
-The test suite now passes on PostgreSQL.
--- a/changelog.d/3746.misc
+++ b/changelog.d/3746.misc
@@ -1 +0,0 @@
-Fix MAU cache invalidation due to missing yield
--- a/changelog.d/3747.bugfix
+++ b/changelog.d/3747.bugfix
@@ -1 +0,0 @@
-Fix bug where we resent "limit exceeded" server notices repeatedly
--- a/changelog.d/3749.feature
+++ b/changelog.d/3749.feature
@@ -1 +0,0 @@
-Add mau_trial_days config param, so that users only get counted as MAU after N days.
--- a/changelog.d/3751.feature
+++ b/changelog.d/3751.feature
@@ -1 +0,0 @@
-Require twisted 17.1 or later (fixes [#3741](https://github.com/matrix-org/synapse/issues/3741)).
--- a/changelog.d/3753.bugfix
+++ b/changelog.d/3753.bugfix
@@ -1 +0,0 @@
-Fix bug where we broke sync when using limit_usage_by_mau but hadn't configured server notices
--- a/changelog.d/3754.bugfix
+++ b/changelog.d/3754.bugfix
@@ -1 +0,0 @@
-Fix 'federation_domain_whitelist' such that an empty list correctly blocks all outbound federation traffic
--- a/changelog.d/3755.bugfix
+++ b/changelog.d/3755.bugfix
@@ -1 +0,0 @@
-Fix tagging of server notice rooms
--- a/changelog.d/3756.bugfix
+++ b/changelog.d/3756.bugfix
@@ -1 +0,0 @@
-Fix tagging of server notice rooms
--- a/changelog.d/3758.bugfix
+++ b/changelog.d/3758.bugfix
@@ -1 +0,0 @@
-Fix 'admin_uri' config variable and error parameter to be 'admin_contact' to match the spec.
--- a/changelog.d/3760.bugfix
+++ b/changelog.d/3760.bugfix
@@ -1 +0,0 @@
-Don't return non-LL-member state in incremental sync state blocks
--- a/changelog.d/3764.misc
+++ b/changelog.d/3764.misc
@@ -1 +0,0 @@
-Make sure that we close db connections opened during init
--- a/changelog.d/3768.bugfix
+++ b/changelog.d/3768.bugfix
@@ -1 +0,0 @@
-Fix bug in sending presence over federation
--- a/changelog.d/3771.misc
+++ b/changelog.d/3771.misc
@@ -0,0 +1 @@
+http/ is now ported to Python 3.
--- a/changelog.d/3777.bugfix
+++ b/changelog.d/3777.bugfix
@@ -1 +0,0 @@
-Fix bug where preserved threepid user comes to sign up and server is mau blocked
--- a/changelog.d/3788.bugfix
+++ b/changelog.d/3788.bugfix
@@ -0,0 +1 @@
+Remove connection ID for replication prometheus metrics, as it creates a large number of new series.
--- a/changelog.d/3790.feature
+++ b/changelog.d/3790.feature
@@ -0,0 +1 @@
+Implement `event_format` filter param in `/sync`
--- a/changelog.d/3795.misc
+++ b/changelog.d/3795.misc
@@ -0,0 +1 @@
+Make /sync slightly faster by avoiding needless copies
--- a/changelog.d/3800.bugfix
+++ b/changelog.d/3800.bugfix
@@ -0,0 +1 @@
+guest users should not be part of mau total
--- a/changelog.d/3803.misc
+++ b/changelog.d/3803.misc
@@ -0,0 +1 @@
+handlers/ is now ported to Python 3.
--- a/changelog.d/3804.bugfix
+++ b/changelog.d/3804.bugfix
@@ -0,0 +1 @@
+Bump dependency on pyopenssl 16.x, to avoid incompatibility with recent Twisted.
--- a/changelog.d/3805.misc
+++ b/changelog.d/3805.misc
@@ -0,0 +1 @@
+Limit the number of PDUs/EDUs per federation transaction
--- a/changelog.d/3806.misc
+++ b/changelog.d/3806.misc
@@ -0,0 +1 @@
+Only start postgres instance for postgres tests on Travis CI
--- a/changelog.d/3808.misc
+++ b/changelog.d/3808.misc
@@ -0,0 +1 @@
+tests/ is now ported to Python 3.
--- a/changelog.d/3810.bugfix
+++ b/changelog.d/3810.bugfix
@@ -0,0 +1 @@
+Fix existing room tags not coming down sync when joining a room
--- a/changelog.d/3822.misc
+++ b/changelog.d/3822.misc
@@ -0,0 +1 @@
+crypto/ is now ported to Python 3.
--- a/changelog.d/3823.misc
+++ b/changelog.d/3823.misc
@@ -0,0 +1 @@
+rest/ is now ported to Python 3.
--- a/changelog.d/3824.bugfix
+++ b/changelog.d/3824.bugfix
@@ -0,0 +1 @@
+Fix jwt import check
--- a/changelog.d/3826.misc
+++ b/changelog.d/3826.misc
@@ -0,0 +1 @@
+add some logging for the keyring queue
--- a/changelog.d/3827.misc
+++ b/changelog.d/3827.misc
@@ -0,0 +1 @@
+speed up lazy loading by 2-3x
--- a/changelog.d/3834.misc
+++ b/changelog.d/3834.misc
@@ -0,0 +1 @@
+Improved Dockerfile to remove build requirements after building reducing the image size.
--- a/changelog.d/3835.bugfix
+++ b/changelog.d/3835.bugfix
@@ -0,0 +1 @@
+fix VOIP crashes under Python 3 (#3821)
--- a/changelog.d/3840.misc
+++ b/changelog.d/3840.misc
@@ -0,0 +1 @@
+Disable lazy loading for incremental syncs for now
--- a/changelog.d/3841.bugfix
+++ b/changelog.d/3841.bugfix
@@ -0,0 +1 @@
+Fix manhole so that it works with latest openssh clients
--- a/changelog.d/3845.bugfix
+++ b/changelog.d/3845.bugfix
@@ -0,0 +1 @@
+Fix outbound requests occasionally wedging, which can result in federation breaking between servers.
--- a/changelog.d/3846.feature
+++ b/changelog.d/3846.feature
@@ -0,0 +1 @@
+Add synapse_admin_mau:registered_reserved_users metric to expose number of real reaserved users 
--- a/changelog.d/3847.misc
+++ b/changelog.d/3847.misc
@@ -0,0 +1 @@
+federation/ is now ported to Python 3.
--- a/changelog.d/3851.bugfix
+++ b/changelog.d/3851.bugfix
@@ -0,0 +1 @@
+Show heroes if room name/canonical alias has been deleted
--- a/changelog.d/3853.misc
+++ b/changelog.d/3853.misc
@@ -0,0 +1 @@
+Log when we retry outbound requests
--- a/changelog.d/3855.misc
+++ b/changelog.d/3855.misc
@@ -0,0 +1 @@
+Removed some excess logging messages.
--- a/changelog.d/3856.misc
+++ b/changelog.d/3856.misc
@@ -0,0 +1 @@
+Speed up purge history for rooms that have been previously purged
--- a/changelog.d/3857.misc
+++ b/changelog.d/3857.misc
@@ -0,0 +1 @@
+Refactor some HTTP timeout code.
--- a/changelog.d/3858.misc
+++ b/changelog.d/3858.misc
@@ -0,0 +1 @@
+Fix running merged builds on CircleCI
--- a/changelog.d/3859.bugfix
+++ b/changelog.d/3859.bugfix
@@ -0,0 +1 @@
+Fix handling of redacted events from federation
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,6 +1,8 @@
 FROM docker.io/python:2-alpine3.8

-RUN apk add --no-cache --virtual .nacl_deps \
+COPY . /synapse
+
+RUN apk add --no-cache --virtual .build_deps \
        build-base \
        libffi-dev \
        libjpeg-turbo-dev \
@@ -8,13 +10,16 @@ RUN apk add --no-cache --virtual .nacl_deps \
        libxslt-dev \
        linux-headers \
        postgresql-dev \
-        su-exec \
-        zlib-dev
-
-COPY . /synapse
-
-# A wheel cache may be provided in ./cache for faster build
-RUN cd /synapse \
+        zlib-dev \
+ && cd /synapse \
+ && apk add --no-cache --virtual .runtime_deps \
+ 	libffi \
+        libjpeg-turbo \
+	libressl \
+	libxslt \
+	libpq \
+	zlib \
+	su-exec \
 && pip install --upgrade \
        lxml \
        pip \
@@ -26,8 +31,9 @@ RUN cd /synapse \
 && rm -rf \
        setup.cfg \
        setup.py \
-        synapse
-
+        synapse \
+ && apk del .build_deps
+ 
 VOLUME ["/data"]

 EXPOSE 8008/tcp 8448/tcp
--- a/setup.cfg
+++ b/setup.cfg
@@ -17,13 +17,14 @@ ignore =
 [pep8]
 max-line-length = 90
 #  W503 requires that binary operators be at the end, not start, of lines. Erik
-#  doesn't like it.  E203 is contrary to PEP8.
-ignore = W503,E203
+#  doesn't like it.  E203 is contrary to PEP8.  E731 is silly.
+ignore = W503,E203,E731

 [flake8]
 # note that flake8 inherits the "ignore" settings from "pep8" (because it uses
 # pep8 to do those checks), but not the "max-line-length" setting
 max-line-length = 90
+ignore=W503,E203,E731

 [isort]
 line_length = 89
--- a/synapse/init.py
+++ b/synapse/init.py
@@ -17,4 +17,14 @@
 """ This is a reference implementation of a Matrix home server.
 """

-__version__ = "0.33.3"
+try:
+    from twisted.internet import protocol
+    from twisted.internet.protocol import Factory
+    from twisted.names.dns import DNSDatagramProtocol
+    protocol.Factory.noisy = False
+    Factory.noisy = False
+    DNSDatagramProtocol.noisy = False
+except ImportError:
+    pass
+
+__version__ = "0.33.4"
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -251,6 +251,7 @@ class FilterCollection(object):
            "include_leave", False
        )
        self.event_fields = filter_json.get("event_fields", [])
+        self.event_format = filter_json.get("event_format", "client")

    def __repr__(self):
        return "<FilterCollection %s>" % (json.dumps(self._filter_json),)
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -307,6 +307,10 @@ class SynapseHomeServer(HomeServer):
 # Gauges to expose monthly active user control metrics
 current_mau_gauge = Gauge("synapse_admin_mau:current", "Current MAU")
 max_mau_gauge = Gauge("synapse_admin_mau:max", "MAU Limit")
+registered_reserved_users_mau_gauge = Gauge(
+    "synapse_admin_mau:registered_reserved_users",
+    "Registered users with reserved threepids"
+)


 def setup(config_options):
@@ -531,10 +535,14 @@ def run(hs):

    @defer.inlineCallbacks
    def generate_monthly_active_users():
-        count = 0
+        current_mau_count = 0
+        reserved_count = 0
+        store = hs.get_datastore()
        if hs.config.limit_usage_by_mau:
-            count = yield hs.get_datastore().get_monthly_active_count()
-        current_mau_gauge.set(float(count))
+            current_mau_count = yield store.get_monthly_active_count()
+            reserved_count = yield store.get_registered_reserved_users_count()
+        current_mau_gauge.set(float(current_mau_count))
+        registered_reserved_users_mau_gauge.set(float(reserved_count))
        max_mau_gauge.set(float(hs.config.max_mau_value))

    hs.get_datastore().initialise_reserved_users(
--- a/synapse/appservice/api.py
+++ b/synapse/appservice/api.py
@@ -13,7 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import urllib
+
+from six.moves import urllib

 from prometheus_client import Counter

@@ -98,7 +99,7 @@ class ApplicationServiceApi(SimpleHttpClient):
    def query_user(self, service, user_id):
        if service.url is None:
            defer.returnValue(False)
-        uri = service.url + ("/users/%s" % urllib.quote(user_id))
+        uri = service.url + ("/users/%s" % urllib.parse.quote(user_id))
        response = None
        try:
            response = yield self.get_json(uri, {
@@ -119,7 +120,7 @@ class ApplicationServiceApi(SimpleHttpClient):
    def query_alias(self, service, alias):
        if service.url is None:
            defer.returnValue(False)
-        uri = service.url + ("/rooms/%s" % urllib.quote(alias))
+        uri = service.url + ("/rooms/%s" % urllib.parse.quote(alias))
        response = None
        try:
            response = yield self.get_json(uri, {
@@ -153,7 +154,7 @@ class ApplicationServiceApi(SimpleHttpClient):
            service.url,
            APP_SERVICE_PREFIX,
            kind,
-            urllib.quote(protocol)
+            urllib.parse.quote(protocol)
        )
        try:
            response = yield self.get_json(uri, fields)
@@ -188,7 +189,7 @@ class ApplicationServiceApi(SimpleHttpClient):
            uri = "%s%s/thirdparty/protocol/%s" % (
                service.url,
                APP_SERVICE_PREFIX,
-                urllib.quote(protocol)
+                urllib.parse.quote(protocol)
            )
            try:
                info = yield self.get_json(uri, {})
@@ -228,7 +229,7 @@ class ApplicationServiceApi(SimpleHttpClient):
        txn_id = str(txn_id)

        uri = service.url + ("/transactions/%s" %
-                             urllib.quote(txn_id))
+                             urllib.parse.quote(txn_id))
        try:
            yield self.put_json(
                uri=uri,
--- a/synapse/config/homeserver.py
+++ b/synapse/config/homeserver.py
@@ -21,7 +21,7 @@ from .consent_config import ConsentConfig
 from .database import DatabaseConfig
 from .emailconfig import EmailConfig
 from .groups import GroupsConfig
-from .jwt import JWTConfig
+from .jwt_config import JWTConfig
 from .key import KeyConfig
 from .logger import LoggingConfig
 from .metrics import MetricsConfig
--- a/synapse/config/jwt_config.py
+++ b/synapse/config/jwt_config.py
--- a/synapse/config/logger.py
+++ b/synapse/config/logger.py
@@ -227,7 +227,22 @@ def setup_logging(config, use_worker_options=False):
    #
    # However this may not be too much of a problem if we are just writing to a file.
    observer = STDLibLogObserver()
+
+    def _log(event):
+
+        if "log_text" in event:
+            if event["log_text"].startswith("DNSDatagramProtocol starting on "):
+                return
+
+            if event["log_text"].startswith("(UDP Port "):
+                return
+
+            if event["log_text"].startswith("Timing out client"):
+                return
+
+        return observer(event)
+
    globalLogBeginner.beginLoggingTo(
-        [observer],
+        [_log],
        redirectStandardIO=not config.no_redirect_stdio,
    )
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -123,6 +123,6 @@ class ClientTLSOptionsFactory(object):

    def get_options(self, host):
        return ClientTLSOptions(
-            host.decode('utf-8'),
+            host,
            CertificateOptions(verify=False).getContext()
        )
--- a/synapse/crypto/keyclient.py
+++ b/synapse/crypto/keyclient.py
@@ -50,7 +50,7 @@ def fetch_server_key(server_name, tls_client_options_factory, path=KEY_API_V1):
                defer.returnValue((server_response, server_certificate))
        except SynapseKeyClientError as e:
            logger.warn("Error getting key for %r: %s", server_name, e)
-            if e.status.startswith("4"):
+            if e.status.startswith(b"4"):
                # Don't retry for 4xx responses.
                raise IOError("Cannot get key for %r" % server_name)
        except (ConnectError, DomainError) as e:
@@ -82,6 +82,12 @@ class SynapseKeyClientProtocol(HTTPClient):
        self._peer = self.transport.getPeer()
        logger.debug("Connected to %s", self._peer)

+        if not isinstance(self.path, bytes):
+            self.path = self.path.encode('ascii')
+
+        if not isinstance(self.host, bytes):
+            self.host = self.host.encode('ascii')
+
        self.sendCommand(b"GET", self.path)
        if self.host:
            self.sendHeader(b"Host", self.host)
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -16,9 +16,10 @@

 import hashlib
 import logging
-import urllib
 from collections import namedtuple

+from six.moves import urllib
+
 from signedjson.key import (
    decode_verify_key_bytes,
    encode_verify_key_base64,
@@ -40,6 +41,7 @@ from synapse.api.errors import Codes, SynapseError
 from synapse.crypto.keyclient import fetch_server_key
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.logcontext import (
+    LoggingContext,
    PreserveLoggingContext,
    preserve_fn,
    run_in_background,
@@ -216,23 +218,34 @@ class Keyring(object):
            servers have completed. Follows the synapse rules of logcontext
            preservation.
        """
+        loop_count = 1
        while True:
            wait_on = [
-                self.key_downloads[server_name]
+                (server_name, self.key_downloads[server_name])
                for server_name in server_names
                if server_name in self.key_downloads
            ]
-            if wait_on:
-                with PreserveLoggingContext():
-                    yield defer.DeferredList(wait_on)
-            else:
+            if not wait_on:
                break
+            logger.info(
+                "Waiting for existing lookups for %s to complete [loop %i]",
+                [w[0] for w in wait_on], loop_count,
+            )
+            with PreserveLoggingContext():
+                yield defer.DeferredList((w[1] for w in wait_on))
+
+            loop_count += 1
+
+        ctx = LoggingContext.current_context()

        def rm(r, server_name_):
-            self.key_downloads.pop(server_name_, None)
+            with PreserveLoggingContext(ctx):
+                logger.debug("Releasing key lookup lock on %s", server_name_)
+                self.key_downloads.pop(server_name_, None)
            return r

        for server_name, deferred in server_to_deferred.items():
+            logger.debug("Got key lookup lock on %s", server_name)
            self.key_downloads[server_name] = deferred
            deferred.addBoth(rm, server_name)

@@ -432,7 +445,7 @@ class Keyring(object):
        # an incoming request.
        query_response = yield self.client.post_json(
            destination=perspective_name,
-            path=b"/_matrix/key/v2/query",
+            path="/_matrix/key/v2/query",
            data={
                u"server_keys": {
                    server_name: {
@@ -513,8 +526,8 @@ class Keyring(object):

            (response, tls_certificate) = yield fetch_server_key(
                server_name, self.hs.tls_client_options_factory,
-                path=(b"/_matrix/key/v2/server/%s" % (
-                    urllib.quote(requested_key_id),
+                path=("/_matrix/key/v2/server/%s" % (
+                    urllib.parse.quote(requested_key_id),
                )).encode("ascii"),
            )

--- a/synapse/events/init.py
+++ b/synapse/events/init.py
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+import six
+
 from synapse.util.caches import intern_dict
 from synapse.util.frozenutils import freeze

@@ -147,6 +149,9 @@ class EventBase(object):
    def items(self):
        return list(self._event_dict.items())

+    def keys(self):
+        return six.iterkeys(self._event_dict)
+

 class FrozenEvent(EventBase):
    def __init__(self, event_dict, internal_metadata_dict={}, rejected_reason=None):
--- a/synapse/federation/federation_base.py
+++ b/synapse/federation/federation_base.py
@@ -13,17 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
+from collections import namedtuple

 import six

 from twisted.internet import defer
+from twisted.internet.defer import DeferredList

-from synapse.api.constants import MAX_DEPTH
+from synapse.api.constants import MAX_DEPTH, EventTypes, Membership
 from synapse.api.errors import Codes, SynapseError
 from synapse.crypto.event_signing import check_event_content_hash
 from synapse.events import FrozenEvent
 from synapse.events.utils import prune_event
 from synapse.http.servlet import assert_params_in_dict
+from synapse.types import get_domain_from_id
 from synapse.util import logcontext, unwrapFirstError

 logger = logging.getLogger(__name__)
@@ -133,34 +136,45 @@ class FederationBase(object):
              * throws a SynapseError if the signature check failed.
            The deferreds run their callbacks in the sentinel logcontext.
        """
-
-        redacted_pdus = [
-            prune_event(pdu)
-            for pdu in pdus
-        ]
-
-        deferreds = self.keyring.verify_json_objects_for_server([
-            (p.origin, p.get_pdu_json())
-            for p in redacted_pdus
-        ])
+        deferreds = _check_sigs_on_pdus(self.keyring, pdus)

        ctx = logcontext.LoggingContext.current_context()

-        def callback(_, pdu, redacted):
+        def callback(_, pdu):
            with logcontext.PreserveLoggingContext(ctx):
                if not check_event_content_hash(pdu):
-                    logger.warn(
-                        "Event content has been tampered, redacting %s: %s",
-                        pdu.event_id, pdu.get_pdu_json()
-                    )
-                    return redacted
+                    # let's try to distinguish between failures because the event was
+                    # redacted (which are somewhat expected) vs actual ball-tampering
+                    # incidents.
+                    #
+                    # This is just a heuristic, so we just assume that if the keys are
+                    # about the same between the redacted and received events, then the
+                    # received event was probably a redacted copy (but we then use our
+                    # *actual* redacted copy to be on the safe side.)
+                    redacted_event = prune_event(pdu)
+                    if (
+                        set(redacted_event.keys()) == set(pdu.keys()) and
+                        set(six.iterkeys(redacted_event.content))
+                            == set(six.iterkeys(pdu.content))
+                    ):
+                        logger.info(
+                            "Event %s seems to have been redacted; using our redacted "
+                            "copy",
+                            pdu.event_id,
+                        )
+                    else:
+                        logger.warning(
+                            "Event %s content has been tampered, redacting",
+                            pdu.event_id, pdu.get_pdu_json(),
+                        )
+                    return redacted_event

                if self.spam_checker.check_event_for_spam(pdu):
                    logger.warn(
                        "Event contains spam, redacting %s: %s",
                        pdu.event_id, pdu.get_pdu_json()
                    )
-                    return redacted
+                    return prune_event(pdu)

                return pdu

@@ -168,21 +182,121 @@ class FederationBase(object):
            failure.trap(SynapseError)
            with logcontext.PreserveLoggingContext(ctx):
                logger.warn(
-                    "Signature check failed for %s",
-                    pdu.event_id,
+                    "Signature check failed for %s: %s",
+                    pdu.event_id, failure.getErrorMessage(),
                )
            return failure

-        for deferred, pdu, redacted in zip(deferreds, pdus, redacted_pdus):
+        for deferred, pdu in zip(deferreds, pdus):
            deferred.addCallbacks(
                callback, errback,
-                callbackArgs=[pdu, redacted],
+                callbackArgs=[pdu],
                errbackArgs=[pdu],
            )

        return deferreds


+class PduToCheckSig(namedtuple("PduToCheckSig", [
+    "pdu", "redacted_pdu_json", "event_id_domain", "sender_domain", "deferreds",
+])):
+    pass
+
+
+def _check_sigs_on_pdus(keyring, pdus):
+    """Check that the given events are correctly signed
+
+    Args:
+        keyring (synapse.crypto.Keyring): keyring object to do the checks
+        pdus (Collection[EventBase]): the events to be checked
+
+    Returns:
+        List[Deferred]: a Deferred for each event in pdus, which will either succeed if
+           the signatures are valid, or fail (with a SynapseError) if not.
+    """
+
+    # (currently this is written assuming the v1 room structure; we'll probably want a
+    # separate function for checking v2 rooms)
+
+    # we want to check that the event is signed by:
+    #
+    # (a) the server which created the event_id
+    #
+    # (b) the sender's server.
+    #
+    #     - except in the case of invites created from a 3pid invite, which are exempt
+    #     from this check, because the sender has to match that of the original 3pid
+    #     invite, but the event may come from a different HS, for reasons that I don't
+    #     entirely grok (why do the senders have to match? and if they do, why doesn't the
+    #     joining server ask the inviting server to do the switcheroo with
+    #     exchange_third_party_invite?).
+    #
+    #     That's pretty awful, since redacting such an invite will render it invalid
+    #     (because it will then look like a regular invite without a valid signature),
+    #     and signatures are *supposed* to be valid whether or not an event has been
+    #     redacted. But this isn't the worst of the ways that 3pid invites are broken.
+    #
+    # let's start by getting the domain for each pdu, and flattening the event back
+    # to JSON.
+    pdus_to_check = [
+        PduToCheckSig(
+            pdu=p,
+            redacted_pdu_json=prune_event(p).get_pdu_json(),
+            event_id_domain=get_domain_from_id(p.event_id),
+            sender_domain=get_domain_from_id(p.sender),
+            deferreds=[],
+        )
+        for p in pdus
+    ]
+
+    # first make sure that the event is signed by the event_id's domain
+    deferreds = keyring.verify_json_objects_for_server([
+        (p.event_id_domain, p.redacted_pdu_json)
+        for p in pdus_to_check
+    ])
+
+    for p, d in zip(pdus_to_check, deferreds):
+        p.deferreds.append(d)
+
+    # now let's look for events where the sender's domain is different to the
+    # event id's domain (normally only the case for joins/leaves), and add additional
+    # checks.
+    pdus_to_check_sender = [
+        p for p in pdus_to_check
+        if p.sender_domain != p.event_id_domain and not _is_invite_via_3pid(p.pdu)
+    ]
+
+    more_deferreds = keyring.verify_json_objects_for_server([
+        (p.sender_domain, p.redacted_pdu_json)
+        for p in pdus_to_check_sender
+    ])
+
+    for p, d in zip(pdus_to_check_sender, more_deferreds):
+        p.deferreds.append(d)
+
+    # replace lists of deferreds with single Deferreds
+    return [_flatten_deferred_list(p.deferreds) for p in pdus_to_check]
+
+
+def _flatten_deferred_list(deferreds):
+    """Given a list of one or more deferreds, either return the single deferred, or
+    combine into a DeferredList.
+    """
+    if len(deferreds) > 1:
+        return DeferredList(deferreds, fireOnOneErrback=True, consumeErrors=True)
+    else:
+        assert len(deferreds) == 1
+        return deferreds[0]
+
+
+def _is_invite_via_3pid(event):
+    return (
+        event.type == EventTypes.Member
+        and event.membership == Membership.INVITE
+        and "third_party_invite" in event.content
+    )
+
+
 def event_from_pdu_json(pdu_json, outlier=False):
    """Construct a FrozenEvent from an event json received over federation

--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -271,10 +271,10 @@ class FederationClient(FederationBase):
                    event_id, destination, e,
                )
            except NotRetryingDestination as e:
-                logger.info(e.message)
+                logger.info(str(e))
                continue
            except FederationDeniedError as e:
-                logger.info(e.message)
+                logger.info(str(e))
                continue
            except Exception as e:
                pdu_attempts[destination] = now
@@ -510,7 +510,7 @@ class FederationClient(FederationBase):
                else:
                    logger.warn(
                        "Failed to %s via %s: %i %s",
-                        description, destination, e.code, e.message,
+                        description, destination, e.code, e.args[0],
                    )
            except Exception:
                logger.warn(
@@ -875,7 +875,7 @@ class FederationClient(FederationBase):
            except Exception as e:
                logger.exception(
                    "Failed to send_third_party_invite via %s: %s",
-                    destination, e.message
+                    destination, str(e)
                )

        raise RuntimeError("Failed to send to any server.")
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -99,7 +99,7 @@ class FederationServer(FederationBase):

    @defer.inlineCallbacks
    @log_function
-    def on_incoming_transaction(self, transaction_data):
+    def on_incoming_transaction(self, origin, transaction_data):
        # keep this as early as possible to make the calculated origin ts as
        # accurate as possible.
        request_time = self._clock.time_msec()
@@ -108,34 +108,33 @@ class FederationServer(FederationBase):

        if not transaction.transaction_id:
            raise Exception("Transaction missing transaction_id")
-        if not transaction.origin:
-            raise Exception("Transaction missing origin")

        logger.debug("[%s] Got transaction", transaction.transaction_id)

        # use a linearizer to ensure that we don't process the same transaction
        # multiple times in parallel.
        with (yield self._transaction_linearizer.queue(
-                (transaction.origin, transaction.transaction_id),
+                (origin, transaction.transaction_id),
        )):
            result = yield self._handle_incoming_transaction(
-                transaction, request_time,
+                origin, transaction, request_time,
            )

        defer.returnValue(result)

    @defer.inlineCallbacks
-    def _handle_incoming_transaction(self, transaction, request_time):
+    def _handle_incoming_transaction(self, origin, transaction, request_time):
        """ Process an incoming transaction and return the HTTP response

        Args:
+            origin (unicode): the server making the request
            transaction (Transaction): incoming transaction
            request_time (int): timestamp that the HTTP request arrived at

        Returns:
            Deferred[(int, object)]: http response code and body
        """
-        response = yield self.transaction_actions.have_responded(transaction)
+        response = yield self.transaction_actions.have_responded(origin, transaction)

        if response:
            logger.debug(
@@ -149,7 +148,7 @@ class FederationServer(FederationBase):

        received_pdus_counter.inc(len(transaction.pdus))

-        origin_host, _ = parse_server_name(transaction.origin)
+        origin_host, _ = parse_server_name(origin)

        pdus_by_room = {}

@@ -190,7 +189,7 @@ class FederationServer(FederationBase):
                event_id = pdu.event_id
                try:
                    yield self._handle_received_pdu(
-                        transaction.origin, pdu
+                        origin, pdu
                    )
                    pdu_results[event_id] = {}
                except FederationError as e:
@@ -212,7 +211,7 @@ class FederationServer(FederationBase):
        if hasattr(transaction, "edus"):
            for edu in (Edu(**x) for x in transaction.edus):
                yield self.received_edu(
-                    transaction.origin,
+                    origin,
                    edu.edu_type,
                    edu.content
                )
@@ -224,6 +223,7 @@ class FederationServer(FederationBase):
        logger.debug("Returning: %s", str(response))

        yield self.transaction_actions.set_response(
+            origin,
            transaction,
            200, response
        )
@@ -838,9 +838,9 @@ class ReplicationFederationHandlerRegistry(FederationHandlerRegistry):
            )

        return self._send_edu(
-                edu_type=edu_type,
-                origin=origin,
-                content=content,
+            edu_type=edu_type,
+            origin=origin,
+            content=content,
        )

    def on_query(self, query_type, args):
@@ -851,6 +851,6 @@ class ReplicationFederationHandlerRegistry(FederationHandlerRegistry):
            return handler(args)

        return self._get_query_client(
-                query_type=query_type,
-                args=args,
+            query_type=query_type,
+            args=args,
        )
--- a/synapse/federation/persistence.py
+++ b/synapse/federation/persistence.py
@@ -36,7 +36,7 @@ class TransactionActions(object):
        self.store = datastore

    @log_function
-    def have_responded(self, transaction):
+    def have_responded(self, origin, transaction):
        """ Have we already responded to a transaction with the same id and
        origin?

@@ -50,11 +50,11 @@ class TransactionActions(object):
                               "transaction_id")

        return self.store.get_received_txn_response(
-            transaction.transaction_id, transaction.origin
+            transaction.transaction_id, origin
        )

    @log_function
-    def set_response(self, transaction, code, response):
+    def set_response(self, origin, transaction, code, response):
        """ Persist how we responded to a transaction.

        Returns:
@@ -66,7 +66,7 @@ class TransactionActions(object):

        return self.store.set_received_txn_response(
            transaction.transaction_id,
-            transaction.origin,
+            origin,
            code,
            response,
        )
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -463,7 +463,19 @@ class TransactionQueue(object):
                # pending_transactions flag.

                pending_pdus = self.pending_pdus_by_dest.pop(destination, [])
+
+                # We can only include at most 50 PDUs per transactions
+                pending_pdus, leftover_pdus = pending_pdus[:50], pending_pdus[50:]
+                if leftover_pdus:
+                    self.pending_pdus_by_dest[destination] = leftover_pdus
+
                pending_edus = self.pending_edus_by_dest.pop(destination, [])
+
+                # We can only include at most 100 EDUs per transactions
+                pending_edus, leftover_edus = pending_edus[:100], pending_edus[100:]
+                if leftover_edus:
+                    self.pending_edus_by_dest[destination] = leftover_edus
+
                pending_presence = self.pending_presence_by_dest.pop(destination, {})

                pending_edus.extend(
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -15,7 +15,8 @@
 # limitations under the License.

 import logging
-import urllib
+
+from six.moves import urllib

 from twisted.internet import defer

@@ -951,4 +952,4 @@ def _create_path(prefix, path, *args):
    Returns:
        str
    """
-    return prefix + path % tuple(urllib.quote(arg, "") for arg in args)
+    return prefix + path % tuple(urllib.parse.quote(arg, "") for arg in args)
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -90,8 +90,8 @@ class Authenticator(object):
    @defer.inlineCallbacks
    def authenticate_request(self, request, content):
        json_request = {
-            "method": request.method,
-            "uri": request.uri,
+            "method": request.method.decode('ascii'),
+            "uri": request.uri.decode('ascii'),
            "destination": self.server_name,
            "signatures": {},
        }
@@ -252,7 +252,7 @@ class BaseFederationServlet(object):
                    by the callback method. None if the request has already been handled.
            """
            content = None
-            if request.method in ["PUT", "POST"]:
+            if request.method in [b"PUT", b"POST"]:
                # TODO: Handle other method types? other content types?
                content = parse_json_object_from_request(request)

@@ -353,7 +353,7 @@ class FederationSendServlet(BaseFederationServlet):

        try:
            code, response = yield self.handler.on_incoming_transaction(
-                transaction_data
+                origin, transaction_data,
            )
        except Exception:
            logger.exception("on_incoming_transaction failed")
@@ -386,7 +386,7 @@ class FederationStateServlet(BaseFederationServlet):
        return self.handler.on_context_state_request(
            origin,
            context,
-            query.get("event_id", [None])[0],
+            parse_string_from_args(query, "event_id", None),
        )


@@ -397,7 +397,7 @@ class FederationStateIdsServlet(BaseFederationServlet):
        return self.handler.on_state_ids_request(
            origin,
            room_id,
-            query.get("event_id", [None])[0],
+            parse_string_from_args(query, "event_id", None),
        )


@@ -405,14 +405,12 @@ class FederationBackfillServlet(BaseFederationServlet):
    PATH = "/backfill/(?P<context>[^/]*)/"

    def on_GET(self, origin, content, query, context):
-        versions = query["v"]
-        limits = query["limit"]
+        versions = [x.decode('ascii') for x in query[b"v"]]
+        limit = parse_integer_from_args(query, "limit", None)

-        if not limits:
+        if not limit:
            return defer.succeed((400, {"error": "Did not include limit param"}))

-        limit = int(limits[-1])
-
        return self.handler.on_backfill_request(origin, context, versions, limit)


@@ -423,7 +421,7 @@ class FederationQueryServlet(BaseFederationServlet):
    def on_GET(self, origin, content, query, query_type):
        return self.handler.on_query_request(
            query_type,
-            {k: v[0].decode("utf-8") for k, v in query.items()}
+            {k.decode('utf8'): v[0].decode("utf-8") for k, v in query.items()}
        )


@@ -630,14 +628,14 @@ class OpenIdUserInfo(BaseFederationServlet):

    @defer.inlineCallbacks
    def on_GET(self, origin, content, query):
-        token = query.get("access_token", [None])[0]
+        token = query.get(b"access_token", [None])[0]
        if token is None:
            defer.returnValue((401, {
                "errcode": "M_MISSING_TOKEN", "error": "Access Token required"
            }))
            return

-        user_id = yield self.handler.on_openid_userinfo(token)
+        user_id = yield self.handler.on_openid_userinfo(token.decode('ascii'))

        if user_id is None:
            defer.returnValue((401, {
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -895,22 +895,24 @@ class AuthHandler(BaseHandler):

        Args:
            password (unicode): Password to hash.
-            stored_hash (unicode): Expected hash value.
+            stored_hash (bytes): Expected hash value.

        Returns:
            Deferred(bool): Whether self.hash(password) == stored_hash.
        """
-
        def _do_validate_hash():
            # Normalise the Unicode in the password
            pw = unicodedata.normalize("NFKC", password)

            return bcrypt.checkpw(
                pw.encode('utf8') + self.hs.config.password_pepper.encode("utf8"),
-                stored_hash.encode('utf8')
+                stored_hash
            )

        if stored_hash:
+            if not isinstance(stored_hash, bytes):
+                stored_hash = stored_hash.encode('ascii')
+
            return make_deferred_yieldable(
                threads.deferToThreadPool(
                    self.hs.get_reactor(),
--- a/synapse/handlers/e2e_keys.py
+++ b/synapse/handlers/e2e_keys.py
@@ -330,7 +330,8 @@ class E2eKeysHandler(object):
                        (algorithm, key_id, ex_json, key)
                    )
            else:
-                new_keys.append((algorithm, key_id, encode_canonical_json(key)))
+                new_keys.append((
+                    algorithm, key_id, encode_canonical_json(key).decode('ascii')))

        yield self.store.add_e2e_one_time_keys(
            user_id, device_id, time_now, new_keys
@@ -358,7 +359,7 @@ def _exception_to_failure(e):
    # Note that some Exceptions (notably twisted's ResponseFailed etc) don't
    # give a string for e.message, which json then fails to serialize.
    return {
-        "status": 503, "message": str(e.message),
+        "status": 503, "message": str(e),
    }


--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -594,7 +594,7 @@ class FederationHandler(BaseHandler):

        required_auth = set(
            a_id
-            for event in events + state_events.values() + auth_events.values()
+            for event in events + list(state_events.values()) + list(auth_events.values())
            for a_id, _ in event.auth_events
        )
        auth_events.update({
@@ -802,7 +802,7 @@ class FederationHandler(BaseHandler):
                    )
                    continue
                except NotRetryingDestination as e:
-                    logger.info(e.message)
+                    logger.info(str(e))
                    continue
                except FederationDeniedError as e:
                    logger.info(e)
@@ -1358,7 +1358,7 @@ class FederationHandler(BaseHandler):
        )

        if state_groups:
-            _, state = state_groups.items().pop()
+            _, state = list(state_groups.items()).pop()
            results = state

            if event.is_state():
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -162,7 +162,7 @@ class RoomListHandler(BaseHandler):
        # Filter out rooms that we don't want to return
        rooms_to_scan = [
            r for r in sorted_rooms
-            if r not in newly_unpublished and rooms_to_num_joined[room_id] > 0
+            if r not in newly_unpublished and rooms_to_num_joined[r] > 0
        ]

        total_room_count = len(rooms_to_scan)
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -54,7 +54,7 @@ class SearchHandler(BaseHandler):
        batch_token = None
        if batch:
            try:
-                b = decode_base64(batch)
+                b = decode_base64(batch).decode('ascii')
                batch_group, batch_group_key, batch_token = b.split("\n")

                assert batch_group is not None
@@ -258,18 +258,18 @@ class SearchHandler(BaseHandler):
                # it returns more from the same group (if applicable) rather
                # than reverting to searching all results again.
                if batch_group and batch_group_key:
-                    global_next_batch = encode_base64("%s\n%s\n%s" % (
+                    global_next_batch = encode_base64(("%s\n%s\n%s" % (
                        batch_group, batch_group_key, pagination_token
-                    ))
+                    )).encode('ascii'))
                else:
-                    global_next_batch = encode_base64("%s\n%s\n%s" % (
+                    global_next_batch = encode_base64(("%s\n%s\n%s" % (
                        "all", "", pagination_token
-                    ))
+                    )).encode('ascii'))

                for room_id, group in room_groups.items():
-                    group["next_batch"] = encode_base64("%s\n%s\n%s" % (
+                    group["next_batch"] = encode_base64(("%s\n%s\n%s" % (
                        "room_id", room_id, pagination_token
-                    ))
+                    )).encode('ascii'))

            allowed_events.extend(room_events)

--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -24,6 +24,7 @@ from twisted.internet import defer

 from synapse.api.constants import EventTypes, Membership
 from synapse.push.clientformat import format_push_rules_for_user
+from synapse.storage.roommember import MemberSummary
 from synapse.types import RoomStreamToken
 from synapse.util.async_helpers import concurrently_execute
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -525,6 +526,8 @@ class SyncHandler(object):
             A deferred dict describing the room summary
        """

+        # FIXME: we could/should get this from room_stats when matthew/stats lands
+
        # FIXME: this promulgates https://github.com/matrix-org/synapse/issues/3305
        last_events, _ = yield self.store.get_recent_event_ids_for_room(
            room_id, end_token=now_token.room_key, limit=1,
@@ -537,44 +540,67 @@ class SyncHandler(object):
        last_event = last_events[-1]
        state_ids = yield self.store.get_state_ids_for_event(
            last_event.event_id, [
-                (EventTypes.Member, None),
                (EventTypes.Name, ''),
                (EventTypes.CanonicalAlias, ''),
            ]
        )

-        member_ids = {
-            state_key: event_id
-            for (t, state_key), event_id in state_ids.iteritems()
-            if t == EventTypes.Member
-        }
+        # this is heavily cached, thus: fast.
+        details = yield self.store.get_room_summary(room_id)
+
        name_id = state_ids.get((EventTypes.Name, ''))
        canonical_alias_id = state_ids.get((EventTypes.CanonicalAlias, ''))

        summary = {}
-
-        # FIXME: it feels very heavy to load up every single membership event
-        # just to calculate the counts.
-        member_events = yield self.store.get_events(member_ids.values())
-
-        joined_user_ids = []
-        invited_user_ids = []
-
-        for ev in member_events.values():
-            if ev.content.get("membership") == Membership.JOIN:
-                joined_user_ids.append(ev.state_key)
-            elif ev.content.get("membership") == Membership.INVITE:
-                invited_user_ids.append(ev.state_key)
+        empty_ms = MemberSummary([], 0)

        # TODO: only send these when they change.
-        summary["m.joined_member_count"] = len(joined_user_ids)
-        summary["m.invited_member_count"] = len(invited_user_ids)
+        summary["m.joined_member_count"] = (
+            details.get(Membership.JOIN, empty_ms).count
+        )
+        summary["m.invited_member_count"] = (
+            details.get(Membership.INVITE, empty_ms).count
+        )

-        if name_id or canonical_alias_id:
-            defer.returnValue(summary)
+        # if the room has a name or canonical_alias set, we can skip
+        # calculating heroes.  we assume that if the event has contents, it'll
+        # be a valid name or canonical_alias - i.e. we're checking that they
+        # haven't been "deleted" by blatting {} over the top.
+        if name_id:
+            name = yield self.store.get_event(name_id, allow_none=False)
+            if name and name.content:
+                defer.returnValue(summary)

-        # FIXME: order by stream ordering, not alphabetic
+        if canonical_alias_id:
+            canonical_alias = yield self.store.get_event(
+                canonical_alias_id, allow_none=False,
+            )
+            if canonical_alias and canonical_alias.content:
+                defer.returnValue(summary)

+        joined_user_ids = [
+            r[0] for r in details.get(Membership.JOIN, empty_ms).members
+        ]
+        invited_user_ids = [
+            r[0] for r in details.get(Membership.INVITE, empty_ms).members
+        ]
+        gone_user_ids = (
+            [r[0] for r in details.get(Membership.LEAVE, empty_ms).members] +
+            [r[0] for r in details.get(Membership.BAN, empty_ms).members]
+        )
+
+        # FIXME: only build up a member_ids list for our heroes
+        member_ids = {}
+        for membership in (
+            Membership.JOIN,
+            Membership.INVITE,
+            Membership.LEAVE,
+            Membership.BAN
+        ):
+            for user_id, event_id in details.get(membership, empty_ms).members:
+                member_ids[user_id] = event_id
+
+        # FIXME: order by stream ordering rather than as returned by SQL
        me = sync_config.user.to_string()
        if (joined_user_ids or invited_user_ids):
            summary['m.heroes'] = sorted(
@@ -586,7 +612,11 @@ class SyncHandler(object):
            )[0:5]
        else:
            summary['m.heroes'] = sorted(
-                [user_id for user_id in member_ids.keys() if user_id != me]
+                [
+                    user_id
+                    for user_id in gone_user_ids
+                    if user_id != me
+                ]
            )[0:5]

        if not sync_config.filter_collection.lazy_load_members():
@@ -719,6 +749,26 @@ class SyncHandler(object):
                    lazy_load_members=lazy_load_members,
                )
            elif batch.limited:
+                state_at_timeline_start = yield self.store.get_state_ids_for_event(
+                    batch.events[0].event_id, types=types,
+                    filtered_types=filtered_types,
+                )
+
+                # for now, we disable LL for gappy syncs - see
+                # https://github.com/vector-im/riot-web/issues/7211#issuecomment-419976346
+                # N.B. this slows down incr syncs as we are now processing way
+                # more state in the server than if we were LLing.
+                #
+                # We still have to filter timeline_start to LL entries (above) in order
+                # for _calculate_state's LL logic to work, as we have to include LL
+                # members for timeline senders in case they weren't loaded in the initial
+                # sync.  We do this by (counterintuitively) by filtering timeline_start
+                # members to just be ones which were timeline senders, which then ensures
+                # all of the rest get included in the state block (if we need to know
+                # about them).
+                types = None
+                filtered_types = None
+
                state_at_previous_sync = yield self.get_state_at(
                    room_id, stream_position=since_token, types=types,
                    filtered_types=filtered_types,
@@ -729,24 +779,21 @@ class SyncHandler(object):
                    filtered_types=filtered_types,
                )

-                state_at_timeline_start = yield self.store.get_state_ids_for_event(
-                    batch.events[0].event_id, types=types,
-                    filtered_types=filtered_types,
-                )
-
                state_ids = _calculate_state(
                    timeline_contains=timeline_state,
                    timeline_start=state_at_timeline_start,
                    previous=state_at_previous_sync,
                    current=current_state_ids,
+                    # we have to include LL members in case LL initial sync missed them
                    lazy_load_members=lazy_load_members,
                )
            else:
                state_ids = {}
                if lazy_load_members:
                    if types:
-                        # We're returning an incremental sync, with no "gap" since
-                        # the previous sync, so normally there would be no state to return
+                        # We're returning an incremental sync, with no
+                        # "gap" since the previous sync, so normally there would be
+                        # no state to return.
                        # But we're lazy-loading, so the client might need some more
                        # member events to understand the events in this timeline.
                        # So we fish out all the member events corresponding to the
@@ -774,7 +821,7 @@ class SyncHandler(object):
                    logger.debug("filtering state from %r...", state_ids)
                    state_ids = {
                        t: event_id
-                        for t, event_id in state_ids.iteritems()
+                        for t, event_id in iteritems(state_ids)
                        if cache.get(t[1]) != event_id
                    }
                    logger.debug("...to %r", state_ids)
@@ -1575,6 +1622,19 @@ class SyncHandler(object):
            newly_joined_room=newly_joined,
        )

+        # When we join the room (or the client requests full_state), we should
+        # send down any existing tags. Usually the user won't have tags in a
+        # newly joined room, unless either a) they've joined before or b) the
+        # tag was added by synapse e.g. for server notice rooms.
+        if full_state:
+            user_id = sync_result_builder.sync_config.user.to_string()
+            tags = yield self.store.get_tags_for_room(user_id, room_id)
+
+            # If there aren't any tags, don't send the empty tags list down
+            # sync
+            if not tags:
+                tags = None
+
        account_data_events = []
        if tags is not None:
            account_data_events.append({
@@ -1603,10 +1663,24 @@ class SyncHandler(object):
        )

        summary = {}
+
+        # we include a summary in room responses when we're lazy loading
+        # members (as the client otherwise doesn't have enough info to form
+        # the name itself).
        if (
            sync_config.filter_collection.lazy_load_members() and
            (
+                # we recalulate the summary:
+                #   if there are membership changes in the timeline, or
+                #   if membership has changed during a gappy sync, or
+                #   if this is an initial sync.
                any(ev.type == EventTypes.Member for ev in batch.events) or
+                (
+                    # XXX: this may include false positives in the form of LL
+                    # members which have snuck into state
+                    batch.limited and
+                    any(t == EventTypes.Member for (t, k) in state)
+                ) or
                since_token is None
            )
        ):
@@ -1636,6 +1710,16 @@ class SyncHandler(object):
                    unread_notifications["highlight_count"] = notifs["highlight_count"]

                sync_result_builder.joined.append(room_sync)
+
+            if batch.limited and since_token:
+                user_id = sync_result_builder.sync_config.user.to_string()
+                logger.info(
+                    "Incremental gappy sync of %s for user %s with %d state events" % (
+                        room_id,
+                        user_id,
+                        len(state),
+                    )
+                )
        elif room_builder.rtype == "archived":
            room_sync = ArchivedSyncResult(
                room_id=room_id,
@@ -1729,17 +1813,17 @@ def _calculate_state(
    event_id_to_key = {
        e: key
        for key, e in itertools.chain(
-            timeline_contains.items(),
-            previous.items(),
-            timeline_start.items(),
-            current.items(),
+            iteritems(timeline_contains),
+            iteritems(previous),
+            iteritems(timeline_start),
+            iteritems(current),
        )
    }

-    c_ids = set(e for e in current.values())
-    ts_ids = set(e for e in timeline_start.values())
-    p_ids = set(e for e in previous.values())
-    tc_ids = set(e for e in timeline_contains.values())
+    c_ids = set(e for e in itervalues(current))
+    ts_ids = set(e for e in itervalues(timeline_start))
+    p_ids = set(e for e in itervalues(previous))
+    tc_ids = set(e for e in itervalues(timeline_contains))

    # If we are lazyloading room members, we explicitly add the membership events
    # for the senders in the timeline into the state block returned by /sync,
@@ -1753,7 +1837,7 @@ def _calculate_state(

    if lazy_load_members:
        p_ids.difference_update(
-            e for t, e in timeline_start.iteritems()
+            e for t, e in iteritems(timeline_start)
            if t[0] == EventTypes.Member
        )

--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -13,24 +13,25 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import logging
-import urllib

-from six import StringIO
+from six import text_type
+from six.moves import urllib

+import treq
 from canonicaljson import encode_canonical_json, json
 from prometheus_client import Counter

 from OpenSSL import SSL
 from OpenSSL.SSL import VERIFY_NONE
-from twisted.internet import defer, protocol, reactor, ssl, task
+from twisted.internet import defer, protocol, reactor, ssl
 from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
 from twisted.web._newclient import ResponseDone
 from twisted.web.client import (
    Agent,
    BrowserLikeRedirectAgent,
    ContentDecoderAgent,
-    FileBodyProducer as TwistedFileBodyProducer,
    GzipDecoder,
    HTTPConnectionPool,
    PartialDownloadError,
@@ -83,18 +84,20 @@ class SimpleHttpClient(object):
        if hs.config.user_agent_suffix:
            self.user_agent = "%s %s" % (self.user_agent, hs.config.user_agent_suffix,)

+        self.user_agent = self.user_agent.encode('ascii')
+
    @defer.inlineCallbacks
-    def request(self, method, uri, *args, **kwargs):
+    def request(self, method, uri, data=b'', headers=None):
        # A small wrapper around self.agent.request() so we can easily attach
        # counters to it
        outgoing_requests_counter.labels(method).inc()

        # log request but strip `access_token` (AS requests for example include this)
-        logger.info("Sending request %s %s", method, redact_uri(uri))
+        logger.info("Sending request %s %s", method, redact_uri(uri.encode('ascii')))

        try:
-            request_deferred = self.agent.request(
-                method, uri, *args, **kwargs
+            request_deferred = treq.request(
+                method, uri, agent=self.agent, data=data, headers=headers
            )
            add_timeout_to_deferred(
                request_deferred, 60, self.hs.get_reactor(),
@@ -105,14 +108,14 @@ class SimpleHttpClient(object):
            incoming_responses_counter.labels(method, response.code).inc()
            logger.info(
                "Received response to  %s %s: %s",
-                method, redact_uri(uri), response.code
+                method, redact_uri(uri.encode('ascii')), response.code
            )
            defer.returnValue(response)
        except Exception as e:
            incoming_responses_counter.labels(method, "ERR").inc()
            logger.info(
                "Error sending request to  %s %s: %s %s",
-                method, redact_uri(uri), type(e).__name__, e.message
+                method, redact_uri(uri.encode('ascii')), type(e).__name__, e.args[0]
            )
            raise

@@ -137,7 +140,8 @@ class SimpleHttpClient(object):
        # TODO: Do we ever want to log message contents?
        logger.debug("post_urlencoded_get_json args: %s", args)

-        query_bytes = urllib.urlencode(encode_urlencode_args(args), True)
+        query_bytes = urllib.parse.urlencode(
+            encode_urlencode_args(args), True).encode("utf8")

        actual_headers = {
            b"Content-Type": [b"application/x-www-form-urlencoded"],
@@ -148,15 +152,14 @@ class SimpleHttpClient(object):

        response = yield self.request(
            "POST",
-            uri.encode("ascii"),
+            uri,
            headers=Headers(actual_headers),
-            bodyProducer=FileBodyProducer(StringIO(query_bytes))
+            data=query_bytes
        )

-        body = yield make_deferred_yieldable(readBody(response))
-
        if 200 <= response.code < 300:
-            defer.returnValue(json.loads(body))
+            body = yield make_deferred_yieldable(treq.json_content(response))
+            defer.returnValue(body)
        else:
            raise HttpResponseException(response.code, response.phrase, body)

@@ -191,9 +194,9 @@ class SimpleHttpClient(object):

        response = yield self.request(
            "POST",
-            uri.encode("ascii"),
+            uri,
            headers=Headers(actual_headers),
-            bodyProducer=FileBodyProducer(StringIO(json_str))
+            data=json_str
        )

        body = yield make_deferred_yieldable(readBody(response))
@@ -248,7 +251,7 @@ class SimpleHttpClient(object):
            ValueError: if the response was not JSON
        """
        if len(args):
-            query_bytes = urllib.urlencode(args, True)
+            query_bytes = urllib.parse.urlencode(args, True)
            uri = "%s?%s" % (uri, query_bytes)

        json_str = encode_canonical_json(json_body)
@@ -262,9 +265,9 @@ class SimpleHttpClient(object):

        response = yield self.request(
            "PUT",
-            uri.encode("ascii"),
+            uri,
            headers=Headers(actual_headers),
-            bodyProducer=FileBodyProducer(StringIO(json_str))
+            data=json_str
        )

        body = yield make_deferred_yieldable(readBody(response))
@@ -293,7 +296,7 @@ class SimpleHttpClient(object):
            HttpResponseException on a non-2xx HTTP response.
        """
        if len(args):
-            query_bytes = urllib.urlencode(args, True)
+            query_bytes = urllib.parse.urlencode(args, True)
            uri = "%s?%s" % (uri, query_bytes)

        actual_headers = {
@@ -304,7 +307,7 @@ class SimpleHttpClient(object):

        response = yield self.request(
            "GET",
-            uri.encode("ascii"),
+            uri,
            headers=Headers(actual_headers),
        )

@@ -339,7 +342,7 @@ class SimpleHttpClient(object):

        response = yield self.request(
            "GET",
-            url.encode("ascii"),
+            url,
            headers=Headers(actual_headers),
        )

@@ -434,12 +437,12 @@ class CaptchaServerHttpClient(SimpleHttpClient):

    @defer.inlineCallbacks
    def post_urlencoded_get_raw(self, url, args={}):
-        query_bytes = urllib.urlencode(encode_urlencode_args(args), True)
+        query_bytes = urllib.parse.urlencode(encode_urlencode_args(args), True)

        response = yield self.request(
            "POST",
-            url.encode("ascii"),
-            bodyProducer=FileBodyProducer(StringIO(query_bytes)),
+            url,
+            data=query_bytes,
            headers=Headers({
                b"Content-Type": [b"application/x-www-form-urlencoded"],
                b"User-Agent": [self.user_agent],
@@ -510,7 +513,7 @@ def encode_urlencode_args(args):


 def encode_urlencode_arg(arg):
-    if isinstance(arg, unicode):
+    if isinstance(arg, text_type):
        return arg.encode('utf-8')
    elif isinstance(arg, list):
        return [encode_urlencode_arg(i) for i in arg]
@@ -542,26 +545,3 @@ class InsecureInterceptableContextFactory(ssl.ContextFactory):

    def creatorForNetloc(self, hostname, port):
        return self
-
-
-class FileBodyProducer(TwistedFileBodyProducer):
-    """Workaround for https://twistedmatrix.com/trac/ticket/8473
-
-    We override the pauseProducing and resumeProducing methods in twisted's
-    FileBodyProducer so that they do not raise exceptions if the task has
-    already completed.
-    """
-
-    def pauseProducing(self):
-        try:
-            super(FileBodyProducer, self).pauseProducing()
-        except task.TaskDone:
-            # task has already completed
-            pass
-
-    def resumeProducing(self):
-        try:
-            super(FileBodyProducer, self).resumeProducing()
-        except task.NotPaused:
-            # task was not paused (probably because it had already completed)
-            pass
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -17,19 +17,19 @@ import cgi
 import logging
 import random
 import sys
-import urllib

-from six import string_types
-from six.moves.urllib import parse as urlparse
+from six import PY3, string_types
+from six.moves import urllib

-from canonicaljson import encode_canonical_json, json
+import treq
+from canonicaljson import encode_canonical_json
 from prometheus_client import Counter
 from signedjson.sign import sign_json

-from twisted.internet import defer, protocol, reactor
+from twisted.internet import defer, protocol
 from twisted.internet.error import DNSLookupError
 from twisted.web._newclient import ResponseDone
-from twisted.web.client import Agent, HTTPConnectionPool, readBody
+from twisted.web.client import Agent, HTTPConnectionPool
 from twisted.web.http_headers import Headers

 import synapse.metrics
@@ -40,10 +40,8 @@ from synapse.api.errors import (
    HttpResponseException,
    SynapseError,
 )
-from synapse.http import cancelled_to_request_timed_out_error
 from synapse.http.endpoint import matrix_federation_endpoint
 from synapse.util import logcontext
-from synapse.util.async_helpers import add_timeout_to_deferred
 from synapse.util.logcontext import make_deferred_yieldable

 logger = logging.getLogger(__name__)
@@ -58,16 +56,22 @@ incoming_responses_counter = Counter("synapse_http_matrixfederationclient_respon
 MAX_LONG_RETRIES = 10
 MAX_SHORT_RETRIES = 3

+if PY3:
+    MAXINT = sys.maxsize
+else:
+    MAXINT = sys.maxint
+

 class MatrixFederationEndpointFactory(object):
    def __init__(self, hs):
+        self.reactor = hs.get_reactor()
        self.tls_client_options_factory = hs.tls_client_options_factory

    def endpointForURI(self, uri):
-        destination = uri.netloc
+        destination = uri.netloc.decode('ascii')

        return matrix_federation_endpoint(
-            reactor, destination, timeout=10,
+            self.reactor, destination, timeout=10,
            tls_client_options_factory=self.tls_client_options_factory
        )

@@ -85,6 +89,7 @@ class MatrixFederationHttpClient(object):
        self.hs = hs
        self.signing_key = hs.config.signing_key[0]
        self.server_name = hs.hostname
+        reactor = hs.get_reactor()
        pool = HTTPConnectionPool(reactor)
        pool.maxPersistentPerHost = 5
        pool.cachedConnectionTimeout = 2 * 60
@@ -93,26 +98,33 @@ class MatrixFederationHttpClient(object):
        )
        self.clock = hs.get_clock()
        self._store = hs.get_datastore()
-        self.version_string = hs.version_string
+        self.version_string = hs.version_string.encode('ascii')
        self._next_id = 1
+        self.default_timeout = 60

    def _create_url(self, destination, path_bytes, param_bytes, query_bytes):
-        return urlparse.urlunparse(
-            ("matrix", destination, path_bytes, param_bytes, query_bytes, "")
+        return urllib.parse.urlunparse(
+            (b"matrix", destination, path_bytes, param_bytes, query_bytes, b"")
        )

    @defer.inlineCallbacks
    def _request(self, destination, method, path,
-                 body_callback, headers_dict={}, param_bytes=b"",
-                 query_bytes=b"", retry_on_dns_fail=True,
+                 json=None, json_callback=None,
+                 param_bytes=b"",
+                 query=None, retry_on_dns_fail=True,
                 timeout=None, long_retries=False,
                 ignore_backoff=False,
                 backoff_on_404=False):
-        """ Creates and sends a request to the given server
+        """
+        Creates and sends a request to the given server.
+
        Args:
            destination (str): The remote server to send the HTTP request to.
            method (str): HTTP method
            path (str): The HTTP path
+            json (dict or None): JSON to send in the body.
+            json_callback (func or None): A callback to generate the JSON.
+            query (dict or None): Query arguments.
            ignore_backoff (bool): true to ignore the historical backoff data
                and try the request anyway.
            backoff_on_404 (bool): Back off if we get a 404
@@ -132,6 +144,11 @@ class MatrixFederationHttpClient(object):
            (May also fail with plenty of other Exceptions for things like DNS
                failures, connection failures, SSL failures.)
        """
+        if timeout:
+            _sec_timeout = timeout / 1000
+        else:
+            _sec_timeout = self.default_timeout
+
        if (
            self.hs.config.federation_domain_whitelist is not None and
            destination not in self.hs.config.federation_domain_whitelist
@@ -146,23 +163,25 @@ class MatrixFederationHttpClient(object):
            ignore_backoff=ignore_backoff,
        )

-        destination = destination.encode("ascii")
+        headers_dict = {}
        path_bytes = path.encode("ascii")
-        with limiter:
-            headers_dict[b"User-Agent"] = [self.version_string]
-            headers_dict[b"Host"] = [destination]
+        if query:
+            query_bytes = encode_query_args(query)
+        else:
+            query_bytes = b""

-            url_bytes = self._create_url(
-                destination, path_bytes, param_bytes, query_bytes
-            )
+        headers_dict = {
+            "User-Agent": [self.version_string],
+            "Host": [destination],
+        }
+
+        with limiter:
+            url = self._create_url(
+                destination.encode("ascii"), path_bytes, param_bytes, query_bytes
+            ).decode('ascii')

            txn_id = "%s-O-%s" % (method, self._next_id)
-            self._next_id = (self._next_id + 1) % (sys.maxint - 1)
-
-            outbound_logger.info(
-                "{%s} [%s] Sending request: %s %s",
-                txn_id, destination, method, url_bytes
-            )
+            self._next_id = (self._next_id + 1) % (MAXINT - 1)

            # XXX: Would be much nicer to retry only at the transaction-layer
            # (once we have reliable transactions in place)
@@ -171,80 +190,97 @@ class MatrixFederationHttpClient(object):
            else:
                retries_left = MAX_SHORT_RETRIES

-            http_url_bytes = urlparse.urlunparse(
-                ("", "", path_bytes, param_bytes, query_bytes, "")
-            )
+            http_url = urllib.parse.urlunparse(
+                (b"", b"", path_bytes, param_bytes, query_bytes, b"")
+            ).decode('ascii')

            log_result = None
-            try:
-                while True:
-                    producer = None
-                    if body_callback:
-                        producer = body_callback(method, http_url_bytes, headers_dict)
+            while True:
+                try:
+                    if json_callback:
+                        json = json_callback()

-                    try:
-                        request_deferred = self.agent.request(
-                            method,
-                            url_bytes,
-                            Headers(headers_dict),
-                            producer
-                        )
-                        add_timeout_to_deferred(
-                            request_deferred,
-                            timeout / 1000. if timeout else 60,
-                            self.hs.get_reactor(),
-                            cancelled_to_request_timed_out_error,
-                        )
-                        response = yield make_deferred_yieldable(
-                            request_deferred,
+                    if json:
+                        data = encode_canonical_json(json)
+                        headers_dict["Content-Type"] = ["application/json"]
+                        self.sign_request(
+                            destination, method, http_url, headers_dict, json
                        )
+                    else:
+                        data = None
+                        self.sign_request(destination, method, http_url, headers_dict)

-                        log_result = "%d %s" % (response.code, response.phrase,)
-                        break
-                    except Exception as e:
-                        if not retry_on_dns_fail and isinstance(e, DNSLookupError):
-                            logger.warn(
-                                "DNS Lookup failed to %s with %s",
-                                destination,
-                                e
-                            )
-                            log_result = "DNS Lookup failed to %s with %s" % (
-                                destination, e
-                            )
-                            raise
+                    outbound_logger.info(
+                        "{%s} [%s] Sending request: %s %s",
+                        txn_id, destination, method, url
+                    )

+                    request_deferred = treq.request(
+                        method,
+                        url,
+                        headers=Headers(headers_dict),
+                        data=data,
+                        agent=self.agent,
+                        reactor=self.hs.get_reactor()
+                    )
+                    request_deferred.addTimeout(_sec_timeout, self.hs.get_reactor())
+                    response = yield make_deferred_yieldable(
+                        request_deferred,
+                    )
+
+                    log_result = "%d %s" % (response.code, response.phrase,)
+                    break
+                except Exception as e:
+                    if not retry_on_dns_fail and isinstance(e, DNSLookupError):
                        logger.warn(
-                            "{%s} Sending request failed to %s: %s %s: %s",
-                            txn_id,
+                            "DNS Lookup failed to %s with %s",
                            destination,
-                            method,
-                            url_bytes,
-                            _flatten_response_never_received(e),
+                            e
+                        )
+                        log_result = "DNS Lookup failed to %s with %s" % (
+                            destination, e
+                        )
+                        raise
+
+                    logger.warn(
+                        "{%s} Sending request failed to %s: %s %s: %s",
+                        txn_id,
+                        destination,
+                        method,
+                        url,
+                        _flatten_response_never_received(e),
+                    )
+
+                    log_result = _flatten_response_never_received(e)
+
+                    if retries_left and not timeout:
+                        if long_retries:
+                            delay = 4 ** (MAX_LONG_RETRIES + 1 - retries_left)
+                            delay = min(delay, 60)
+                            delay *= random.uniform(0.8, 1.4)
+                        else:
+                            delay = 0.5 * 2 ** (MAX_SHORT_RETRIES - retries_left)
+                            delay = min(delay, 2)
+                            delay *= random.uniform(0.8, 1.4)
+
+                        logger.debug(
+                            "{%s} Waiting %s before sending to %s...",
+                            txn_id,
+                            delay,
+                            destination
                        )

-                        log_result = _flatten_response_never_received(e)
-
-                        if retries_left and not timeout:
-                            if long_retries:
-                                delay = 4 ** (MAX_LONG_RETRIES + 1 - retries_left)
-                                delay = min(delay, 60)
-                                delay *= random.uniform(0.8, 1.4)
-                            else:
-                                delay = 0.5 * 2 ** (MAX_SHORT_RETRIES - retries_left)
-                                delay = min(delay, 2)
-                                delay *= random.uniform(0.8, 1.4)
-
-                            yield self.clock.sleep(delay)
-                            retries_left -= 1
-                        else:
-                            raise
-            finally:
-                outbound_logger.info(
-                    "{%s} [%s] Result: %s",
-                    txn_id,
-                    destination,
-                    log_result,
-                )
+                        yield self.clock.sleep(delay)
+                        retries_left -= 1
+                    else:
+                        raise
+                finally:
+                    outbound_logger.info(
+                        "{%s} [%s] Result: %s",
+                        txn_id,
+                        destination,
+                        log_result,
+                    )

            if 200 <= response.code < 300:
                pass
@@ -252,7 +288,9 @@ class MatrixFederationHttpClient(object):
                # :'(
                # Update transactions table?
                with logcontext.PreserveLoggingContext():
-                    body = yield readBody(response)
+                    d = treq.content(response)
+                    d.addTimeout(_sec_timeout, self.hs.get_reactor())
+                    body = yield make_deferred_yieldable(d)
                raise HttpResponseException(
                    response.code, response.phrase, body
                )
@@ -297,11 +335,11 @@ class MatrixFederationHttpClient(object):
        auth_headers = []

        for key, sig in request["signatures"][self.server_name].items():
-            auth_headers.append(bytes(
+            auth_headers.append((
                "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % (
                    self.server_name, key, sig,
-                )
-            ))
+                )).encode('ascii')
+            )

        headers_dict[b"Authorization"] = auth_headers

@@ -347,24 +385,14 @@ class MatrixFederationHttpClient(object):
        """

        if not json_data_callback:
-            def json_data_callback():
-                return data
-
-        def body_callback(method, url_bytes, headers_dict):
-            json_data = json_data_callback()
-            self.sign_request(
-                destination, method, url_bytes, headers_dict, json_data
-            )
-            producer = _JsonProducer(json_data)
-            return producer
+            json_data_callback = lambda: data

        response = yield self._request(
            destination,
            "PUT",
            path,
-            body_callback=body_callback,
-            headers_dict={"Content-Type": ["application/json"]},
-            query_bytes=encode_query_args(args),
+            json_callback=json_data_callback,
+            query=args,
            long_retries=long_retries,
            timeout=timeout,
            ignore_backoff=ignore_backoff,
@@ -376,8 +404,10 @@ class MatrixFederationHttpClient(object):
            check_content_type_is_json(response.headers)

        with logcontext.PreserveLoggingContext():
-            body = yield readBody(response)
-        defer.returnValue(json.loads(body))
+            d = treq.json_content(response)
+            d.addTimeout(self.default_timeout, self.hs.get_reactor())
+            body = yield make_deferred_yieldable(d)
+        defer.returnValue(body)

    @defer.inlineCallbacks
    def post_json(self, destination, path, data={}, long_retries=False,
@@ -410,20 +440,12 @@ class MatrixFederationHttpClient(object):
            Fails with ``FederationDeniedError`` if this destination
            is not on our federation whitelist
        """
-
-        def body_callback(method, url_bytes, headers_dict):
-            self.sign_request(
-                destination, method, url_bytes, headers_dict, data
-            )
-            return _JsonProducer(data)
-
        response = yield self._request(
            destination,
            "POST",
            path,
-            query_bytes=encode_query_args(args),
-            body_callback=body_callback,
-            headers_dict={"Content-Type": ["application/json"]},
+            query=args,
+            json=data,
            long_retries=long_retries,
            timeout=timeout,
            ignore_backoff=ignore_backoff,
@@ -434,9 +456,16 @@ class MatrixFederationHttpClient(object):
            check_content_type_is_json(response.headers)

        with logcontext.PreserveLoggingContext():
-            body = yield readBody(response)
+            d = treq.json_content(response)
+            if timeout:
+                _sec_timeout = timeout / 1000
+            else:
+                _sec_timeout = self.default_timeout

-        defer.returnValue(json.loads(body))
+            d.addTimeout(_sec_timeout, self.hs.get_reactor())
+            body = yield make_deferred_yieldable(d)
+
+        defer.returnValue(body)

    @defer.inlineCallbacks
    def get_json(self, destination, path, args=None, retry_on_dns_fail=True,
@@ -471,16 +500,11 @@ class MatrixFederationHttpClient(object):

        logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail)

-        def body_callback(method, url_bytes, headers_dict):
-            self.sign_request(destination, method, url_bytes, headers_dict)
-            return None
-
        response = yield self._request(
            destination,
            "GET",
            path,
-            query_bytes=encode_query_args(args),
-            body_callback=body_callback,
+            query=args,
            retry_on_dns_fail=retry_on_dns_fail,
            timeout=timeout,
            ignore_backoff=ignore_backoff,
@@ -491,9 +515,11 @@ class MatrixFederationHttpClient(object):
            check_content_type_is_json(response.headers)

        with logcontext.PreserveLoggingContext():
-            body = yield readBody(response)
+            d = treq.json_content(response)
+            d.addTimeout(self.default_timeout, self.hs.get_reactor())
+            body = yield make_deferred_yieldable(d)

-        defer.returnValue(json.loads(body))
+        defer.returnValue(body)

    @defer.inlineCallbacks
    def delete_json(self, destination, path, long_retries=False,
@@ -523,13 +549,11 @@ class MatrixFederationHttpClient(object):
            Fails with ``FederationDeniedError`` if this destination
            is not on our federation whitelist
        """
-
        response = yield self._request(
            destination,
            "DELETE",
            path,
-            query_bytes=encode_query_args(args),
-            headers_dict={"Content-Type": ["application/json"]},
+            query=args,
            long_retries=long_retries,
            timeout=timeout,
            ignore_backoff=ignore_backoff,
@@ -540,9 +564,11 @@ class MatrixFederationHttpClient(object):
            check_content_type_is_json(response.headers)

        with logcontext.PreserveLoggingContext():
-            body = yield readBody(response)
+            d = treq.json_content(response)
+            d.addTimeout(self.default_timeout, self.hs.get_reactor())
+            body = yield make_deferred_yieldable(d)

-        defer.returnValue(json.loads(body))
+        defer.returnValue(body)

    @defer.inlineCallbacks
    def get_file(self, destination, path, output_stream, args={},
@@ -569,26 +595,11 @@ class MatrixFederationHttpClient(object):
            Fails with ``FederationDeniedError`` if this destination
            is not on our federation whitelist
        """
-
-        encoded_args = {}
-        for k, vs in args.items():
-            if isinstance(vs, string_types):
-                vs = [vs]
-            encoded_args[k] = [v.encode("UTF-8") for v in vs]
-
-        query_bytes = urllib.urlencode(encoded_args, True)
-        logger.debug("Query bytes: %s Retry DNS: %s", query_bytes, retry_on_dns_fail)
-
-        def body_callback(method, url_bytes, headers_dict):
-            self.sign_request(destination, method, url_bytes, headers_dict)
-            return None
-
        response = yield self._request(
            destination,
            "GET",
            path,
-            query_bytes=query_bytes,
-            body_callback=body_callback,
+            query=args,
            retry_on_dns_fail=retry_on_dns_fail,
            ignore_backoff=ignore_backoff,
        )
@@ -597,9 +608,9 @@ class MatrixFederationHttpClient(object):

        try:
            with logcontext.PreserveLoggingContext():
-                length = yield _readBodyToFile(
-                    response, output_stream, max_size
-                )
+                d = _readBodyToFile(response, output_stream, max_size)
+                d.addTimeout(self.default_timeout, self.hs.get_reactor())
+                length = yield make_deferred_yieldable(d)
        except Exception:
            logger.exception("Failed to download body")
            raise
@@ -639,30 +650,6 @@ def _readBodyToFile(response, stream, max_size):
    return d


-class _JsonProducer(object):
-    """ Used by the twisted http client to create the HTTP body from json
-    """
-    def __init__(self, jsn):
-        self.reset(jsn)
-
-    def reset(self, jsn):
-        self.body = encode_canonical_json(jsn)
-        self.length = len(self.body)
-
-    def startProducing(self, consumer):
-        consumer.write(self.body)
-        return defer.succeed(None)
-
-    def pauseProducing(self):
-        pass
-
-    def stopProducing(self):
-        pass
-
-    def resumeProducing(self):
-        pass
-
-
 def _flatten_response_never_received(e):
    if hasattr(e, "reasons"):
        reasons = ", ".join(
@@ -693,7 +680,7 @@ def check_content_type_is_json(headers):
            "No Content-Type header"
        )

-    c_type = c_type[0]  # only the first header
+    c_type = c_type[0].decode('ascii')  # only the first header
    val, options = cgi.parse_header(c_type)
    if val != "application/json":
        raise RuntimeError(
@@ -711,6 +698,6 @@ def encode_query_args(args):
            vs = [vs]
        encoded_args[k] = [v.encode("UTF-8") for v in vs]

-    query_bytes = urllib.urlencode(encoded_args, True)
+    query_bytes = urllib.parse.urlencode(encoded_args, True)

-    return query_bytes
+    return query_bytes.encode('utf8')
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -204,14 +204,14 @@ class SynapseRequest(Request):
        self.start_time = time.time()
        self.request_metrics = RequestMetrics()
        self.request_metrics.start(
-            self.start_time, name=servlet_name, method=self.method,
+            self.start_time, name=servlet_name, method=self.method.decode('ascii'),
        )

        self.site.access_logger.info(
            "%s - %s - Received request: %s %s",
            self.getClientIP(),
            self.site.site_tag,
-            self.method,
+            self.method.decode('ascii'),
            self.get_redacted_uri()
        )

--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -40,9 +40,10 @@ REQUIREMENTS = {
    "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"],
    "service_identity>=1.0.0": ["service_identity>=1.0.0"],
    "Twisted>=17.1.0": ["twisted>=17.1.0"],
+    "treq>=15.1": ["treq>=15.1"],

-    # We use crypto.get_elliptic_curve which is only supported in >=0.15
-    "pyopenssl>=0.15": ["OpenSSL>=0.15"],
+    # Twisted has required pyopenssl 16.0 since about Twisted 16.6.
+    "pyopenssl>=16.0.0": ["OpenSSL>=16.0.0"],

    "pyyaml": ["yaml"],
    "pyasn1": ["pyasn1"],
--- a/synapse/replication/tcp/protocol.py
+++ b/synapse/replication/tcp/protocol.py
@@ -590,9 +590,9 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol):
 pending_commands = LaterGauge(
    "synapse_replication_tcp_protocol_pending_commands",
    "",
-    ["name", "conn_id"],
+    ["name"],
    lambda: {
-        (p.name, p.conn_id): len(p.pending_commands) for p in connected_connections
+        (p.name,): len(p.pending_commands) for p in connected_connections
    },
 )

@@ -607,9 +607,9 @@ def transport_buffer_size(protocol):
 transport_send_buffer = LaterGauge(
    "synapse_replication_tcp_protocol_transport_send_buffer",
    "",
-    ["name", "conn_id"],
+    ["name"],
    lambda: {
-        (p.name, p.conn_id): transport_buffer_size(p) for p in connected_connections
+        (p.name,): transport_buffer_size(p) for p in connected_connections
    },
 )

@@ -632,9 +632,9 @@ def transport_kernel_read_buffer_size(protocol, read=True):
 tcp_transport_kernel_send_buffer = LaterGauge(
    "synapse_replication_tcp_protocol_transport_kernel_send_buffer",
    "",
-    ["name", "conn_id"],
+    ["name"],
    lambda: {
-        (p.name, p.conn_id): transport_kernel_read_buffer_size(p, False)
+        (p.name,): transport_kernel_read_buffer_size(p, False)
        for p in connected_connections
    },
 )
@@ -643,9 +643,9 @@ tcp_transport_kernel_send_buffer = LaterGauge(
 tcp_transport_kernel_read_buffer = LaterGauge(
    "synapse_replication_tcp_protocol_transport_kernel_read_buffer",
    "",
-    ["name", "conn_id"],
+    ["name"],
    lambda: {
-        (p.name, p.conn_id): transport_kernel_read_buffer_size(p, True)
+        (p.name,): transport_kernel_read_buffer_size(p, True)
        for p in connected_connections
    },
 )
@@ -654,9 +654,9 @@ tcp_transport_kernel_read_buffer = LaterGauge(
 tcp_inbound_commands = LaterGauge(
    "synapse_replication_tcp_protocol_inbound_commands",
    "",
-    ["command", "name", "conn_id"],
+    ["command", "name"],
    lambda: {
-        (k[0], p.name, p.conn_id): count
+        (k[0], p.name,): count
        for p in connected_connections
        for k, count in iteritems(p.inbound_commands_counter)
    },
@@ -665,9 +665,9 @@ tcp_inbound_commands = LaterGauge(
 tcp_outbound_commands = LaterGauge(
    "synapse_replication_tcp_protocol_outbound_commands",
    "",
-    ["command", "name", "conn_id"],
+    ["command", "name"],
    lambda: {
-        (k[0], p.name, p.conn_id): count
+        (k[0], p.name,): count
        for p in connected_connections
        for k, count in iteritems(p.outbound_commands_counter)
    },
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -101,7 +101,7 @@ class UserRegisterServlet(ClientV1RestServlet):

        nonce = self.hs.get_secrets().token_hex(64)
        self.nonces[nonce] = int(self.reactor.seconds())
-        return (200, {"nonce": nonce.encode('ascii')})
+        return (200, {"nonce": nonce})

    @defer.inlineCallbacks
    def on_POST(self, request):
@@ -164,7 +164,7 @@ class UserRegisterServlet(ClientV1RestServlet):
            key=self.hs.config.registration_shared_secret.encode(),
            digestmod=hashlib.sha1,
        )
-        want_mac.update(nonce)
+        want_mac.update(nonce.encode('utf8'))
        want_mac.update(b"\x00")
        want_mac.update(username)
        want_mac.update(b"\x00")
@@ -173,7 +173,10 @@ class UserRegisterServlet(ClientV1RestServlet):
        want_mac.update(b"admin" if admin else b"notadmin")
        want_mac = want_mac.hexdigest()

-        if not hmac.compare_digest(want_mac, got_mac.encode('ascii')):
+        if not hmac.compare_digest(
+                want_mac.encode('ascii'),
+                got_mac.encode('ascii')
+        ):
            raise SynapseError(403, "HMAC incorrect")

        # Reuse the parts of RegisterRestServlet to reduce code duplication
--- a/synapse/rest/client/v1/events.py
+++ b/synapse/rest/client/v1/events.py
@@ -45,20 +45,20 @@ class EventStreamRestServlet(ClientV1RestServlet):
        is_guest = requester.is_guest
        room_id = None
        if is_guest:
-            if "room_id" not in request.args:
+            if b"room_id" not in request.args:
                raise SynapseError(400, "Guest users must specify room_id param")
-        if "room_id" in request.args:
-            room_id = request.args["room_id"][0]
+        if b"room_id" in request.args:
+            room_id = request.args[b"room_id"][0].decode('ascii')

        pagin_config = PaginationConfig.from_request(request)
        timeout = EventStreamRestServlet.DEFAULT_LONGPOLL_TIME_MS
-        if "timeout" in request.args:
+        if b"timeout" in request.args:
            try:
-                timeout = int(request.args["timeout"][0])
+                timeout = int(request.args[b"timeout"][0])
            except ValueError:
                raise SynapseError(400, "timeout must be in milliseconds.")

-        as_client_event = "raw" not in request.args
+        as_client_event = b"raw" not in request.args

        chunk = yield self.event_stream_handler.get_stream(
            requester.user.to_string(),
--- a/synapse/rest/client/v1/initial_sync.py
+++ b/synapse/rest/client/v1/initial_sync.py
@@ -32,7 +32,7 @@ class InitialSyncRestServlet(ClientV1RestServlet):
    @defer.inlineCallbacks
    def on_GET(self, request):
        requester = yield self.auth.get_user_by_req(request)
-        as_client_event = "raw" not in request.args
+        as_client_event = b"raw" not in request.args
        pagination_config = PaginationConfig.from_request(request)
        include_archived = parse_boolean(request, "archived", default=False)
        content = yield self.initial_sync_handler.snapshot_all_rooms(
--- a/synapse/rest/client/v1/login.py
+++ b/synapse/rest/client/v1/login.py
@@ -14,10 +14,9 @@
 # limitations under the License.

 import logging
-import urllib
 import xml.etree.ElementTree as ET

-from six.moves.urllib import parse as urlparse
+from six.moves import urllib

 from canonicaljson import json
 from saml2 import BINDING_HTTP_POST, config
@@ -134,7 +133,7 @@ class LoginRestServlet(ClientV1RestServlet):
                                       LoginRestServlet.SAML2_TYPE):
                relay_state = ""
                if "relay_state" in login_submission:
-                    relay_state = "&RelayState=" + urllib.quote(
+                    relay_state = "&RelayState=" + urllib.parse.quote(
                                  login_submission["relay_state"])
                result = {
                    "uri": "%s%s" % (self.idp_redirect_url, relay_state)
@@ -366,7 +365,7 @@ class SAML2RestServlet(ClientV1RestServlet):
            (user_id, token) = yield handler.register_saml2(username)
            # Forward to the RelayState callback along with ava
            if 'RelayState' in request.args:
-                request.redirect(urllib.unquote(
+                request.redirect(urllib.parse.unquote(
                                 request.args['RelayState'][0]) +
                                 '?status=authenticated&access_token=' +
                                 token + '&user_id=' + user_id + '&ava=' +
@@ -377,7 +376,7 @@ class SAML2RestServlet(ClientV1RestServlet):
                                     "user_id": user_id, "token": token,
                                     "ava": saml2_auth.ava}))
        elif 'RelayState' in request.args:
-            request.redirect(urllib.unquote(
+            request.redirect(urllib.parse.unquote(
                             request.args['RelayState'][0]) +
                             '?status=not_authenticated')
            finish_request(request)
@@ -390,21 +389,22 @@ class CasRedirectServlet(ClientV1RestServlet):

    def __init__(self, hs):
        super(CasRedirectServlet, self).__init__(hs)
-        self.cas_server_url = hs.config.cas_server_url
-        self.cas_service_url = hs.config.cas_service_url
+        self.cas_server_url = hs.config.cas_server_url.encode('ascii')
+        self.cas_service_url = hs.config.cas_service_url.encode('ascii')

    def on_GET(self, request):
        args = request.args
-        if "redirectUrl" not in args:
+        if b"redirectUrl" not in args:
            return (400, "Redirect URL not specified for CAS auth")
-        client_redirect_url_param = urllib.urlencode({
-            "redirectUrl": args["redirectUrl"][0]
-        })
-        hs_redirect_url = self.cas_service_url + "/_matrix/client/api/v1/login/cas/ticket"
-        service_param = urllib.urlencode({
-            "service": "%s?%s" % (hs_redirect_url, client_redirect_url_param)
-        })
-        request.redirect("%s/login?%s" % (self.cas_server_url, service_param))
+        client_redirect_url_param = urllib.parse.urlencode({
+            b"redirectUrl": args[b"redirectUrl"][0]
+        }).encode('ascii')
+        hs_redirect_url = (self.cas_service_url +
+                           b"/_matrix/client/api/v1/login/cas/ticket")
+        service_param = urllib.parse.urlencode({
+            b"service": b"%s?%s" % (hs_redirect_url, client_redirect_url_param)
+        }).encode('ascii')
+        request.redirect(b"%s/login?%s" % (self.cas_server_url, service_param))
        finish_request(request)


@@ -422,11 +422,11 @@ class CasTicketServlet(ClientV1RestServlet):

    @defer.inlineCallbacks
    def on_GET(self, request):
-        client_redirect_url = request.args["redirectUrl"][0]
+        client_redirect_url = request.args[b"redirectUrl"][0]
        http_client = self.hs.get_simple_http_client()
        uri = self.cas_server_url + "/proxyValidate"
        args = {
-            "ticket": request.args["ticket"],
+            "ticket": request.args[b"ticket"][0].decode('ascii'),
            "service": self.cas_service_url
        }
        try:
@@ -471,11 +471,11 @@ class CasTicketServlet(ClientV1RestServlet):
        finish_request(request)

    def add_login_token_to_redirect_url(self, url, token):
-        url_parts = list(urlparse.urlparse(url))
-        query = dict(urlparse.parse_qsl(url_parts[4]))
+        url_parts = list(urllib.parse.urlparse(url))
+        query = dict(urllib.parse.parse_qsl(url_parts[4]))
        query.update({"loginToken": token})
-        url_parts[4] = urllib.urlencode(query)
-        return urlparse.urlunparse(url_parts)
+        url_parts[4] = urllib.parse.urlencode(query).encode('ascii')
+        return urllib.parse.urlunparse(url_parts)

    def parse_cas_response(self, cas_response_body):
        user = None
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Erik Johnston	6c0f8d9d50	Merge pull request #3856 from matrix-org/erikj/speed_up_purge Make purge history slightly faster	2018-09-13 16:14:46 +01:00
Erik Johnston	ed5331a627	comment	2018-09-13 16:10:56 +01:00
Amber Brown	cb64fe2cb7	Merge pull request #3859 from matrix-org/erikj/add_iterkeys Fix handling of redacted events from federation	2018-09-14 01:06:44 +10:00
Erik Johnston	13193a6e2b	Newsfile	2018-09-13 15:46:45 +01:00
Amber Brown	3126b88d35	fix circleci merged builds (#3858 ) * fix * changelog	2018-09-14 00:44:31 +10:00
Erik Johnston	89a76d1889	Fix handling of redacted events from federation If we receive an event that doesn't pass their content hash check (e.g. due to already being redacted) then we hit a bug which causes an exception to be raised, which then promplty stops the event (and request) from being processed. This effects all sorts of federation APIs, including joining rooms with a redacted state event.	2018-09-13 15:44:12 +01:00
Amber Brown	bfa0b759e0	Attempt to figure out what's going on with timeouts (#3857 )	2018-09-14 00:15:51 +10:00
Erik Johnston	9cbd0094f0	pep8	2018-09-13 15:15:35 +01:00
Erik Johnston	9dbe38ea7d	Create indices after insertion	2018-09-13 15:05:52 +01:00
Erik Johnston	93139a1fb8	Merge branch 'develop' of github.com:matrix-org/synapse into erikj/speed_up_purge	2018-09-13 12:57:09 +01:00
Erik Johnston	e7cd7cb0f0	Newsfile	2018-09-13 12:55:40 +01:00
Erik Johnston	c857f5ef9b	Make purge history slightly faster Don't pull out events that are outliers and won't be deleted, as nothing should happen to them.	2018-09-13 12:48:10 +01:00
Amber Brown	b7d2fb5eb9	Remove some superfluous logging (#3855 )	2018-09-13 19:59:32 +10:00
Neil Johnson	f30a303590	Merge pull request #3846 from matrix-org/neilj/expose-registered-users expose number of real reserved users	2018-09-12 17:14:04 +01:00
Matthew Hodgson	2ac1abbc7e	show heroes if a room has a 'deleted' name/canonical_alias (#3851 )	2018-09-12 17:11:05 +01:00
Erik Johnston	fa0d464fa4	Merge pull request #3853 from matrix-org/erikj/log_outbound_each_time Log outbound requests when we retry	2018-09-12 16:55:40 +01:00
Matthew Hodgson	0403cf0783	argh pep8	2018-09-12 16:54:28 +01:00
Matthew Hodgson	0e200e366d	correctly log gappy sync metrics	2018-09-12 16:47:20 +01:00
Matthew Hodgson	11bfc2af1c	fix logline	2018-09-12 16:45:42 +01:00
Erik Johnston	3db016b641	Newsfile	2018-09-12 16:25:18 +01:00
Neil Johnson	8decd6233d	improve naming	2018-09-12 16:22:15 +01:00
Erik Johnston	8c5b84441b	Log outbound requests when we retry	2018-09-12 16:22:14 +01:00
Erik Johnston	54f8616d2c	Merge pull request #3841 from matrix-org/erikj/manhole_key_length Change the manhole SSH key to have more bits	2018-09-12 14:33:39 +01:00
Amber Brown	65cd8ccc79	Add JUnit summaries to CircleCI as well as merged runs (#3704 )	2018-09-12 23:29:21 +10:00
Amber Brown	7ca097f77e	Port federation/ to py3 (#3847 )	2018-09-12 23:23:32 +10:00
Neil Johnson	5cea4e16c7	towncrier	2018-09-12 12:03:31 +01:00
Neil Johnson	0ddf486724	expose number of real reserved users	2018-09-12 11:58:52 +01:00
Amber Brown	546aee7e52	Merge pull request #3835 from krombel/fix_3821 fix VOIP crashes under Python 3	2018-09-12 20:44:18 +10:00
Amber Brown	33716c4aea	Merge pull request #3826 from matrix-org/rav/logging_for_keyring add some logging for the keyring queue	2018-09-12 20:43:47 +10:00
Amber Brown	bc635026c5	Merge pull request #3824 from matrix-org/rav/fix_jwt_import Fix jwt import check	2018-09-12 20:41:57 +10:00
Amber Brown	02aa41809b	Port rest/ to Python 3 (#3823 )	2018-09-12 20:41:31 +10:00
Amber Brown	8fd93b5eea	Port crypto/ to Python 3 (#3822 )	2018-09-12 20:16:31 +10:00
Amber Brown	4073f73edc	Merge pull request #3845 from matrix-org/erikj/timeout_reads Timeout reading body for outbound HTTP requests	2018-09-12 20:16:08 +10:00
Erik Johnston	649c647955	Newsfile	2018-09-12 11:03:42 +01:00
Erik Johnston	4084a774a8	Timeout reading body for outbound HTTP requests	2018-09-12 10:10:20 +01:00
Matthew Hodgson	b041115415	Speed up lazy loading (#3827 ) * speed up room summaries by pulling their data from room_memberships rather than room state * disable LL for incr syncs, and log incr sync stats (#3840)	2018-09-12 00:50:39 +01:00
Erik Johnston	9a68778ac2	Newsfile	2018-09-11 10:44:40 +01:00
Erik Johnston	9e05c8d309	Change the manhole SSH key to have more bits Newer versions of openssh client refuse to connect to the old key due to its length.	2018-09-11 10:42:10 +01:00
Erik Johnston	037a06e8f0	Merge pull request #3834 from mvgorcum/develop Remove build requirements after building docker image	2018-09-10 16:53:25 +01:00
Jan Christian Grünhage	af10fa6536	add runtime dependencies	2018-09-10 17:39:49 +02:00
Mathijs van Gorcum	e957428a15	Newsfile	2018-09-10 14:53:05 +00:00
Mathijs van Gorcum	e586916cda	Move COPY before RUN and merge RUNs	2018-09-10 14:02:42 +00:00
Krombel	3572a206d3	add changelog	2018-09-10 14:33:08 +02:00
Krombel	7bc22539ff	fix VOIP crashes under Python 3 (#3821 )	2018-09-10 14:30:08 +02:00
Mathijs van Gorcum	b7e7712f07	Remove build requirements after building	2018-09-10 12:21:42 +00:00
Richard van der Hoff	1e4c7fff5f	changelog	2018-09-07 16:37:30 +01:00
Amber Brown	9a5ea511b5	Merge pull request #3810 from matrix-org/erikj/send_tags_down_sync_on_join Send existing room tags down sync on join	2018-09-07 23:28:42 +10:00
Richard van der Hoff	e33a538af3	Merge pull request #3783 from cwmke/develop Add apache vhost config to Readme	2018-09-07 14:25:23 +01:00
Richard van der Hoff	edda9f5cac	changelog	2018-09-07 14:23:35 +01:00
Richard van der Hoff	b8ad756bd0	Fix jwt import check This handy code attempted to check that we could import jwt, but utterly failed to check it was the right jwt. Fixes https://github.com/matrix-org/synapse/issues/3793	2018-09-07 14:20:54 +01:00
Amber Brown	771d213ac5	Merge branch 'master' into develop	2018-09-07 21:45:38 +10:00
Amber Brown	b60749a1ec	changelog	2018-09-07 21:41:57 +10:00
Amber Brown	6febd8e8f7	version	2018-09-07 21:40:57 +10:00
Richard van der Hoff	cd7ef43872	clearer logging when things fail, too	2018-09-06 23:56:47 +01:00
Richard van der Hoff	806964b5de	add some logging for the keyring queue why is it so damn slow?	2018-09-06 18:51:06 +01:00
Amber Brown	52ec6e9dfa	Port tests/ to Python 3 (#3808 )	2018-09-07 02:58:18 +10:00
Neil Johnson	c5440b2ca0	Merge pull request #3800 from matrix-org/neilj/remove-guests-from-mau-count guest users should not be part of mau total	2018-09-06 17:45:55 +01:00
Neil Johnson	84a750e0c3	ensure guests never enter mau list	2018-09-06 17:22:53 +01:00
Erik Johnston	7298efd361	Newsfile	2018-09-06 17:13:33 +01:00
Erik Johnston	f60c9e2a01	Don't send empty tags list down sync	2018-09-06 17:01:41 +01:00
Erik Johnston	7baf66ef5d	Send existing room tags down sync on join When a user joined a room any existing tags were not sent down the sync stream. Ordinarily this isn't a problem because the user needs to be in the room to have set tags in it, however synapse will sometimes add tags for a user to a room, e.g. for server notices, which need to come down sync.	2018-09-06 16:46:51 +01:00
Amber Brown	654324eded	Merge pull request #3805 from matrix-org/erikj/limit_transaction_pdus_edus Limit the number of PDUs/EDUs per fedreation transaction	2018-09-07 01:33:31 +10:00
Amber Brown	1d371fc5b3	Merge pull request #3806 from matrix-org/erikj/limit_postgres_travis Only start postgres instance for postgres tests on Travis CI	2018-09-07 01:06:21 +10:00
Erik Johnston	b07a2cbee9	Spelling	2018-09-06 15:53:02 +01:00
Amber Brown	70fd75cd1d	Merge pull request #3788 from matrix-org/erikj/remove_conn_id Remove conn_id from repl prometheus metrics	2018-09-07 00:48:19 +10:00
Erik Johnston	5d848992bf	Newsfile	2018-09-06 15:28:46 +01:00
Erik Johnston	3b4223aa23	Only start postgres instance for postgres tests on Travis CI	2018-09-06 15:27:54 +01:00
Erik Johnston	28f5bfdcf7	Newsfile	2018-09-06 15:25:58 +01:00
Amber Brown	ee7c8bd2b5	Merge pull request #3795 from matrix-org/erikj/faster_sync_state User iter* during sync state calculations	2018-09-07 00:24:43 +10:00
Erik Johnston	6707a3212c	Limit the number of PDUs/EDUs per fedreation transaction	2018-09-06 15:23:55 +01:00
Amber Brown	135f3b4390	Merge pull request #3804 from matrix-org/rav/fix_openssl_dep bump dep on pyopenssl to 16.x	2018-09-07 00:23:39 +10:00
Amber Brown	2608ebc04c	Port handlers/ to Python 3 (#3803 )	2018-09-07 00:22:23 +10:00
Erik Johnston	599f65bb89	Merge branch 'master' of github.com:matrix-org/synapse into release-v0.33.4	2018-09-06 14:13:58 +01:00
Erik Johnston	417e7077aa	Bump version and changelog	2018-09-06 14:08:55 +01:00
Erik Johnston	d64b24dfe6	Merge tag 'v0.33.3.1' into release-v0.33.4 Synapse 0.33.3.1 (2018-09-06) ============================= SECURITY FIXES -------------- - Fix an issue where event signatures were not always correctly validated ([\#3796](https://github.com/matrix-org/synapse/issues/3796)) - Fix an issue where server_acls could be circumvented for incoming events ([\#3796](https://github.com/matrix-org/synapse/issues/3796)) Internal Changes ---------------- - Unignore synctl in .dockerignore to fix docker builds ([\#3802](https://github.com/matrix-org/synapse/issues/3802))	2018-09-06 14:08:33 +01:00
Richard van der Hoff	4f8baab0c4	Merge branch 'master' into develop	2018-09-06 13:05:22 +01:00
Richard van der Hoff	b3c2ebba32	changelog	2018-09-06 12:55:17 +01:00
Richard van der Hoff	625542878d	bump dep on pyopenssl to 16.x	2018-09-06 12:53:15 +01:00
Richard van der Hoff	2fd17b5ad1	Merge tag 'v0.33.3.1' Synapse 0.33.3.1 (2018-09-06) ============================= SECURITY FIXES -------------- - Fix an issue where event signatures were not always correctly validated ([\#3796](https://github.com/matrix-org/synapse/issues/3796)) - Fix an issue where server_acls could be circumvented for incoming events ([\#3796](https://github.com/matrix-org/synapse/issues/3796)) Internal Changes ---------------- - Unignore synctl in .dockerignore to fix docker builds ([\#3802](https://github.com/matrix-org/synapse/issues/3802))	2018-09-06 12:31:35 +01:00
Amber Brown	10587f7f32	Merge pull request #3802 from matrix-org/jcgruenhage/docker-unignore-synctl remove synctl from .dockerignore	2018-09-06 19:47:33 +10:00
Richard van der Hoff	80189ed27c	prepare v0.33.3.1	2018-09-06 10:26:23 +01:00
Jan Christian Grünhage	0cd7b209e2	Create 3802.misc	2018-09-06 10:24:59 +01:00
Jan Christian Grünhage	78d1042c10	remove synctl from .dockerignore	2018-09-06 10:24:59 +01:00
Jan Christian Grünhage	af3125226d	Create 3802.misc	2018-09-06 10:46:00 +02:00
Jan Christian Grünhage	9c8cd855da	remove synctl from .dockerignore	2018-09-06 10:39:55 +02:00
Neil Johnson	92657be7d0	towncrier	2018-09-05 22:33:45 +01:00
Neil Johnson	61b05727fa	guest users should not be part of mau total	2018-09-05 22:30:36 +01:00
Richard van der Hoff	dfba1d843d	Merge pull request #3790 from matrix-org/rav/respect_event_format_in_filter Implement 'event_format' filter param in /sync	2018-09-05 16:24:14 +01:00
Erik Johnston	2254790ae4	Newsfile	2018-09-05 16:21:18 +01:00
Erik Johnston	7419764351	User iter* during sync state calculations	2018-09-05 16:19:50 +01:00
Amber Brown	2d2828dcbc	Port http/ to Python 3 (#3771 )	2018-09-06 00:10:47 +10:00
Richard van der Hoff	c127c8d042	Fix origin handling for pushed transactions Use the actual origin for push transactions, rather than whatever the remote server claimed.	2018-09-05 13:08:07 +01:00
Richard van der Hoff	804dd41e18	Check that signatures on events are valid We should check that both the sender's server, and the server which created the event_id (which may be different from whatever the remote server has told us the origin is), have signed the event.	2018-09-05 13:08:07 +01:00
Richard van der Hoff	c91bd295f5	changelog	2018-09-04 15:23:19 +01:00
Richard van der Hoff	87c18d12ee	Implement 'event_format' filter param in /sync This has been specced and part-implemented; let's implement it for /sync (but no other endpoints yet :/).	2018-09-04 15:20:09 +01:00
Amber Brown	7e9ced4178	version and towncrier	2018-09-04 21:12:04 +10:00
Erik Johnston	3e242dc149	Remove conn_id	2018-09-04 11:45:52 +01:00
Erik Johnston	87b111f96a	Newsfile	2018-09-03 17:26:15 +01:00
Erik Johnston	b13836da7f	Remove conn_id from repl prometheus metrics `conn_id` gets set to a random string, and so we end up filling up prometheus with tonnes of data series, which is bad.	2018-09-03 17:22:49 +01:00
Colin W	81942c109d	Remove end '/'s	2018-09-02 23:28:03 -05:00
Colin W	a395f1ddb3	Update readme on develop branch	2018-09-02 17:30:07 -05:00
				`@@ -1 +0,0 @@`
				`Removed the link to the unmaintained matrix-synapse-auto-deploy project from the readme.`
				`@@ -1 +0,0 @@`
				`Refactor state module to support multiple room versions`
				`@@ -0,0 +1 @@`
				`CircleCI tests now run on the potential merge of a PR.`
				`@@ -1 +0,0 @@`
				`Fix error collecting prometheus metrics when run on dedicated thread due to threading concurrency issues`
				`@@ -1 +0,0 @@`
				`Allow guests to use /rooms/:roomId/event/:eventId`
				`@@ -1 +0,0 @@`
				`The synapse.storage module has been ported to Python 3.`
				`@@ -1 +0,0 @@`
				`Split the state_group_cache into member and non-member state events (and so speed up LL /sync)`
				`@@ -1 +0,0 @@`
				`Log failure to authenticate remote servers as warnings (without stack traces)`
				`@@ -1 +0,0 @@`
				`The CONTRIBUTING guidelines have been updated to mention our use of Markdown and that .misc files have content.`