changelog

Merge branch 'develop' into matthew/lazy_load_yourself
Merge pull request #3910 from matrix-org/erikj/update_timeout
2018-09-19 16:21:15 +01:00 · 2018-09-19 16:06:06 +01:00 · 2018-09-19 11:45:11 +01:00 · 2018-09-19 11:19:47 +01:00 · 2018-09-19 11:07:29 +01:00 · 2018-09-19 11:07:27 +01:00
178 changed files with 2546 additions and 1341 deletions
@@ -9,6 +9,8 @@ jobs:
      - store_artifacts:
          path: ~/project/logs
          destination: logs
+      - store_test_results:
+          path: logs
  sytestpy2postgres:
    machine: true
    steps:
@@ -18,15 +20,45 @@ jobs:
      - store_artifacts:
          path: ~/project/logs
          destination: logs
+      - store_test_results:
+          path: logs
+  sytestpy2merged:
+    machine: true
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy2
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs
+
+  sytestpy2postgresmerged:
+    machine: true
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy2
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs
+
  sytestpy3:
    machine: true
    steps:
      - checkout
      - run: docker pull matrixdotorg/sytest-synapsepy3
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs hawkowl/sytestpy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
      - store_artifacts:
          path: ~/project/logs
          destination: logs
+      - store_test_results:
+          path: logs
  sytestpy3postgres:
    machine: true
    steps:
@@ -36,6 +68,32 @@ jobs:
      - store_artifacts:
          path: ~/project/logs
          destination: logs
+      - store_test_results:
+          path: logs
+  sytestpy3merged:
+    machine: true
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs
+  sytestpy3postgresmerged:
+    machine: true
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs

 workflows:
  version: 2
@@ -43,6 +101,21 @@ workflows:
    jobs:
      - sytestpy2
      - sytestpy2postgres
-# Currently broken while the Python 3 port is incomplete
-#      - sytestpy3
-#      - sytestpy3postgres
+      - sytestpy3
+      - sytestpy3postgres
+      - sytestpy2merged:
+          filters:
+            branches:
+              ignore: /develop|master/
+      - sytestpy2postgresmerged:
+          filters:
+            branches:
+              ignore: /develop|master/
+      - sytestpy3merged:
+          filters:
+            branches:
+              ignore: /develop|master/
+      - sytestpy3postgresmerged:
+          filters:
+            branches:
+              ignore: /develop|master/
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -e
+
+# CircleCI doesn't give CIRCLE_PR_NUMBER in the environment for non-forked PRs. Wonderful.
+# In this case, we just need to do some ~shell magic~ to strip it out of the PULL_REQUEST URL.
+echo 'export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}"' >> $BASH_ENV
+source $BASH_ENV
+
+if [[ -z "${CIRCLE_PR_NUMBER}" ]]
+then
+    echo "Can't figure out what the PR number is!"
+    exit 1
+fi
+
+# Get the reference, using the GitHub API
+GITBASE=`curl -q https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
+
+# Show what we are before
+git show -s
+
+# Set up username so it can do a merge
+git config --global user.email bot@matrix.org
+git config --global user.name "A robot"
+
+# Fetch and merge. If it doesn't work, it will raise due to set -e.
+git fetch -u origin $GITBASE
+git merge --no-edit origin/$GITBASE
+
+# Show what we are after.
+git show -s
@@ -3,6 +3,5 @@ Dockerfile
 .gitignore
 demo/etc
 tox.ini
-synctl
 .git/*
 .tox/*
@@ -4,6 +4,7 @@

 .DS_Store
 _trial_temp/
+_trial_temp*/
 logs/
 dbs/
 *.egg
@@ -44,6 +45,7 @@ media_store/
 build/
 venv/
 venv*/
+*venv/

 localhost-800*/
 static/client/register/register_config.js
@@ -8,9 +8,6 @@ before_script:
  - git remote set-branches --add origin develop
  - git fetch origin develop

-services:
-  - postgresql
-
 matrix:
  fast_finish: true
  include:
@@ -25,6 +22,11 @@ matrix:

  - python: 2.7
    env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
+    services:
+      - postgresql
+
+  - python: 3.5
+    env: TOX_ENV=py35

  - python: 3.6
    env: TOX_ENV=py36
@@ -1,3 +1,77 @@
+Synapse 0.33.4 (2018-09-07)
+===========================
+
+Internal Changes
+----------------
+
+- Unignore synctl in .dockerignore to fix docker builds ([\#3802](https://github.com/matrix-org/synapse/issues/3802))
+
+
+Synapse 0.33.4rc2 (2018-09-06)
+==============================
+
+Pull in security fixes from v0.33.3.1
+
+
+Synapse 0.33.3.1 (2018-09-06)
+=============================
+
+SECURITY FIXES
+--------------
+
+- Fix an issue where event signatures were not always correctly validated ([\#3796](https://github.com/matrix-org/synapse/issues/3796))
+- Fix an issue where server_acls could be circumvented for incoming events ([\#3796](https://github.com/matrix-org/synapse/issues/3796))
+
+
+Internal Changes
+----------------
+
+- Unignore synctl in .dockerignore to fix docker builds ([\#3802](https://github.com/matrix-org/synapse/issues/3802))
+
+
+Synapse 0.33.4rc1 (2018-09-04)
+==============================
+
+Features
+--------
+
+- Support profile API endpoints on workers ([\#3659](https://github.com/matrix-org/synapse/issues/3659))
+- Server notices for resource limit blocking ([\#3680](https://github.com/matrix-org/synapse/issues/3680))
+- Allow guests to use /rooms/:roomId/event/:eventId ([\#3724](https://github.com/matrix-org/synapse/issues/3724))
+- Add mau_trial_days config param, so that users only get counted as MAU after N days. ([\#3749](https://github.com/matrix-org/synapse/issues/3749))
+- Require twisted 17.1 or later (fixes [#3741](https://github.com/matrix-org/synapse/issues/3741)). ([\#3751](https://github.com/matrix-org/synapse/issues/3751))
+
+
+Bugfixes
+--------
+
+- Fix error collecting prometheus metrics when run on dedicated thread due to threading concurrency issues ([\#3722](https://github.com/matrix-org/synapse/issues/3722))
+- Fix bug where we resent "limit exceeded" server notices repeatedly ([\#3747](https://github.com/matrix-org/synapse/issues/3747))
+- Fix bug where we broke sync when using limit_usage_by_mau but hadn't configured server notices ([\#3753](https://github.com/matrix-org/synapse/issues/3753))
+- Fix 'federation_domain_whitelist' such that an empty list correctly blocks all outbound federation traffic ([\#3754](https://github.com/matrix-org/synapse/issues/3754))
+- Fix tagging of server notice rooms ([\#3755](https://github.com/matrix-org/synapse/issues/3755), [\#3756](https://github.com/matrix-org/synapse/issues/3756))
+- Fix 'admin_uri' config variable and error parameter to be 'admin_contact' to match the spec. ([\#3758](https://github.com/matrix-org/synapse/issues/3758))
+- Don't return non-LL-member state in incremental sync state blocks ([\#3760](https://github.com/matrix-org/synapse/issues/3760))
+- Fix bug in sending presence over federation ([\#3768](https://github.com/matrix-org/synapse/issues/3768))
+- Fix bug where preserved threepid user comes to sign up and server is mau blocked ([\#3777](https://github.com/matrix-org/synapse/issues/3777))
+
+Internal Changes
+----------------
+
+- Removed the link to the unmaintained matrix-synapse-auto-deploy project from the readme. ([\#3378](https://github.com/matrix-org/synapse/issues/3378))
+- Refactor state module to support multiple room versions ([\#3673](https://github.com/matrix-org/synapse/issues/3673))
+- The synapse.storage module has been ported to Python 3. ([\#3725](https://github.com/matrix-org/synapse/issues/3725))
+- Split the state_group_cache into member and non-member state events (and so speed up LL /sync) ([\#3726](https://github.com/matrix-org/synapse/issues/3726))
+- Log failure to authenticate remote servers as warnings (without stack traces) ([\#3727](https://github.com/matrix-org/synapse/issues/3727))
+- The CONTRIBUTING guidelines have been updated to mention our use of Markdown and that .misc files have content. ([\#3730](https://github.com/matrix-org/synapse/issues/3730))
+- Reference the need for an HTTP replication port when using the federation_reader worker ([\#3734](https://github.com/matrix-org/synapse/issues/3734))
+- Fix minor spelling error in federation client documentation. ([\#3735](https://github.com/matrix-org/synapse/issues/3735))
+- Remove redundant state resolution function ([\#3737](https://github.com/matrix-org/synapse/issues/3737))
+- The test suite now passes on PostgreSQL. ([\#3740](https://github.com/matrix-org/synapse/issues/3740))
+- Fix MAU cache invalidation due to missing yield ([\#3746](https://github.com/matrix-org/synapse/issues/3746))
+- Make sure that we close db connections opened during init ([\#3764](https://github.com/matrix-org/synapse/issues/3764))
+
+
 Synapse 0.33.3 (2018-08-22)
 ===========================

@@ -157,7 +157,7 @@ if you prefer.

 In case of problems, please see the _`Troubleshooting` section below.

-There is an offical synapse image available at 
+There is an offical synapse image available at
 https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with
 the docker-compose file available at `contrib/docker <contrib/docker>`_. Further information on
 this including configuration options is available in the README on
@@ -459,37 +459,13 @@ https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-

 Windows Install
 ---------------
-Synapse can be installed on Cygwin. It requires the following Cygwin packages:
-
- gcc
- git
- libffi-devel
- openssl (and openssl-devel, python-openssl)
- python
- python-setuptools
-
-The content repository requires additional packages and will be unable to process
-uploads without them:
-
- libjpeg8
- libjpeg8-devel
- zlib
-
-If you choose to install Synapse without these packages, you will need to reinstall
-``pillow`` for changes to be applied, e.g. ``pip uninstall pillow`` ``pip install
-pillow --user``
-
-Troubleshooting:
-
- You may need to upgrade ``setuptools`` to get this to work correctly:
-  ``pip install setuptools --upgrade``.
- You may encounter errors indicating that ``ffi.h`` is missing, even with
-  ``libffi-devel`` installed. If you do, copy the ``.h`` files:
-  ``cp /usr/lib/libffi-3.0.13/include/*.h /usr/include``
- You may need to install libsodium from source in order to install PyNacl. If
-  you do, you may need to create a symlink to ``libsodium.a`` so ``ld`` can find
-  it: ``ln -s /usr/local/lib/libsodium.a /usr/lib/libsodium.a``

+If you wish to run or develop Synapse on Windows, the Windows Subsystem For
+Linux provides a Linux environment on Windows 10 which is capable of using the
+Debian, Fedora, or source installation methods. More information about WSL can
+be found at https://docs.microsoft.com/en-us/windows/wsl/install-win10 for
+Windows 10 and https://docs.microsoft.com/en-us/windows/wsl/install-on-server
+for Windows Server.

 Troubleshooting
 ===============
@@ -742,6 +718,18 @@ so an example nginx configuration might look like::
      }
  }

+and an example apache configuration may look like::
+
+    <VirtualHost *:443>
+        SSLEngine on
+        ServerName matrix.example.com;
+
+        <Location /_matrix>
+            ProxyPass http://127.0.0.1:8008/_matrix nocanon
+            ProxyPassReverse http://127.0.0.1:8008/_matrix
+        </Location>
+    </VirtualHost>
+
 You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true``
 for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are
 recorded correctly.
@@ -896,7 +884,7 @@ to install using pip and a virtualenv::

    virtualenv -p python2.7 env
    source env/bin/activate
-    python synapse/python_dependencies.py | xargs pip install
+    python -m synapse.python_dependencies | xargs pip install
    pip install lxml mock

 This will run a process of downloading and installing all the needed
@@ -951,5 +939,13 @@ variable.  The default is 0.5, which can be decreased to reduce RAM usage
 in memory constrained enviroments, or increased if performance starts to
 degrade.

+Using `libjemalloc <http://jemalloc.net/>`_ can also yield a significant
+improvement in overall amount, and especially in terms of giving back RAM
+to the OS. To use it, the library must simply be put in the LD_PRELOAD
+environment variable when launching Synapse. On Debian, this can be done
+by installing the ``libjemalloc1`` package and adding this line to
+``/etc/default/matrix-synapse``::
+
+    LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1

 .. _`key_management`: https://matrix.org/docs/spec/server_server/unstable.html#retrieving-server-keys
@@ -1 +0,0 @@
-Removed the link to the unmaintained matrix-synapse-auto-deploy project from the readme.
@@ -0,0 +1 @@
+Python 3.5+ is now supported.
@@ -1 +0,0 @@
-Support profile API endpoints on workers
@@ -1 +0,0 @@
-Refactor state module to support multiple room versions
@@ -1 +0,0 @@
-Server notices for resource limit blocking
@@ -0,0 +1 @@
+CircleCI tests now run on the potential merge of a PR.
@@ -1 +0,0 @@
-Fix error collecting prometheus metrics when run on dedicated thread due to threading concurrency issues
@@ -1 +0,0 @@
-Allow guests to use /rooms/:roomId/event/:eventId
@@ -1 +0,0 @@
-The synapse.storage module has been ported to Python 3.
@@ -1 +0,0 @@
-Split the state_group_cache into member and non-member state events (and so speed up LL /sync)
@@ -1 +0,0 @@
-Log failure to authenticate remote servers as warnings (without stack traces)
@@ -1 +0,0 @@
-The CONTRIBUTING guidelines have been updated to mention our use of Markdown and that .misc files have content.
@@ -1 +0,0 @@
-Reference the need for an HTTP replication port when using the federation_reader worker
@@ -1 +0,0 @@
-Fix minor spelling error in federation client documentation.
@@ -1 +0,0 @@
-Remove redundant state resolution function
@@ -1 +0,0 @@
-The test suite now passes on PostgreSQL.
@@ -1 +0,0 @@
-Fix MAU cache invalidation due to missing yield
@@ -1 +0,0 @@
-Fix bug where we resent "limit exceeded" server notices repeatedly
@@ -1 +0,0 @@
-Add mau_trial_days config param, so that users only get counted as MAU after N days.
@@ -1 +0,0 @@
-Require twisted 17.1 or later (fixes [#3741](https://github.com/matrix-org/synapse/issues/3741)).
@@ -1 +0,0 @@
-Fix bug where we broke sync when using limit_usage_by_mau but hadn't configured server notices
@@ -1 +0,0 @@
-Fix 'federation_domain_whitelist' such that an empty list correctly blocks all outbound federation traffic
@@ -1 +0,0 @@
-Fix tagging of server notice rooms
@@ -1 +0,0 @@
-Fix tagging of server notice rooms
@@ -1 +0,0 @@
-Fix 'admin_uri' config variable and error parameter to be 'admin_contact' to match the spec.
@@ -1 +0,0 @@
-Don't return non-LL-member state in incremental sync state blocks
@@ -1 +0,0 @@
-Make sure that we close db connections opened during init
@@ -1 +0,0 @@
-Fix bug in sending presence over federation
@@ -0,0 +1 @@
+http/ is now ported to Python 3.
@@ -1 +0,0 @@
-Fix bug where preserved threepid user comes to sign up and server is mau blocked
@@ -0,0 +1 @@
+Remove connection ID for replication prometheus metrics, as it creates a large number of new series.
@@ -0,0 +1 @@
+Implement `event_format` filter param in `/sync`
@@ -0,0 +1 @@
+Make /sync slightly faster by avoiding needless copies
@@ -0,0 +1 @@
+guest users should not be part of mau total
@@ -0,0 +1 @@
+handlers/ is now ported to Python 3.
@@ -0,0 +1 @@
+Bump dependency on pyopenssl 16.x, to avoid incompatibility with recent Twisted.
@@ -0,0 +1 @@
+Limit the number of PDUs/EDUs per federation transaction
@@ -0,0 +1 @@
+Only start postgres instance for postgres tests on Travis CI
@@ -0,0 +1 @@
+tests/ is now ported to Python 3.
@@ -0,0 +1 @@
+Fix existing room tags not coming down sync when joining a room
@@ -0,0 +1 @@
+crypto/ is now ported to Python 3.
@@ -0,0 +1 @@
+rest/ is now ported to Python 3.
@@ -0,0 +1 @@
+Fix jwt import check
@@ -0,0 +1 @@
+add some logging for the keyring queue
@@ -0,0 +1 @@
+speed up lazy loading by 2-3x
@@ -0,0 +1 @@
+Improved Dockerfile to remove build requirements after building reducing the image size.
@@ -0,0 +1 @@
+fix VOIP crashes under Python 3 (#3821)
@@ -0,0 +1 @@
+Disable lazy loading for incremental syncs for now
@@ -0,0 +1 @@
+Fix manhole so that it works with latest openssh clients
@@ -0,0 +1 @@
+Fix outbound requests occasionally wedging, which can result in federation breaking between servers.
@@ -0,0 +1 @@
+Add synapse_admin_mau:registered_reserved_users metric to expose number of real reaserved users 
@@ -0,0 +1 @@
+federation/ is now ported to Python 3.
@@ -0,0 +1 @@
+Show heroes if room name/canonical alias has been deleted
@@ -0,0 +1 @@
+Log when we retry outbound requests
@@ -0,0 +1 @@
+Removed some excess logging messages.
@@ -0,0 +1 @@
+Speed up purge history for rooms that have been previously purged
@@ -0,0 +1 @@
+Refactor some HTTP timeout code.
@@ -0,0 +1 @@
+Fix running merged builds on CircleCI
@@ -0,0 +1 @@
+Fix handling of redacted events from federation
@@ -0,0 +1 @@
+Fix typo in replication stream exception.
@@ -0,0 +1 @@
+Add in flight real time metrics for Measure blocks
@@ -0,0 +1 @@
+Disable buffering and automatic retrying in treq requests to prevent timeouts.
@@ -0,0 +1,2 @@
+Remove documentation regarding installation on Cygwin, the use of WSL is 
+recommended instead.
@@ -0,0 +1 @@
+Mitigate outbound federation randomly becoming wedged
@@ -0,0 +1 @@
+mention jemalloc in the README
@@ -0,0 +1 @@
+Don't ratelimit autojoins
@@ -0,0 +1 @@
+Adding the ability to change MAX_UPLOAD_SIZE for the docker container variables.
@@ -0,0 +1 @@
+Remove unmaintained "nuke-room-from-db.sh" script
@@ -0,0 +1 @@
+Fix 500 error when deleting unknown room alias
@@ -0,0 +1 @@
+Fix some b'abcd' noise in logs and metrics
@@ -0,0 +1 @@
+Report "python_version" in the phone home stats
@@ -0,0 +1 @@
+Fix some b'abcd' noise in logs and metrics
@@ -0,0 +1 @@
+Fix typo in README, synaspse -> synapse
@@ -0,0 +1 @@
+When we join a room, always try the server we used for the alias lookup first, to avoid unresponsive and out-of-date servers.
@@ -0,0 +1 @@
+Increase the timeout when filling missing events in federation requests
@@ -0,0 +1 @@
+Improve logging of outbound federation requests
@@ -0,0 +1 @@
+Fix incorrect server-name indication for outgoing federation requests
@@ -0,0 +1 @@
+Improve logging of outbound federation requests
@@ -0,0 +1 @@
+Fix bug where things occaisonally were not being timed out correctly.
@@ -0,0 +1 @@
+Always LL ourselves if we're in a room
@@ -1,6 +1,8 @@
 FROM docker.io/python:2-alpine3.8

-RUN apk add --no-cache --virtual .nacl_deps \
+COPY . /synapse
+
+RUN apk add --no-cache --virtual .build_deps \
        build-base \
        libffi-dev \
        libjpeg-turbo-dev \
@@ -8,13 +10,16 @@ RUN apk add --no-cache --virtual .nacl_deps \
        libxslt-dev \
        linux-headers \
        postgresql-dev \
-        su-exec \
-        zlib-dev
-
-COPY . /synapse
-
-# A wheel cache may be provided in ./cache for faster build
-RUN cd /synapse \
+        zlib-dev \
+ && cd /synapse \
+ && apk add --no-cache --virtual .runtime_deps \
+ 	libffi \
+        libjpeg-turbo \
+	libressl \
+	libxslt \
+	libpq \
+	zlib \
+	su-exec \
 && pip install --upgrade \
        lxml \
        pip \
@@ -26,8 +31,9 @@ RUN cd /synapse \
 && rm -rf \
        setup.cfg \
        setup.py \
-        synapse
-
+        synapse \
+ && apk del .build_deps
+ 
 VOLUME ["/data"]

 EXPOSE 8008/tcp 8448/tcp
@@ -88,6 +88,7 @@ variables are available for configuration:
 * ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN
  uris to enable TURN for this homeserver.
 * ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required.
+* ``SYNAPSE_MAX_UPLOAD_SIZE``, set this variable to change the max upload size [default `10M`].

 Shared secrets, that will be initialized to random values if not set:

@@ -85,7 +85,7 @@ federation_rc_concurrent: 3

 media_store_path: "/data/media"
 uploads_path: "/data/uploads"
-max_upload_size: "10M"
+max_upload_size: "{{ SYNAPSE_MAX_UPLOAD_SIZE or "10M" }}"
 max_image_pixels: "32M"
 dynamic_thumbnails: false

@@ -1,57 +0,0 @@
-#!/bin/bash
-
-## CAUTION:
-## This script will remove (hopefully) all trace of the given room ID from
-## your homeserver.db
-
-## Do not run it lightly.
-
-set -e
-
-if [ "$1" == "-h" ] || [ "$1" == "" ]; then
-  echo "Call with ROOM_ID as first option and then pipe it into the database. So for instance you might run"
-  echo " nuke-room-from-db.sh <room_id> | sqlite3 homeserver.db"
-  echo "or"
-  echo " nuke-room-from-db.sh <room_id> | psql --dbname=synapse"
-  exit
-fi
-
-ROOMID="$1"
-
-cat <<EOF
-DELETE FROM event_forward_extremities WHERE room_id = '$ROOMID';
-DELETE FROM event_backward_extremities WHERE room_id = '$ROOMID';
-DELETE FROM event_edges WHERE room_id = '$ROOMID';
-DELETE FROM room_depth WHERE room_id = '$ROOMID';
-DELETE FROM state_forward_extremities WHERE room_id = '$ROOMID';
-DELETE FROM events WHERE room_id = '$ROOMID';
-DELETE FROM event_json WHERE room_id = '$ROOMID';
-DELETE FROM state_events WHERE room_id = '$ROOMID';
-DELETE FROM current_state_events WHERE room_id = '$ROOMID';
-DELETE FROM room_memberships WHERE room_id = '$ROOMID';
-DELETE FROM feedback WHERE room_id = '$ROOMID';
-DELETE FROM topics WHERE room_id = '$ROOMID';
-DELETE FROM room_names WHERE room_id = '$ROOMID';
-DELETE FROM rooms WHERE room_id = '$ROOMID';
-DELETE FROM room_hosts WHERE room_id = '$ROOMID';
-DELETE FROM room_aliases WHERE room_id = '$ROOMID';
-DELETE FROM state_groups WHERE room_id = '$ROOMID';
-DELETE FROM state_groups_state WHERE room_id = '$ROOMID';
-DELETE FROM receipts_graph WHERE room_id = '$ROOMID';
-DELETE FROM receipts_linearized WHERE room_id = '$ROOMID';
-DELETE FROM event_search WHERE room_id = '$ROOMID'; 
-DELETE FROM guest_access WHERE room_id = '$ROOMID';
-DELETE FROM history_visibility WHERE room_id = '$ROOMID';
-DELETE FROM room_tags WHERE room_id = '$ROOMID';
-DELETE FROM room_tags_revisions WHERE room_id = '$ROOMID';
-DELETE FROM room_account_data WHERE room_id = '$ROOMID';
-DELETE FROM event_push_actions WHERE room_id = '$ROOMID';
-DELETE FROM local_invites WHERE room_id = '$ROOMID';
-DELETE FROM pusher_throttle WHERE room_id = '$ROOMID';
-DELETE FROM event_reports WHERE room_id = '$ROOMID';
-DELETE FROM public_room_list_stream WHERE room_id = '$ROOMID';
-DELETE FROM stream_ordering_to_exterm WHERE room_id = '$ROOMID';
-DELETE FROM event_auth WHERE room_id = '$ROOMID';
-DELETE FROM appservice_room_list WHERE room_id = '$ROOMID';
-VACUUM;
-EOF
@@ -17,13 +17,14 @@ ignore =
 [pep8]
 max-line-length = 90
 #  W503 requires that binary operators be at the end, not start, of lines. Erik
-#  doesn't like it.  E203 is contrary to PEP8.
-ignore = W503,E203
+#  doesn't like it.  E203 is contrary to PEP8.  E731 is silly.
+ignore = W503,E203,E731

 [flake8]
 # note that flake8 inherits the "ignore" settings from "pep8" (because it uses
 # pep8 to do those checks), but not the "max-line-length" setting
 max-line-length = 90
+ignore=W503,E203,E731

 [isort]
 line_length = 89
@@ -17,4 +17,14 @@
 """ This is a reference implementation of a Matrix home server.
 """

-__version__ = "0.33.3"
+try:
+    from twisted.internet import protocol
+    from twisted.internet.protocol import Factory
+    from twisted.names.dns import DNSDatagramProtocol
+    protocol.Factory.noisy = False
+    Factory.noisy = False
+    DNSDatagramProtocol.noisy = False
+except ImportError:
+    pass
+
+__version__ = "0.33.4"
@@ -251,6 +251,7 @@ class FilterCollection(object):
            "include_leave", False
        )
        self.event_fields = filter_json.get("event_fields", [])
+        self.event_format = filter_json.get("event_format", "client")

    def __repr__(self):
        return "<FilterCollection %s>" % (json.dumps(self._filter_json),)
@@ -307,6 +307,10 @@ class SynapseHomeServer(HomeServer):
 # Gauges to expose monthly active user control metrics
 current_mau_gauge = Gauge("synapse_admin_mau:current", "Current MAU")
 max_mau_gauge = Gauge("synapse_admin_mau:max", "MAU Limit")
+registered_reserved_users_mau_gauge = Gauge(
+    "synapse_admin_mau:registered_reserved_users",
+    "Registered users with reserved threepids"
+)


 def setup(config_options):
@@ -453,6 +457,10 @@ def run(hs):
        stats["homeserver"] = hs.config.server_name
        stats["timestamp"] = now
        stats["uptime_seconds"] = uptime
+        version = sys.version_info
+        stats["python_version"] = "{}.{}.{}".format(
+            version.major, version.minor, version.micro
+        )
        stats["total_users"] = yield hs.get_datastore().count_all_users()

        total_nonbridged_users = yield hs.get_datastore().count_nonbridged_users()
@@ -531,10 +539,14 @@ def run(hs):

    @defer.inlineCallbacks
    def generate_monthly_active_users():
-        count = 0
+        current_mau_count = 0
+        reserved_count = 0
+        store = hs.get_datastore()
        if hs.config.limit_usage_by_mau:
-            count = yield hs.get_datastore().get_monthly_active_count()
-        current_mau_gauge.set(float(count))
+            current_mau_count = yield store.get_monthly_active_count()
+            reserved_count = yield store.get_registered_reserved_users_count()
+        current_mau_gauge.set(float(current_mau_count))
+        registered_reserved_users_mau_gauge.set(float(reserved_count))
        max_mau_gauge.set(float(hs.config.max_mau_value))

    hs.get_datastore().initialise_reserved_users(
@@ -13,7 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import urllib
+
+from six.moves import urllib

 from prometheus_client import Counter

@@ -98,7 +99,7 @@ class ApplicationServiceApi(SimpleHttpClient):
    def query_user(self, service, user_id):
        if service.url is None:
            defer.returnValue(False)
-        uri = service.url + ("/users/%s" % urllib.quote(user_id))
+        uri = service.url + ("/users/%s" % urllib.parse.quote(user_id))
        response = None
        try:
            response = yield self.get_json(uri, {
@@ -119,7 +120,7 @@ class ApplicationServiceApi(SimpleHttpClient):
    def query_alias(self, service, alias):
        if service.url is None:
            defer.returnValue(False)
-        uri = service.url + ("/rooms/%s" % urllib.quote(alias))
+        uri = service.url + ("/rooms/%s" % urllib.parse.quote(alias))
        response = None
        try:
            response = yield self.get_json(uri, {
@@ -153,7 +154,7 @@ class ApplicationServiceApi(SimpleHttpClient):
            service.url,
            APP_SERVICE_PREFIX,
            kind,
-            urllib.quote(protocol)
+            urllib.parse.quote(protocol)
        )
        try:
            response = yield self.get_json(uri, fields)
@@ -188,7 +189,7 @@ class ApplicationServiceApi(SimpleHttpClient):
            uri = "%s%s/thirdparty/protocol/%s" % (
                service.url,
                APP_SERVICE_PREFIX,
-                urllib.quote(protocol)
+                urllib.parse.quote(protocol)
            )
            try:
                info = yield self.get_json(uri, {})
@@ -228,7 +229,7 @@ class ApplicationServiceApi(SimpleHttpClient):
        txn_id = str(txn_id)

        uri = service.url + ("/transactions/%s" %
-                             urllib.quote(txn_id))
+                             urllib.parse.quote(txn_id))
        try:
            yield self.put_json(
                uri=uri,
@@ -21,7 +21,7 @@ from .consent_config import ConsentConfig
 from .database import DatabaseConfig
 from .emailconfig import EmailConfig
 from .groups import GroupsConfig
-from .jwt import JWTConfig
+from .jwt_config import JWTConfig
 from .key import KeyConfig
 from .logger import LoggingConfig
 from .metrics import MetricsConfig
@@ -227,7 +227,22 @@ def setup_logging(config, use_worker_options=False):
    #
    # However this may not be too much of a problem if we are just writing to a file.
    observer = STDLibLogObserver()
+
+    def _log(event):
+
+        if "log_text" in event:
+            if event["log_text"].startswith("DNSDatagramProtocol starting on "):
+                return
+
+            if event["log_text"].startswith("(UDP Port "):
+                return
+
+            if event["log_text"].startswith("Timing out client"):
+                return
+
+        return observer(event)
+
    globalLogBeginner.beginLoggingTo(
-        [observer],
+        [_log],
        redirectStandardIO=not config.no_redirect_stdio,
    )
--- a/Show More
+++ b/Show More
				`@@ -1 +0,0 @@`
				`Removed the link to the unmaintained matrix-synapse-auto-deploy project from the readme.`
				`@@ -1 +0,0 @@`
				`Refactor state module to support multiple room versions`
				`@@ -0,0 +1 @@`
				`CircleCI tests now run on the potential merge of a PR.`
				`@@ -1 +0,0 @@`
				`Fix error collecting prometheus metrics when run on dedicated thread due to threading concurrency issues`
				`@@ -1 +0,0 @@`
				`Allow guests to use /rooms/:roomId/event/:eventId`
				`@@ -1 +0,0 @@`
				`The synapse.storage module has been ported to Python 3.`
				`@@ -1 +0,0 @@`
				`Split the state_group_cache into member and non-member state events (and so speed up LL /sync)`
				`@@ -1 +0,0 @@`
				`Log failure to authenticate remote servers as warnings (without stack traces)`
				`@@ -1 +0,0 @@`
				`The CONTRIBUTING guidelines have been updated to mention our use of Markdown and that .misc files have content.`
				`@@ -1 +0,0 @@`
				`Reference the need for an HTTP replication port when using the federation_reader worker`