Merge branch 'develop' of github.com:matrix-org/synapse into neilj/update_limits_error_codes

.circleci/config.yml

@@ -1,21 +1,5 @@
 version: 2
 jobs:
-  dockerhubuploadrelease:
-    machine: true
-    steps:
-      - checkout
-      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:$CIRCLE_TAG .
-      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      - run: docker push matrixdotorg/synapse:$CIRCLE_TAG
-  dockerhubuploadlatest:
-    machine: true
-    steps:
-      - checkout
-      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:$CIRCLE_SHA1 .
-      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      - run: docker tag matrixdotorg/synapse:$CIRCLE_SHA1 matrixdotorg/synapse:latest
-      - run: docker push matrixdotorg/synapse:$CIRCLE_SHA1
-      - run: docker push matrixdotorg/synapse:latest
   sytestpy2:
     machine: true
     steps:
@@ -25,8 +9,6 @@ jobs:
       - store_artifacts:
           path: ~/project/logs
           destination: logs
-      - store_test_results:
-          path: logs
   sytestpy2postgres:
     machine: true
     steps:
@@ -36,45 +18,15 @@ jobs:
       - store_artifacts:
           path: ~/project/logs
           destination: logs
-      - store_test_results:
-          path: logs
-  sytestpy2merged:
-    machine: true
-    steps:
-      - checkout
-      - run: bash .circleci/merge_base_branch.sh
-      - run: docker pull matrixdotorg/sytest-synapsepy2
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2
-      - store_artifacts:
-          path: ~/project/logs
-          destination: logs
-      - store_test_results:
-          path: logs
-
-  sytestpy2postgresmerged:
-    machine: true
-    steps:
-      - checkout
-      - run: bash .circleci/merge_base_branch.sh
-      - run: docker pull matrixdotorg/sytest-synapsepy2
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2
-      - store_artifacts:
-          path: ~/project/logs
-          destination: logs
-      - store_test_results:
-          path: logs
-
   sytestpy3:
     machine: true
     steps:
       - checkout
       - run: docker pull matrixdotorg/sytest-synapsepy3
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs hawkowl/sytestpy3
       - store_artifacts:
           path: ~/project/logs
           destination: logs
-      - store_test_results:
-          path: logs
   sytestpy3postgres:
     machine: true
     steps:
@@ -84,76 +36,13 @@ jobs:
       - store_artifacts:
           path: ~/project/logs
           destination: logs
-      - store_test_results:
-          path: logs
-  sytestpy3merged:
-    machine: true
-    steps:
-      - checkout
-      - run: bash .circleci/merge_base_branch.sh
-      - run: docker pull matrixdotorg/sytest-synapsepy3
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
-      - store_artifacts:
-          path: ~/project/logs
-          destination: logs
-      - store_test_results:
-          path: logs
-  sytestpy3postgresmerged:
-    machine: true
-    steps:
-      - checkout
-      - run: bash .circleci/merge_base_branch.sh
-      - run: docker pull matrixdotorg/sytest-synapsepy3
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3
-      - store_artifacts:
-          path: ~/project/logs
-          destination: logs
-      - store_test_results:
-          path: logs
 
 workflows:
   version: 2
   build:
     jobs:
-      - sytestpy2:
-          filters:
-            branches:
-              only: /develop|master|release-.*/
-      - sytestpy2postgres:
-          filters:
-            branches:
-              only: /develop|master|release-.*/
-      - sytestpy3:
-          filters:
-            branches:
-              only: /develop|master|release-.*/
-      - sytestpy3postgres:
-          filters:
-            branches:
-              only: /develop|master|release-.*/
-      - sytestpy2merged:
-          filters:
-            branches:
-              ignore: /develop|master|release-.*/
-      - sytestpy2postgresmerged:
-          filters:
-            branches:
-              ignore: /develop|master|release-.*/
-      - sytestpy3merged:
-          filters:
-            branches:
-              ignore: /develop|master|release-.*/
-      - sytestpy3postgresmerged:
-          filters:
-            branches:
-              ignore: /develop|master|release-.*/
-      - dockerhubuploadrelease:
-          filters:
-            tags:
-              only: /^v[0-9].[0-9]+.[0-9]+(.[0-9]+)?/
-            branches:
-              ignore: /.*/
-      - dockerhubuploadlatest:
-          filters:
-            branches:
-              only: master
+      - sytestpy2
+      - sytestpy2postgres
+# Currently broken while the Python 3 port is incomplete
+#      - sytestpy3
+#      - sytestpy3postgres

.circleci/merge_base_branch.sh

@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-# CircleCI doesn't give CIRCLE_PR_NUMBER in the environment for non-forked PRs. Wonderful.
-# In this case, we just need to do some ~shell magic~ to strip it out of the PULL_REQUEST URL.
-echo 'export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}"' >> $BASH_ENV
-source $BASH_ENV
-
-if [[ -z "${CIRCLE_PR_NUMBER}" ]]
-then
-    echo "Can't figure out what the PR number is!"
-    exit 1
-fi
-
-# Get the reference, using the GitHub API
-GITBASE=`curl -q https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
-
-# Show what we are before
-git show -s
-
-# Set up username so it can do a merge
-git config --global user.email bot@matrix.org
-git config --global user.name "A robot"
-
-# Fetch and merge. If it doesn't work, it will raise due to set -e.
-git fetch -u origin $GITBASE
-git merge --no-edit origin/$GITBASE
-
-# Show what we are after.
-git show -s

.dockerignore

@@ -3,5 +3,6 @@ Dockerfile
 .gitignore
 demo/etc
 tox.ini
+synctl
 .git/*
 .tox/*

.gitignore

@@ -1,11 +1,9 @@
 *.pyc
 .*.swp
 *~
-*.lock
 
 .DS_Store
 _trial_temp/
-_trial_temp*/
 logs/
 dbs/
 *.egg
@@ -46,7 +44,6 @@ media_store/
 build/
 venv/
 venv*/
-*venv/
 
 localhost-800*/
 static/client/register/register_config.js

.travis.yml

@@ -8,6 +8,9 @@ before_script:
   - git remote set-branches --add origin develop
   - git fetch origin develop
 
+services:
+  - postgresql
+
 matrix:
   fast_finish: true
   include:
@@ -20,31 +23,22 @@ matrix:
   - python: 2.7
     env: TOX_ENV=py27
 
-  - python: 2.7
-    env: TOX_ENV=py27-old
-
   - python: 2.7
     env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
-    services:
-      - postgresql
-
-  - python: 3.5
-    env: TOX_ENV=py35
 
   - python: 3.6
     env: TOX_ENV=py36
 
-  - python: 3.6
-    env: TOX_ENV=py36-postgres TRIAL_FLAGS="-j 4"
-    services:
-      - postgresql
-
   - python: 3.6
     env: TOX_ENV=check_isort
 
   - python: 3.6
     env: TOX_ENV=check-newsfragment
 
+  allow_failures:
+  - python: 2.7
+    env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
+
 install:
   - pip install tox
 

CHANGES.md

@@ -1,233 +1,3 @@
-Synapse 0.33.5.1 (2018-09-25)
-=============================
-
-Internal Changes
-----------------
-
-- Fix incompatibility with older Twisted version in tests. Thanks 
-  @OlegGirko! ([\#3940](https://github.com/matrix-org/synapse/issues/3940))
-
-
-Synapse 0.33.5 (2018-09-24)
-===========================
-
-No significant changes.
-
-
-Synapse 0.33.5rc1 (2018-09-17)
-==============================
-
-Features
---------
-
-- Python 3.5 and 3.6 support is now in beta. ([\#3576](https://github.com/matrix-org/synapse/issues/3576))
-- Implement `event_format` filter param in `/sync` ([\#3790](https://github.com/matrix-org/synapse/issues/3790))
-- Add synapse_admin_mau:registered_reserved_users metric to expose number of real reaserved users ([\#3846](https://github.com/matrix-org/synapse/issues/3846))
-
-
-Bugfixes
---------
-
-- Remove connection ID for replication prometheus metrics, as it creates a large number of new series. ([\#3788](https://github.com/matrix-org/synapse/issues/3788))
-- guest users should not be part of mau total ([\#3800](https://github.com/matrix-org/synapse/issues/3800))
-- Bump dependency on pyopenssl 16.x, to avoid incompatibility with recent Twisted. ([\#3804](https://github.com/matrix-org/synapse/issues/3804))
-- Fix existing room tags not coming down sync when joining a room ([\#3810](https://github.com/matrix-org/synapse/issues/3810))
-- Fix jwt import check ([\#3824](https://github.com/matrix-org/synapse/issues/3824))
-- fix VOIP crashes under Python 3 (#3821) ([\#3835](https://github.com/matrix-org/synapse/issues/3835))
-- Fix manhole so that it works with latest openssh clients ([\#3841](https://github.com/matrix-org/synapse/issues/3841))
-- Fix outbound requests occasionally wedging, which can result in federation breaking between servers. ([\#3845](https://github.com/matrix-org/synapse/issues/3845))
-- Show heroes if room name/canonical alias has been deleted ([\#3851](https://github.com/matrix-org/synapse/issues/3851))
-- Fix handling of redacted events from federation ([\#3859](https://github.com/matrix-org/synapse/issues/3859))
--  ([\#3874](https://github.com/matrix-org/synapse/issues/3874))
-- Mitigate outbound federation randomly becoming wedged ([\#3875](https://github.com/matrix-org/synapse/issues/3875))
-
-
-Internal Changes
-----------------
-
-- CircleCI tests now run on the potential merge of a PR. ([\#3704](https://github.com/matrix-org/synapse/issues/3704))
-- http/ is now ported to Python 3. ([\#3771](https://github.com/matrix-org/synapse/issues/3771))
-- Improve human readable error messages for threepid registration/account update ([\#3789](https://github.com/matrix-org/synapse/issues/3789))
-- Make /sync slightly faster by avoiding needless copies ([\#3795](https://github.com/matrix-org/synapse/issues/3795))
-- handlers/ is now ported to Python 3. ([\#3803](https://github.com/matrix-org/synapse/issues/3803))
-- Limit the number of PDUs/EDUs per federation transaction ([\#3805](https://github.com/matrix-org/synapse/issues/3805))
-- Only start postgres instance for postgres tests on Travis CI ([\#3806](https://github.com/matrix-org/synapse/issues/3806))
-- tests/ is now ported to Python 3. ([\#3808](https://github.com/matrix-org/synapse/issues/3808))
-- crypto/ is now ported to Python 3. ([\#3822](https://github.com/matrix-org/synapse/issues/3822))
-- rest/ is now ported to Python 3. ([\#3823](https://github.com/matrix-org/synapse/issues/3823))
-- add some logging for the keyring queue ([\#3826](https://github.com/matrix-org/synapse/issues/3826))
-- speed up lazy loading by 2-3x ([\#3827](https://github.com/matrix-org/synapse/issues/3827))
-- Improved Dockerfile to remove build requirements after building reducing the image size. ([\#3834](https://github.com/matrix-org/synapse/issues/3834))
-- Disable lazy loading for incremental syncs for now ([\#3840](https://github.com/matrix-org/synapse/issues/3840))
-- federation/ is now ported to Python 3. ([\#3847](https://github.com/matrix-org/synapse/issues/3847))
-- Log when we retry outbound requests ([\#3853](https://github.com/matrix-org/synapse/issues/3853))
-- Removed some excess logging messages. ([\#3855](https://github.com/matrix-org/synapse/issues/3855))
-- Speed up purge history for rooms that have been previously purged ([\#3856](https://github.com/matrix-org/synapse/issues/3856))
-- Refactor some HTTP timeout code. ([\#3857](https://github.com/matrix-org/synapse/issues/3857))
-- Fix running merged builds on CircleCI ([\#3858](https://github.com/matrix-org/synapse/issues/3858))
-- Fix typo in replication stream exception. ([\#3860](https://github.com/matrix-org/synapse/issues/3860))
-- Add in flight real time metrics for Measure blocks ([\#3871](https://github.com/matrix-org/synapse/issues/3871))
-- Disable buffering and automatic retrying in treq requests to prevent timeouts. ([\#3872](https://github.com/matrix-org/synapse/issues/3872))
-- mention jemalloc in the README ([\#3877](https://github.com/matrix-org/synapse/issues/3877))
-- Remove unmaintained "nuke-room-from-db.sh" script ([\#3888](https://github.com/matrix-org/synapse/issues/3888))
-
-
-Synapse 0.33.4 (2018-09-07)
-===========================
-
-Internal Changes
-----------------
-
-- Unignore synctl in .dockerignore to fix docker builds ([\#3802](https://github.com/matrix-org/synapse/issues/3802))
-
-
-Synapse 0.33.4rc2 (2018-09-06)
-==============================
-
-Pull in security fixes from v0.33.3.1
-
-
-Synapse 0.33.3.1 (2018-09-06)
-=============================
-
-SECURITY FIXES
---------------
-
-- Fix an issue where event signatures were not always correctly validated ([\#3796](https://github.com/matrix-org/synapse/issues/3796))
-- Fix an issue where server_acls could be circumvented for incoming events ([\#3796](https://github.com/matrix-org/synapse/issues/3796))
-
-
-Internal Changes
-----------------
-
-- Unignore synctl in .dockerignore to fix docker builds ([\#3802](https://github.com/matrix-org/synapse/issues/3802))
-
-
-Synapse 0.33.4rc1 (2018-09-04)
-==============================
-
-Features
---------
-
-- Support profile API endpoints on workers ([\#3659](https://github.com/matrix-org/synapse/issues/3659))
-- Server notices for resource limit blocking ([\#3680](https://github.com/matrix-org/synapse/issues/3680))
-- Allow guests to use /rooms/:roomId/event/:eventId ([\#3724](https://github.com/matrix-org/synapse/issues/3724))
-- Add mau_trial_days config param, so that users only get counted as MAU after N days. ([\#3749](https://github.com/matrix-org/synapse/issues/3749))
-- Require twisted 17.1 or later (fixes [#3741](https://github.com/matrix-org/synapse/issues/3741)). ([\#3751](https://github.com/matrix-org/synapse/issues/3751))
-
-
-Bugfixes
---------
-
-- Fix error collecting prometheus metrics when run on dedicated thread due to threading concurrency issues ([\#3722](https://github.com/matrix-org/synapse/issues/3722))
-- Fix bug where we resent "limit exceeded" server notices repeatedly ([\#3747](https://github.com/matrix-org/synapse/issues/3747))
-- Fix bug where we broke sync when using limit_usage_by_mau but hadn't configured server notices ([\#3753](https://github.com/matrix-org/synapse/issues/3753))
-- Fix 'federation_domain_whitelist' such that an empty list correctly blocks all outbound federation traffic ([\#3754](https://github.com/matrix-org/synapse/issues/3754))
-- Fix tagging of server notice rooms ([\#3755](https://github.com/matrix-org/synapse/issues/3755), [\#3756](https://github.com/matrix-org/synapse/issues/3756))
-- Fix 'admin_uri' config variable and error parameter to be 'admin_contact' to match the spec. ([\#3758](https://github.com/matrix-org/synapse/issues/3758))
-- Don't return non-LL-member state in incremental sync state blocks ([\#3760](https://github.com/matrix-org/synapse/issues/3760))
-- Fix bug in sending presence over federation ([\#3768](https://github.com/matrix-org/synapse/issues/3768))
-- Fix bug where preserved threepid user comes to sign up and server is mau blocked ([\#3777](https://github.com/matrix-org/synapse/issues/3777))
-
-Internal Changes
-----------------
-
-- Removed the link to the unmaintained matrix-synapse-auto-deploy project from the readme. ([\#3378](https://github.com/matrix-org/synapse/issues/3378))
-- Refactor state module to support multiple room versions ([\#3673](https://github.com/matrix-org/synapse/issues/3673))
-- The synapse.storage module has been ported to Python 3. ([\#3725](https://github.com/matrix-org/synapse/issues/3725))
-- Split the state_group_cache into member and non-member state events (and so speed up LL /sync) ([\#3726](https://github.com/matrix-org/synapse/issues/3726))
-- Log failure to authenticate remote servers as warnings (without stack traces) ([\#3727](https://github.com/matrix-org/synapse/issues/3727))
-- The CONTRIBUTING guidelines have been updated to mention our use of Markdown and that .misc files have content. ([\#3730](https://github.com/matrix-org/synapse/issues/3730))
-- Reference the need for an HTTP replication port when using the federation_reader worker ([\#3734](https://github.com/matrix-org/synapse/issues/3734))
-- Fix minor spelling error in federation client documentation. ([\#3735](https://github.com/matrix-org/synapse/issues/3735))
-- Remove redundant state resolution function ([\#3737](https://github.com/matrix-org/synapse/issues/3737))
-- The test suite now passes on PostgreSQL. ([\#3740](https://github.com/matrix-org/synapse/issues/3740))
-- Fix MAU cache invalidation due to missing yield ([\#3746](https://github.com/matrix-org/synapse/issues/3746))
-- Make sure that we close db connections opened during init ([\#3764](https://github.com/matrix-org/synapse/issues/3764))
-
-
-Synapse 0.33.3 (2018-08-22)
-===========================
-
-Bugfixes
---------
-
-- Fix bug introduced in v0.33.3rc1 which made the ToS give a 500 error ([\#3732](https://github.com/matrix-org/synapse/issues/3732))
-
-
-Synapse 0.33.3rc2 (2018-08-21)
-==============================
-
-Bugfixes
---------
-
-- Fix bug in v0.33.3rc1 which caused infinite loops and OOMs ([\#3723](https://github.com/matrix-org/synapse/issues/3723))
-
-
-Synapse 0.33.3rc1 (2018-08-21)
-==============================
-
-Features
---------
-
-- Add support for the SNI extension to federation TLS connections. Thanks to @vojeroen! ([\#3439](https://github.com/matrix-org/synapse/issues/3439))
-- Add /_media/r0/config ([\#3184](https://github.com/matrix-org/synapse/issues/3184))
-- speed up /members API and add `at` and `membership` params as per MSC1227 ([\#3568](https://github.com/matrix-org/synapse/issues/3568))
-- implement `summary` block in /sync response as per MSC688 ([\#3574](https://github.com/matrix-org/synapse/issues/3574))
-- Add lazy-loading support to /messages as per MSC1227 ([\#3589](https://github.com/matrix-org/synapse/issues/3589))
-- Add ability to limit number of monthly active users on the server ([\#3633](https://github.com/matrix-org/synapse/issues/3633))
-- Support more federation endpoints on workers ([\#3653](https://github.com/matrix-org/synapse/issues/3653))
-- Basic support for room versioning ([\#3654](https://github.com/matrix-org/synapse/issues/3654))
-- Ability to disable client/server Synapse via conf toggle ([\#3655](https://github.com/matrix-org/synapse/issues/3655))
-- Ability to whitelist specific threepids against monthly active user limiting ([\#3662](https://github.com/matrix-org/synapse/issues/3662))
-- Add some metrics for the appservice and federation event sending loops ([\#3664](https://github.com/matrix-org/synapse/issues/3664))
-- Where server is disabled, block ability for locked out users to read new messages ([\#3670](https://github.com/matrix-org/synapse/issues/3670))
-- set admin uri via config, to be used in error messages where the user should contact the administrator ([\#3687](https://github.com/matrix-org/synapse/issues/3687))
-- Synapse's presence functionality can now be disabled with the "use_presence" configuration option. ([\#3694](https://github.com/matrix-org/synapse/issues/3694))
-- For resource limit blocked users, prevent writing into rooms ([\#3708](https://github.com/matrix-org/synapse/issues/3708))
-
-
-Bugfixes
---------
-
-- Fix occasional glitches in the synapse_event_persisted_position metric ([\#3658](https://github.com/matrix-org/synapse/issues/3658))
-- Fix bug on deleting 3pid when using identity servers that don't support unbind API ([\#3661](https://github.com/matrix-org/synapse/issues/3661))
-- Make the tests pass on Twisted < 18.7.0 ([\#3676](https://github.com/matrix-org/synapse/issues/3676))
-- Don’t ship recaptcha_ajax.js, use it directly from Google ([\#3677](https://github.com/matrix-org/synapse/issues/3677))
-- Fixes test_reap_monthly_active_users so it passes under postgres ([\#3681](https://github.com/matrix-org/synapse/issues/3681))
-- Fix mau blocking calulation bug on login ([\#3689](https://github.com/matrix-org/synapse/issues/3689))
-- Fix missing yield in synapse.storage.monthly_active_users.initialise_reserved_users ([\#3692](https://github.com/matrix-org/synapse/issues/3692))
-- Improve HTTP request logging to include all requests ([\#3700](https://github.com/matrix-org/synapse/issues/3700))
-- Avoid timing out requests while we are streaming back the response ([\#3701](https://github.com/matrix-org/synapse/issues/3701))
-- Support more federation endpoints on workers ([\#3705](https://github.com/matrix-org/synapse/issues/3705), [\#3713](https://github.com/matrix-org/synapse/issues/3713))
-- Fix "Starting db txn 'get_all_updated_receipts' from sentinel context" warning ([\#3710](https://github.com/matrix-org/synapse/issues/3710))
-- Fix bug where `state_cache` cache factor ignored environment variables ([\#3719](https://github.com/matrix-org/synapse/issues/3719))
-
-
-Deprecations and Removals
--------------------------
-
-- The Shared-Secret registration method of the legacy v1/register REST endpoint has been removed. For a replacement, please see [the admin/register API documentation](https://github.com/matrix-org/synapse/blob/master/docs/admin_api/register_api.rst). ([\#3703](https://github.com/matrix-org/synapse/issues/3703))
-
-
-Internal Changes
-----------------
-
-- The test suite now can run under PostgreSQL. ([\#3423](https://github.com/matrix-org/synapse/issues/3423))
-- Refactor HTTP replication endpoints to reduce code duplication ([\#3632](https://github.com/matrix-org/synapse/issues/3632))
-- Tests now correctly execute on Python 3. ([\#3647](https://github.com/matrix-org/synapse/issues/3647))
-- Sytests can now be run inside a Docker container. ([\#3660](https://github.com/matrix-org/synapse/issues/3660))
-- Port over enough to Python 3 to allow the sytests to start. ([\#3668](https://github.com/matrix-org/synapse/issues/3668))
-- Update docker base image from alpine 3.7 to 3.8. ([\#3669](https://github.com/matrix-org/synapse/issues/3669))
-- Rename synapse.util.async to synapse.util.async_helpers to mitigate async becoming a keyword on Python 3.7. ([\#3678](https://github.com/matrix-org/synapse/issues/3678))
-- Synapse's tests are now formatted with the black autoformatter. ([\#3679](https://github.com/matrix-org/synapse/issues/3679))
-- Implemented a new testing base class to reduce test boilerplate. ([\#3684](https://github.com/matrix-org/synapse/issues/3684))
-- Rename MAU prometheus metrics ([\#3690](https://github.com/matrix-org/synapse/issues/3690))
-- add new error type ResourceLimit ([\#3707](https://github.com/matrix-org/synapse/issues/3707))
-- Logcontexts for replication command handlers ([\#3709](https://github.com/matrix-org/synapse/issues/3709))
-- Update admin register API documentation to reference a real user ID. ([\#3712](https://github.com/matrix-org/synapse/issues/3712))
-
-
 Synapse 0.33.2 (2018-08-09)
 ===========================
 
@@ -254,7 +24,7 @@ Features
 Bugfixes
 --------
 
-- Make /directory/list API return 404 for room not found instead of 400. Thanks to @fuzzmz! ([\#3620](https://github.com/matrix-org/synapse/issues/3620))
+- Make /directory/list API return 404 for room not found instead of 400 ([\#2952](https://github.com/matrix-org/synapse/issues/2952))
 - Default inviter_display_name to mxid for email invites ([\#3391](https://github.com/matrix-org/synapse/issues/3391))
 - Don't generate TURN credentials if no TURN config options are set ([\#3514](https://github.com/matrix-org/synapse/issues/3514))
 - Correctly announce deleted devices over federation ([\#3520](https://github.com/matrix-org/synapse/issues/3520))

CONTRIBUTING.rst

@@ -30,28 +30,12 @@ use github's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Travis CI 
-<https://travis-ci.org/matrix-org/synapse>`_ for continuous integration. All
-pull requests to synapse get automatically tested by Travis and CircleCI.
-If your change breaks the build, this will be shown in GitHub, so please
-keep an eye on the pull request for feedback.
-
-To run unit tests in a local development environment, you can use:
-
-- ``tox -e py27`` (requires tox to be installed by ``pip install tox``) for
-  SQLite-backed Synapse on Python 2.7.
-- ``tox -e py35`` for SQLite-backed Synapse on Python 3.5.
-- ``tox -e py36`` for SQLite-backed Synapse on Python 3.6.
-- ``tox -e py27-postgres`` for PostgreSQL-backed Synapse on Python 2.7
-  (requires a running local PostgreSQL with access to create databases).
-- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 2.7
-  (requires Docker). Entirely self-contained, recommended if you don't want to
-  set up PostgreSQL yourself.
-
-Docker images are available for running the integration tests (SyTest) locally,
-see the `documentation in the SyTest repo
-<https://github.com/matrix-org/sytest/blob/develop/docker/README.md>`_ for more
-information.
+We use `Jenkins <http://matrix.org/jenkins>`_ and 
+`Travis <https://travis-ci.org/matrix-org/synapse>`_ for continuous
+integration. All pull requests to synapse get automatically tested by Travis; 
+the Jenkins builds require an adminstrator to start them. If your change 
+breaks the build, this will be shown in github, so please keep an eye on the 
+pull request for feedback.
 
 Code style
 ~~~~~~~~~~
@@ -72,18 +56,17 @@ entry. These are managed by Towncrier
 (https://github.com/hawkowl/towncrier).
 
 To create a changelog entry, make a new file in the ``changelog.d``
-file named in the format of ``PRnumber.type``. The type can be
+file named in the format of ``issuenumberOrPR.type``. The type can be
 one of ``feature``, ``bugfix``, ``removal`` (also used for
 deprecations), or ``misc`` (for internal-only changes). The content of
-the file is your changelog entry, which can contain Markdown
-formatting. Adding credits to the changelog is encouraged, we value
-your contributions and would like to have you shouted out in the
-release notes!
+the file is your changelog entry, which can contain RestructuredText
+formatting. A note of contributors is welcomed in changelogs for
+non-misc changes (the content of misc changes is not displayed).
 
-For example, a fix in PR #1234 would have its changelog entry in
-``changelog.d/1234.bugfix``, and contain content like "The security levels of
-Florbs are now validated when recieved over federation. Contributed by Jane
-Matrix".
+For example, a fix for a bug reported in #1234 would have its
+changelog entry in ``changelog.d/1234.bugfix``, and contain content
+like "The security levels of Florbs are now validated when
+recieved over federation. Contributed by Jane Matrix".
 
 Attribution
 ~~~~~~~~~~~
@@ -93,8 +76,7 @@ AUTHORS.rst file for the project in question. Please feel free to include a
 change to AUTHORS.rst in your pull request to list yourself and a short
 description of the area(s) you've worked on. Also, we sometimes have swag to
 give away to contributors - if you feel that Matrix-branded apparel is missing
-from your life, please mail us your shipping address to matrix at matrix.org and
-we'll try to fix it :)
+from your life, please mail us your shipping address to matrix at matrix.org and we'll try to fix it :)
 
 Sign off
 ~~~~~~~~
@@ -143,7 +125,7 @@ the contribution or otherwise have the right to contribute it to Matrix::
         personal information I submit with it, including my sign-off) is
         maintained indefinitely and may be redistributed consistent with
         this project or the open source license(s) involved.
-
+        
 If you agree to this for your contribution, then all that's needed is to
 include the line in your commit or pull request comment::
 
@@ -161,9 +143,4 @@ flag to ``git commit``, which uses the name and email set in your
 Conclusion
 ~~~~~~~~~~
 
-That's it!  Matrix is a very open and collaborative project as you might expect
-given our obsession with open communication.  If we're going to successfully
-matrix together all the fragmented communication technologies out there we are
-reliant on contributions and collaboration from the community to do so.  So
-please get involved - and we hope you have as much fun hacking on Matrix as we
-do!
+That's it!  Matrix is a very open and collaborative project as you might expect given our obsession with open communication.  If we're going to successfully matrix together all the fragmented communication technologies out there we are reliant on contributions and collaboration from the community to do so.  So please get involved - and we hope you have as much fun hacking on Matrix as we do!

MANIFEST.in

@@ -28,7 +28,6 @@ exclude jenkins*.sh
 exclude jenkins*
 exclude Dockerfile
 exclude .dockerignore
-exclude test_postgresql.sh
 recursive-exclude jenkins *.sh
 
 include pyproject.toml

README.rst

@@ -157,7 +157,7 @@ if you prefer.
 
 In case of problems, please see the _`Troubleshooting` section below.
 
-There is an offical synapse image available at
+There is an offical synapse image available at 
 https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with
 the docker-compose file available at `contrib/docker <contrib/docker>`_. Further information on
 this including configuration options is available in the README on
@@ -167,6 +167,11 @@ Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
 Dockerfile to automate a synapse server in a single Docker image, at
 https://hub.docker.com/r/avhost/docker-matrix/tags/
 
+Also, Martin Giess has created an auto-deployment process with vagrant/ansible,
+tested with VirtualBox/AWS/DigitalOcean - see 
+https://github.com/EMnify/matrix-synapse-auto-deploy
+for details.
+
 Configuring synapse
 -------------------
 
@@ -459,13 +464,37 @@ https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-
 
 Windows Install
 ---------------
+Synapse can be installed on Cygwin. It requires the following Cygwin packages:
+
+- gcc
+- git
+- libffi-devel
+- openssl (and openssl-devel, python-openssl)
+- python
+- python-setuptools
+
+The content repository requires additional packages and will be unable to process
+uploads without them:
+
+- libjpeg8
+- libjpeg8-devel
+- zlib
+
+If you choose to install Synapse without these packages, you will need to reinstall
+``pillow`` for changes to be applied, e.g. ``pip uninstall pillow`` ``pip install
+pillow --user``
+
+Troubleshooting:
+
+- You may need to upgrade ``setuptools`` to get this to work correctly:
+  ``pip install setuptools --upgrade``.
+- You may encounter errors indicating that ``ffi.h`` is missing, even with
+  ``libffi-devel`` installed. If you do, copy the ``.h`` files:
+  ``cp /usr/lib/libffi-3.0.13/include/*.h /usr/include``
+- You may need to install libsodium from source in order to install PyNacl. If
+  you do, you may need to create a symlink to ``libsodium.a`` so ``ld`` can find
+  it: ``ln -s /usr/local/lib/libsodium.a /usr/lib/libsodium.a``
 
-If you wish to run or develop Synapse on Windows, the Windows Subsystem For
-Linux provides a Linux environment on Windows 10 which is capable of using the
-Debian, Fedora, or source installation methods. More information about WSL can
-be found at https://docs.microsoft.com/en-us/windows/wsl/install-win10 for
-Windows 10 and https://docs.microsoft.com/en-us/windows/wsl/install-on-server
-for Windows Server.
 
 Troubleshooting
 ===============
@@ -718,18 +747,6 @@ so an example nginx configuration might look like::
       }
   }
 
-and an example apache configuration may look like::
-
-    <VirtualHost *:443>
-        SSLEngine on
-        ServerName matrix.example.com;
-
-        <Location /_matrix>
-            ProxyPass http://127.0.0.1:8008/_matrix nocanon
-            ProxyPassReverse http://127.0.0.1:8008/_matrix
-        </Location>
-    </VirtualHost>
-
 You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true``
 for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are
 recorded correctly.
@@ -884,7 +901,7 @@ to install using pip and a virtualenv::
 
     virtualenv -p python2.7 env
     source env/bin/activate
-    python -m synapse.python_dependencies | xargs pip install
+    python synapse/python_dependencies.py | xargs pip install
     pip install lxml mock
 
 This will run a process of downloading and installing all the needed
@@ -939,13 +956,5 @@ variable.  The default is 0.5, which can be decreased to reduce RAM usage
 in memory constrained enviroments, or increased if performance starts to
 degrade.
 
-Using `libjemalloc <http://jemalloc.net/>`_ can also yield a significant
-improvement in overall amount, and especially in terms of giving back RAM
-to the OS. To use it, the library must simply be put in the LD_PRELOAD
-environment variable when launching Synapse. On Debian, this can be done
-by installing the ``libjemalloc1`` package and adding this line to
-``/etc/default/matrix-synapse``::
-
-    LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
 
 .. _`key_management`: https://matrix.org/docs/spec/server_server/unstable.html#retrieving-server-keys

changelog.d/1491.feature

@@ -0,0 +1 @@
+Add support for the SNI extension to federation TLS connections

changelog.d/3423.misc

@@ -0,0 +1 @@
+The test suite now can run under PostgreSQL.

changelog.d/3578.bugfix

@@ -1 +0,0 @@
-Fix problem when playing media from Chrome using direct URL (thanks @remjey!)

changelog.d/3632.misc

@@ -0,0 +1 @@
+Refactor HTTP replication endpoints to reduce code duplication

changelog.d/3633.feature

@@ -0,0 +1 @@
+Add ability to limit number of monthly active users on the server

changelog.d/3647.misc

@@ -0,0 +1 @@
+Tests now correctly execute on Python 3.

changelog.d/3653.feature

@@ -0,0 +1 @@
+Support more federation endpoints on workers

changelog.d/3654.feature

@@ -0,0 +1 @@
+Basic support for room versioning

changelog.d/3655.feature

@@ -0,0 +1 @@
+Ability to disable client/server Synapse via conf toggle

changelog.d/3658.bugfix

@@ -0,0 +1 @@
+Fix occasional glitches in the synapse_event_persisted_position metric

changelog.d/3660.misc

@@ -0,0 +1 @@
+Sytests can now be run inside a Docker container.

changelog.d/3661.bugfix

@@ -0,0 +1 @@
+Fix bug on deleting 3pid when using identity servers that don't support unbind API

changelog.d/3662.feature

@@ -0,0 +1 @@
+Ability to whitelist specific threepids against monthly active user limiting

changelog.d/3664.feature

@@ -0,0 +1 @@
+Add some metrics for the appservice and federation event sending loops

changelog.d/3669.misc

@@ -0,0 +1 @@
+Update docker base image from alpine 3.7 to 3.8.

changelog.d/3670.feature

@@ -0,0 +1 @@
+Where server is disabled, block ability for locked out users to read new messages

changelog.d/3676.bugfix

@@ -0,0 +1 @@
+Make the tests pass on Twisted < 18.7.0

changelog.d/3677.bugfix

@@ -0,0 +1 @@
+Don’t ship recaptcha_ajax.js, use it directly from Google

changelog.d/3678.misc

@@ -0,0 +1 @@
+Rename synapse.util.async to synapse.util.async_helpers to mitigate async becoming a keyword on Python 3.7.

changelog.d/3679.misc

@@ -0,0 +1 @@
+Synapse's tests are now formatted with the black autoformatter.

changelog.d/3681.bugfix

@@ -0,0 +1 @@
+Fixes test_reap_monthly_active_users so it passes under postgres

changelog.d/3684.misc

@@ -0,0 +1 @@
+Implemented a new testing base class to reduce test boilerplate.

changelog.d/3687.feature

@@ -0,0 +1 @@
+set admin uri via config, to be used in error messages where the user should contact the administrator

changelog.d/3690.misc

@@ -0,0 +1 @@
+Rename MAU prometheus metrics

changelog.d/3692.bugfix

@@ -0,0 +1 @@
+Fix missing yield in synapse.storage.monthly_active_users.initialise_reserved_users

changelog.d/3699.misc

@@ -1,2 +0,0 @@
-Unit tests can now be run under PostgreSQL in Docker using 
-``test_postgresql.sh``.

changelog.d/3794.misc

@@ -1 +0,0 @@
-Speed up calculation of typing updates for replication

changelog.d/3836.bugfix

@@ -1 +0,0 @@
-support registering regular users non-interactively with register_new_matrix_user script

changelog.d/3868.bugfix

@@ -1 +0,0 @@
-Fix broken invite email links for self hosted riots 

changelog.d/3873.misc

@@ -1,2 +0,0 @@
-Remove documentation regarding installation on Cygwin, the use of WSL is 
-recommended instead.

changelog.d/3879.bugfix

@@ -1 +0,0 @@
-Don't ratelimit autojoins

changelog.d/3883.feature

@@ -1 +0,0 @@
-Adding the ability to change MAX_UPLOAD_SIZE for the docker container variables.

changelog.d/3889.bugfix

@@ -1 +0,0 @@
-Fix 500 error when deleting unknown room alias

changelog.d/3892.bugfix

@@ -1 +0,0 @@
-Fix some b'abcd' noise in logs and metrics

changelog.d/3894.feature

@@ -1 +0,0 @@
-Report "python_version" in the phone home stats

changelog.d/3895.bugfix

@@ -1 +0,0 @@
-Fix some b'abcd' noise in logs and metrics

changelog.d/3897.misc

@@ -1 +0,0 @@
-Fix typo in README, synaspse -> synapse

changelog.d/3899.bugfix

@@ -1 +0,0 @@
-When we join a room, always try the server we used for the alias lookup first, to avoid unresponsive and out-of-date servers.

changelog.d/3903.misc

@@ -1 +0,0 @@
-Increase the timeout when filling missing events in federation requests

changelog.d/3904.misc

@@ -1 +0,0 @@
-Improve the logging when handling a federation transaction

changelog.d/3906.misc

@@ -1 +0,0 @@
-Improve logging of outbound federation requests

changelog.d/3907.bugfix

@@ -1 +0,0 @@
-Fix incorrect server-name indication for outgoing federation requests

changelog.d/3908.bugfix

@@ -1 +0,0 @@
-Fix adding client IPs to the database failing on Python 3.

changelog.d/3909.misc

@@ -1 +0,0 @@
-Improve logging of outbound federation requests

changelog.d/3910.bugfix

@@ -1 +0,0 @@
-Fix bug where things occaisonally were not being timed out correctly.

changelog.d/3911.misc

@@ -1 +0,0 @@
-Fix the docker image building on python 3

changelog.d/3912.misc

@@ -1 +0,0 @@
-Add a regression test for logging failed HTTP requests on Python 3.

changelog.d/3914.bugfix

@@ -1 +0,0 @@
-Fix bug where outbound federation would stop talking to some servers when using workers

changelog.d/3916.feature

@@ -1 +0,0 @@
-Always LL ourselves if we're in a room

changelog.d/3924.misc

@@ -1 +0,0 @@
-Comments and interface cleanup for on_receive_pdu

changelog.d/3925.misc

@@ -1 +0,0 @@
-Fix spurious exceptions when remote http client closes conncetion

changelog.d/3927.misc

@@ -1 +0,0 @@
-Log exceptions thrown by background tasks

changelog.d/3932.bugfix

@@ -1 +0,0 @@
-Fix some instances of ExpiringCache not expiring cache items

changelog.d/3936.bugfix

@@ -1 +0,0 @@
-Fix out-of-bounds error when LLing yourself

changelog.d/3946.misc

@@ -1 +0,0 @@
-Automate pushes to docker hub

changelog.d/3947.misc

@@ -1 +0,0 @@
-Require attrs 16.0.0 or later

changelog.d/3948.misc

@@ -1 +0,0 @@
-Fix incompatibility with python3 on alpine

changelog.d/3952.misc

@@ -1 +0,0 @@
-Run the test suite on the oldest supported versions of our dependencies in CI.

changelog.d/3956.bugfix

@@ -1 +0,0 @@
-Fix exceptions from metrics handler

changelog.d/3957.misc

@@ -1 +0,0 @@
-CircleCI now only runs merged jobs on PRs, and commit jobs on develop, master, and release branches.

changelog.d/3958.misc

@@ -1 +0,0 @@
-Fix docstrings and add tests for state store methods

changelog.d/3959.feature

@@ -1 +0,0 @@
-Include eventid in log lines when processing incoming federation transactions

changelog.d/3961.bugfix

@@ -1 +0,0 @@
-Fix errors due to concurrent monthly_active_user upserts 

changelog.d/3963.misc

@@ -1 +0,0 @@
-fix docstring for FederationClient.get_state_for_room

changelog.d/3965.misc

@@ -1 +0,0 @@
-Run notify_app_services as a bg process

changelog.d/3966.misc

@@ -1 +0,0 @@
-Improve the logging when handling a federation transaction

changelog.d/3967.misc

@@ -1 +0,0 @@
-Clarifications in FederationHandler

changelog.d/3970.bugfix

@@ -1 +0,0 @@
-Replaced all occurences of e.message with str(e). Contributed by Schnuffle

docker/Dockerfile

@@ -1,9 +1,6 @@
-ARG PYTHON_VERSION=2
-FROM docker.io/python:${PYTHON_VERSION}-alpine3.8
+FROM docker.io/python:2-alpine3.8
 
-COPY . /synapse
-
-RUN apk add --no-cache --virtual .build_deps \
+RUN apk add --no-cache --virtual .nacl_deps \
         build-base \
         libffi-dev \
         libjpeg-turbo-dev \
@@ -11,16 +8,13 @@ RUN apk add --no-cache --virtual .build_deps \
         libxslt-dev \
         linux-headers \
         postgresql-dev \
-        zlib-dev \
- && cd /synapse \
- && apk add --no-cache --virtual .runtime_deps \
- 	libffi \
-        libjpeg-turbo \
-	libressl \
-	libxslt \
-	libpq \
-	zlib \
-	su-exec \
+        su-exec \
+        zlib-dev
+
+COPY . /synapse
+
+# A wheel cache may be provided in ./cache for faster build
+RUN cd /synapse \
  && pip install --upgrade \
         lxml \
         pip \
@@ -32,9 +26,8 @@ RUN apk add --no-cache --virtual .build_deps \
  && rm -rf \
         setup.cfg \
         setup.py \
-        synapse \
- && apk del .build_deps
- 
+        synapse
+
 VOLUME ["/data"]
 
 EXPOSE 8008/tcp 8448/tcp

docker/Dockerfile-pgtests

@@ -1,12 +0,0 @@
-# Use the Sytest image that comes with a lot of the build dependencies
-# pre-installed
-FROM matrixdotorg/sytest:latest
-
-# The Sytest image doesn't come with python, so install that
-RUN apt-get -qq install -y python python-dev python-pip
-
-# We need tox to run the tests in run_pg_tests.sh
-RUN pip install tox
-
-ADD run_pg_tests.sh /pg_tests.sh
-ENTRYPOINT /pg_tests.sh

docker/README.md

@@ -88,7 +88,6 @@ variables are available for configuration:
 * ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN
   uris to enable TURN for this homeserver.
 * ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required.
-* ``SYNAPSE_MAX_UPLOAD_SIZE``, set this variable to change the max upload size [default `10M`].
 
 Shared secrets, that will be initialized to random values if not set:
 

docker/conf/homeserver.yaml

@@ -85,7 +85,7 @@ federation_rc_concurrent: 3
 
 media_store_path: "/data/media"
 uploads_path: "/data/uploads"
-max_upload_size: "{{ SYNAPSE_MAX_UPLOAD_SIZE or "10M" }}"
+max_upload_size: "10M"
 max_image_pixels: "32M"
 dynamic_thumbnails: false
 

docker/run_pg_tests.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-# This script runs the PostgreSQL tests inside a Docker container. It expects
-# the relevant source files to be mounted into /src (done automatically by the
-# caller script). It will set up the database, run it, and then use the tox
-# configuration to run the tests.
-
-set -e
-
-# Set PGUSER so Synapse's tests know what user to connect to the database with
-export PGUSER=postgres
-
-# Initialise & start the database
-su -c '/usr/lib/postgresql/9.6/bin/initdb -D /var/lib/postgresql/data -E "UTF-8" --lc-collate="en_US.UTF-8" --lc-ctype="en_US.UTF-8" --username=postgres' postgres
-su -c '/usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres
-
-# Run the tests
-cd /src
-export TRIAL_FLAGS="-j 4"
-tox --workdir=/tmp -e py27-postgres

docker/start.py

@@ -5,7 +5,6 @@ import os
 import sys
 import subprocess
 import glob
-import codecs
 
 # Utility functions
 convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ))
@@ -24,7 +23,7 @@ def generate_secrets(environ, secrets):
                 with open(filename) as handle: value = handle.read()
             else:
                 print("Generating a random secret for {}".format(name))
-                value = codecs.encode(os.urandom(32), "hex").decode()
+                value = os.urandom(32).encode("hex")
                 with open(filename, "w") as handle: handle.write(value)
             environ[secret] = value
 

docs/admin_api/register_api.rst

@@ -33,7 +33,7 @@ As an example::
 
   < {
      "access_token": "token_here",
-     "user_id": "@pepper_roni:localhost",
+     "user_id": "@pepper_roni@test",
      "home_server": "test",
      "device_id": "device_id_here"
     }

docs/workers.rst

@@ -74,7 +74,7 @@ replication endpoints that it's talking to on the main synapse process.
 ``worker_replication_port`` should point to the TCP replication listener port and
 ``worker_replication_http_port`` should point to the HTTP replication port.
 
-Currently, the ``event_creator`` and ``federation_reader`` workers require specifying
+Currently, only the ``event_creator`` worker requires specifying
 ``worker_replication_http_port``.
 
 For instance::
@@ -241,14 +241,6 @@ regular expressions::
 
     ^/_matrix/client/(api/v1|r0|unstable)/keys/upload
 
-If ``use_presence`` is False in the homeserver config, it can also handle REST
-endpoints matching the following regular expressions::
-
-    ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status
-
-This "stub" presence handler will pass through ``GET`` request but make the
-``PUT`` effectively a no-op.
-
 It will proxy any requests it cannot handle to the main synapse instance. It
 must therefore be configured with the location of the main instance, via
 the ``worker_main_http_uri`` setting in the frontend_proxy worker configuration
@@ -265,7 +257,6 @@ Handles some event creation. It can handle REST endpoints matching::
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$
     ^/_matrix/client/(api/v1|r0|unstable)/join/
-    ^/_matrix/client/(api/v1|r0|unstable)/profile/
 
 It will create events locally and then send them on to the main synapse
 instance to be persisted and handled.

jenkins/prepare_synapse.sh

@@ -31,5 +31,5 @@ $TOX_BIN/pip install 'setuptools>=18.5'
 $TOX_BIN/pip install 'pip>=10'
 
 { python synapse/python_dependencies.py
-  echo lxml
+  echo lxml psycopg2
 } | xargs $TOX_BIN/pip install

scripts-dev/dump_macaroon.py

@@ -21,4 +21,4 @@ try:
     verifier.verify(macaroon, key)
     print "Signature is correct"
 except Exception as e:
-    print str(e)
+    print e.message

scripts-dev/next_github_number.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# Fetch the current GitHub issue number, add one to it -- presto! The likely
-# next PR number.
-CURRENT_NUMBER=`curl -s "https://api.github.com/repos/matrix-org/synapse/issues?state=all&per_page=1" | jq -r ".[0].number"`
-CURRENT_NUMBER=$((CURRENT_NUMBER+1))
-echo $CURRENT_NUMBER

scripts-dev/nuke-room-from-db.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+## CAUTION:
+## This script will remove (hopefully) all trace of the given room ID from
+## your homeserver.db
+
+## Do not run it lightly.
+
+set -e
+
+if [ "$1" == "-h" ] || [ "$1" == "" ]; then
+  echo "Call with ROOM_ID as first option and then pipe it into the database. So for instance you might run"
+  echo " nuke-room-from-db.sh <room_id> | sqlite3 homeserver.db"
+  echo "or"
+  echo " nuke-room-from-db.sh <room_id> | psql --dbname=synapse"
+  exit
+fi
+
+ROOMID="$1"
+
+cat <<EOF
+DELETE FROM event_forward_extremities WHERE room_id = '$ROOMID';
+DELETE FROM event_backward_extremities WHERE room_id = '$ROOMID';
+DELETE FROM event_edges WHERE room_id = '$ROOMID';
+DELETE FROM room_depth WHERE room_id = '$ROOMID';
+DELETE FROM state_forward_extremities WHERE room_id = '$ROOMID';
+DELETE FROM events WHERE room_id = '$ROOMID';
+DELETE FROM event_json WHERE room_id = '$ROOMID';
+DELETE FROM state_events WHERE room_id = '$ROOMID';
+DELETE FROM current_state_events WHERE room_id = '$ROOMID';
+DELETE FROM room_memberships WHERE room_id = '$ROOMID';
+DELETE FROM feedback WHERE room_id = '$ROOMID';
+DELETE FROM topics WHERE room_id = '$ROOMID';
+DELETE FROM room_names WHERE room_id = '$ROOMID';
+DELETE FROM rooms WHERE room_id = '$ROOMID';
+DELETE FROM room_hosts WHERE room_id = '$ROOMID';
+DELETE FROM room_aliases WHERE room_id = '$ROOMID';
+DELETE FROM state_groups WHERE room_id = '$ROOMID';
+DELETE FROM state_groups_state WHERE room_id = '$ROOMID';
+DELETE FROM receipts_graph WHERE room_id = '$ROOMID';
+DELETE FROM receipts_linearized WHERE room_id = '$ROOMID';
+DELETE FROM event_search WHERE room_id = '$ROOMID'; 
+DELETE FROM guest_access WHERE room_id = '$ROOMID';
+DELETE FROM history_visibility WHERE room_id = '$ROOMID';
+DELETE FROM room_tags WHERE room_id = '$ROOMID';
+DELETE FROM room_tags_revisions WHERE room_id = '$ROOMID';
+DELETE FROM room_account_data WHERE room_id = '$ROOMID';
+DELETE FROM event_push_actions WHERE room_id = '$ROOMID';
+DELETE FROM local_invites WHERE room_id = '$ROOMID';
+DELETE FROM pusher_throttle WHERE room_id = '$ROOMID';
+DELETE FROM event_reports WHERE room_id = '$ROOMID';
+DELETE FROM public_room_list_stream WHERE room_id = '$ROOMID';
+DELETE FROM stream_ordering_to_exterm WHERE room_id = '$ROOMID';
+DELETE FROM event_auth WHERE room_id = '$ROOMID';
+DELETE FROM appservice_room_list WHERE room_id = '$ROOMID';
+VACUUM;
+EOF

scripts/register_new_matrix_user

@@ -133,7 +133,7 @@ def register_new_user(user, password, server_location, shared_secret, admin):
             print "Passwords do not match"
             sys.exit(1)
 
-    if admin is None:
+    if not admin:
         admin = raw_input("Make admin [no]: ")
         if admin in ("y", "yes", "true"):
             admin = True
@@ -160,16 +160,10 @@ if __name__ == "__main__":
         default=None,
         help="New password for user. Will prompt if omitted.",
     )
-    admin_group = parser.add_mutually_exclusive_group()
-    admin_group.add_argument(
+    parser.add_argument(
         "-a", "--admin",
         action="store_true",
-        help="Register new user as an admin. Will prompt if --no-admin is not set either.",
-    )
-    admin_group.add_argument(
-        "--no-admin",
-        action="store_true",
-        help="Register new user as a regular user. Will prompt if --admin is not set either.",
+        help="Register new user as an admin. Will prompt if omitted.",
     )
 
     group = parser.add_mutually_exclusive_group(required=True)
@@ -203,8 +197,4 @@ if __name__ == "__main__":
     else:
         secret = args.shared_secret
 
-    admin = None
-    if args.admin or args.no_admin:
-        admin = args.admin
-
-    register_new_user(args.user, args.password, args.server_url, secret, admin)
+    register_new_user(args.user, args.password, args.server_url, secret, args.admin)

setup.cfg

@@ -17,14 +17,13 @@ ignore =
 [pep8]
 max-line-length = 90
 #  W503 requires that binary operators be at the end, not start, of lines. Erik
-#  doesn't like it.  E203 is contrary to PEP8.  E731 is silly.
-ignore = W503,E203,E731
+#  doesn't like it.  E203 is contrary to PEP8.
+ignore = W503,E203
 
 [flake8]
 # note that flake8 inherits the "ignore" settings from "pep8" (because it uses
 # pep8 to do those checks), but not the "max-line-length" setting
 max-line-length = 90
-ignore=W503,E203,E731
 
 [isort]
 line_length = 89

synapse/__init__.py

@@ -17,14 +17,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-try:
-    from twisted.internet import protocol
-    from twisted.internet.protocol import Factory
-    from twisted.names.dns import DNSDatagramProtocol
-    protocol.Factory.noisy = False
-    Factory.noisy = False
-    DNSDatagramProtocol.noisy = False
-except ImportError:
-    pass
-
-__version__ = "0.33.5.1"
+__version__ = "0.33.2"

synapse/api/auth.py

@@ -25,8 +25,7 @@ from twisted.internet import defer
 import synapse.types
 from synapse import event_auth
 from synapse.api.constants import EventTypes, JoinRules, Membership
-from synapse.api.errors import AuthError, Codes, ResourceLimitError
-from synapse.config.server import is_threepid_reserved
+from synapse.api.errors import AuthError, Codes
 from synapse.types import UserID
 from synapse.util.caches import CACHE_SIZE_FACTOR, register_cache
 from synapse.util.caches.lrucache import LruCache
@@ -212,7 +211,7 @@ class Auth(object):
             user_agent = request.requestHeaders.getRawHeaders(
                 b"User-Agent",
                 default=[b""]
-            )[0].decode('ascii', 'surrogateescape')
+            )[0]
             if user and access_token and ip_addr:
                 yield self.store.insert_client_ip(
                     user_id=user.to_string(),
@@ -683,7 +682,7 @@ class Auth(object):
         Returns:
             bool: False if no access_token was given, True otherwise.
         """
-        query_params = request.args.get(b"access_token")
+        query_params = request.args.get("access_token")
         auth_headers = request.requestHeaders.getRawHeaders(b"Authorization")
         return bool(query_params) or bool(auth_headers)
 
@@ -699,7 +698,7 @@ class Auth(object):
                 401 since some of the old clients depended on auth errors returning
                 403.
         Returns:
-            unicode: The access_token
+            str: The access_token
         Raises:
             AuthError: If there isn't an access_token in the request.
         """
@@ -721,9 +720,9 @@ class Auth(object):
                     "Too many Authorization headers.",
                     errcode=Codes.MISSING_TOKEN,
                 )
-            parts = auth_headers[0].split(b" ")
-            if parts[0] == b"Bearer" and len(parts) == 2:
-                return parts[1].decode('ascii')
+            parts = auth_headers[0].split(" ")
+            if parts[0] == "Bearer" and len(parts) == 2:
+                return parts[1]
             else:
                 raise AuthError(
                     token_not_found_http_status,
@@ -739,7 +738,7 @@ class Auth(object):
                     errcode=Codes.MISSING_TOKEN
                 )
 
-            return query_params[0].decode('ascii')
+            return query_params[0]
 
     @defer.inlineCallbacks
     def check_in_room_or_world_readable(self, room_id, user_id):
@@ -776,56 +775,31 @@ class Auth(object):
             )
 
     @defer.inlineCallbacks
-    def check_auth_blocking(self, user_id=None, threepid=None):
+    def check_auth_blocking(self, user_id=None):
         """Checks if the user should be rejected for some external reason,
         such as monthly active user limiting or global disable flag
 
         Args:
             user_id(str|None): If present, checks for presence against existing
             MAU cohort
-
-            threepid(dict|None): If present, checks for presence against configured
-            reserved threepid. Used in cases where the user is trying register
-            with a MAU blocked server, normally they would be rejected but their
-            threepid is on the reserved list. user_id and
-            threepid should never be set at the same time.
         """
-
-        # Never fail an auth check for the server notices users
-        # This can be a problem where event creation is prohibited due to blocking
-        if user_id == self.hs.config.server_notices_mxid:
-            return
-
         if self.hs.config.hs_disabled:
-            raise ResourceLimitError(
+            raise AuthError(
                 403, self.hs.config.hs_disabled_message,
                 errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
-                admin_contact=self.hs.config.admin_contact,
-                limit_type=self.hs.config.hs_disabled_limit_type
+                admin_uri=self.hs.config.admin_uri,
             )
         if self.hs.config.limit_usage_by_mau is True:
-            assert not (user_id and threepid)
-
-            # If the user is already part of the MAU cohort or a trial user
+            # If the user is already part of the MAU cohort
             if user_id:
                 timestamp = yield self.store.user_last_seen_monthly_active(user_id)
                 if timestamp:
                     return
-
-                is_trial = yield self.store.is_trial_user(user_id)
-                if is_trial:
-                    return
-            elif threepid:
-                # If the user does not exist yet, but is signing up with a
-                # reserved threepid then pass auth check
-                if is_threepid_reserved(self.hs.config, threepid):
-                    return
             # Else if there is no room in the MAU bucket, bail
             current_mau = yield self.store.get_monthly_active_count()
             if current_mau >= self.hs.config.max_mau_value:
-                raise ResourceLimitError(
-                    403, "Monthly Active User Limit Exceeded",
-                    admin_contact=self.hs.config.admin_contact,
-                    errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
-                    limit_type="monthly_active_user"
+                raise AuthError(
+                    403, "Monthly Active User Limits AU Limit Exceeded",
+                    admin_uri=self.hs.config.admin_uri,
+                    errcode=Codes.RESOURCE_LIMIT_EXCEEDED
                 )

synapse/api/constants.py

@@ -78,7 +78,6 @@ class EventTypes(object):
     Name = "m.room.name"
 
     ServerACL = "m.room.server_acl"
-    Pinned = "m.room.pinned_events"
 
 
 class RejectedReason(object):
@@ -98,17 +97,9 @@ class ThirdPartyEntityKind(object):
     LOCATION = "location"
 
 
-class RoomVersions(object):
-    V1 = "1"
-    VDH_TEST = "vdh-test-version"
-
-
 # the version we will give rooms which are created on this server
-DEFAULT_ROOM_VERSION = RoomVersions.V1
+DEFAULT_ROOM_VERSION = "1"
 
 # vdh-test-version is a placeholder to get room versioning support working and tested
 # until we have a working v2.
-KNOWN_ROOM_VERSIONS = {RoomVersions.V1, RoomVersions.VDH_TEST}
-
-ServerNoticeMsgType = "m.server_notice"
-ServerNoticeLimitReached = "m.server_notice.usage_limit_reached"
+KNOWN_ROOM_VERSIONS = {"1", "vdh-test-version"}

synapse/api/errors.py

@@ -224,34 +224,15 @@ class NotFoundError(SynapseError):
 
 class AuthError(SynapseError):
     """An error raised when there was a problem authorising an event."""
-
-    def __init__(self, *args, **kwargs):
-        if "errcode" not in kwargs:
-            kwargs["errcode"] = Codes.FORBIDDEN
-        super(AuthError, self).__init__(*args, **kwargs)
-
-
-class ResourceLimitError(SynapseError):
-    """
-    Any error raised when there is a problem with resource usage.
-    For instance, the monthly active user limit for the server has been exceeded
-    """
-    def __init__(
-        self, code, msg,
-        errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
-        admin_contact=None,
-        limit_type=None,
-    ):
-        self.admin_contact = admin_contact
-        self.limit_type = limit_type
-        super(ResourceLimitError, self).__init__(code, msg, errcode=errcode)
+    def __init__(self, code, msg, errcode=Codes.FORBIDDEN, admin_uri=None):
+        self.admin_uri = admin_uri
+        super(AuthError, self).__init__(code, msg, errcode=errcode)
 
     def error_dict(self):
         return cs_error(
             self.msg,
             self.errcode,
-            admin_contact=self.admin_contact,
-            limit_type=self.limit_type
+            admin_uri=self.admin_uri,
         )
 
 

synapse/api/filtering.py

@@ -226,7 +226,7 @@ class Filtering(object):
             jsonschema.validate(user_filter_json, USER_FILTER_SCHEMA,
                                 format_checker=FormatChecker())
         except jsonschema.ValidationError as e:
-            raise SynapseError(400, str(e))
+            raise SynapseError(400, e.message)
 
 
 class FilterCollection(object):
@@ -251,7 +251,6 @@ class FilterCollection(object):
             "include_leave", False
         )
         self.event_fields = filter_json.get("event_fields", [])
-        self.event_format = filter_json.get("event_format", "client")
 
     def __repr__(self):
         return "<FilterCollection %s>" % (json.dumps(self._filter_json),)

synapse/api/ratelimiting.py

@@ -72,7 +72,7 @@ class Ratelimiter(object):
         return allowed, time_allowed
 
     def prune_message_counts(self, time_now_s):
-        for user_id in list(self.message_counts.keys()):
+        for user_id in self.message_counts.keys():
             message_count, time_start, msg_rate_hz = (
                 self.message_counts[user_id]
             )

synapse/app/__init__.py

@@ -24,7 +24,7 @@ try:
     python_dependencies.check_requirements()
 except python_dependencies.MissingRequirementError as e:
     message = "\n".join([
-        "Missing Requirement: %s" % (str(e),),
+        "Missing Requirement: %s" % (e.message,),
         "To install run:",
         "    pip install --upgrade --force \"%s\"" % (e.dependency,),
         "",

synapse/app/_base.py

@@ -140,7 +140,7 @@ def listen_metrics(bind_addresses, port):
         logger.info("Metrics now reporting on %s:%d", host, port)
 
 
-def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50):
+def listen_tcp(bind_addresses, port, factory, backlog=50):
     """
     Create a TCP socket for a port and several addresses
     """
@@ -156,9 +156,7 @@ def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50):
             check_bind_error(e, address, bind_addresses)
 
 
-def listen_ssl(
-    bind_addresses, port, factory, context_factory, reactor=reactor, backlog=50
-):
+def listen_ssl(bind_addresses, port, factory, context_factory, backlog=50):
     """
     Create an SSL socket for a port and several addresses
     """