Up no output timeout

.circleci/config.yml

@@ -14,7 +14,7 @@ jobs:
           platforms: linux/amd64
       - docker_build:
           tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64,linux/arm/v7,linux/arm64
 
   dockerhubuploadlatest:
     docker:
@@ -22,12 +22,12 @@ jobs:
     steps:
       - checkout
       - docker_prepare
-      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      # - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       # for `latest`, we don't want the arm images to disappear, so don't update the tag
       # until all of the platforms are built.
       - docker_build:
-          tag: -t matrixdotorg/synapse:latest
-          platforms: linux/amd64,linux/arm64
+          tag: -t 127.0.0.1:5000/synapse:erikj-test
+          platforms: linux/amd64,linux/arm/v7,linux/arm64
 
 workflows:
   build:
@@ -41,7 +41,7 @@ workflows:
       - dockerhubuploadlatest:
           filters:
             branches:
-              only: master
+              only: erikj/arm_docker_cache
 
 commands:
   docker_prepare:
@@ -52,9 +52,9 @@ commands:
         default: "v0.4.1"
     steps:
       - setup_remote_docker:
-          # 19.03.13 was the most recent available on circleci at the time of
+          # 20.10.2 was the most recent available on circleci at the time of
           # writing.
-          version: 19.03.13
+          version: 20.10.2
       - run: apk add --no-cache curl
       - run: mkdir -vp ~/.docker/cli-plugins/ ~/dockercache
       - run: curl --silent -L "https://github.com/docker/buildx/releases/download/<< parameters.buildx_version >>/buildx-<< parameters.buildx_version >>.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
@@ -64,7 +64,10 @@ commands:
       # create a context named `builder` for the builds
       - run: docker context create builder
       # create a buildx builder using the new context, and set it as the default
-      - run: docker buildx create builder --use
+      - run: docker buildx create --driver docker-container --driver-opt network=host builder --use
+      # Start a registry so that have somewhere to store our temporary docker
+      # images (as multi arch builds don't work with stand local docker store)
+      - run: docker run -d -p 127.0.0.1:5000:5000 --name registry registry:2
 
   docker_build:
     description: Builds and pushed images to dockerhub using buildx
@@ -75,4 +78,7 @@ commands:
       tag:
         type: string
     steps:
-      - run: docker buildx build -f docker/Dockerfile --push --platform << parameters.platforms >> --label gitsha1=${CIRCLE_SHA1} << parameters.tag >> --progress=plain .
+      - run: docker buildx build -f docker/Dockerfile-cargo-cache --push -t 127.0.0.1:5000/cargo_cache --platform << parameters.platforms >> --progress=plain .
+      - run:
+          command: docker buildx build -f docker/Dockerfile --push --platform << parameters.platforms >> --label gitsha1=${CIRCLE_SHA1} << parameters.tag >> --build-arg BASE_IMAGE=127.0.0.1:5000/cargo_cache --build-arg CARGO_NET_OFFLINE=true --progress=plain  .
+          no_output_timeout: 30m

CHANGES.md

@@ -1,108 +1,9 @@
-Synapse 1.28.0 (2021-02-25)
-===========================
-
-Note that this release drops support for ARMv7 in the official Docker images, due to repeated problems building for ARMv7 (and the associated maintenance burden this entails).
-
-This release also fixes the documentation included in v1.27.0 around the callback URI for SAML2 identity providers. If your server is configured to use single sign-on via a SAML2 IdP, you may need to make configuration changes. Please review [UPGRADE.rst](UPGRADE.rst) for more details on these changes.
-
-
-Internal Changes
-----------------
-
-- Revert change in v1.28.0rc1 to remove the deprecated SAML endpoint. ([\#9474](https://github.com/matrix-org/synapse/issues/9474))
-
-
-Synapse 1.28.0rc1 (2021-02-19)
-==============================
-
-Removal warning
----------------
-
-The v1 list accounts API is deprecated and will be removed in a future release.
-This API was undocumented and misleading. It can be replaced by the
-[v2 list accounts API](https://github.com/matrix-org/synapse/blob/release-v1.28.0/docs/admin_api/user_admin_api.rst#list-accounts),
-which has been available since Synapse 1.7.0 (2019-12-13).
-
-Please check if you're using any scripts which use the admin API and replace
-`GET /_synapse/admin/v1/users/<user_id>` with `GET /_synapse/admin/v2/users`.
-
-
-Features
---------
-
-- New admin API to get the context of an event: `/_synapse/admin/rooms/{roomId}/context/{eventId}`. ([\#9150](https://github.com/matrix-org/synapse/issues/9150))
-- Further improvements to the user experience of registration via single sign-on. ([\#9300](https://github.com/matrix-org/synapse/issues/9300), [\#9301](https://github.com/matrix-org/synapse/issues/9301))
-- Add hook to spam checker modules that allow checking file uploads and remote downloads. ([\#9311](https://github.com/matrix-org/synapse/issues/9311))
-- Add support for receiving OpenID Connect authentication responses via form `POST`s rather than `GET`s. ([\#9376](https://github.com/matrix-org/synapse/issues/9376))
-- Add the shadow-banning status to the admin API for user info. ([\#9400](https://github.com/matrix-org/synapse/issues/9400))
-
-
-Bugfixes
---------
-
-- Fix long-standing bug where sending email notifications would fail for rooms that the server had since left. ([\#9257](https://github.com/matrix-org/synapse/issues/9257))
-- Fix bug introduced in Synapse 1.27.0rc1 which meant the "session expired" error page during SSO registration was badly formatted. ([\#9296](https://github.com/matrix-org/synapse/issues/9296))
-- Assert a maximum length for some parameters for spec compliance. ([\#9321](https://github.com/matrix-org/synapse/issues/9321), [\#9393](https://github.com/matrix-org/synapse/issues/9393))
-- Fix additional errors when previewing URLs: "AttributeError 'NoneType' object has no attribute 'xpath'" and "ValueError: Unicode strings with encoding declaration are not supported. Please use bytes input or XML fragments without declaration.". ([\#9333](https://github.com/matrix-org/synapse/issues/9333))
-- Fix a bug causing Synapse to impose the wrong type constraints on fields when processing responses from appservices to `/_matrix/app/v1/thirdparty/user/{protocol}`. ([\#9361](https://github.com/matrix-org/synapse/issues/9361))
-- Fix bug where Synapse would occasionally stop reconnecting to Redis after the connection was lost. ([\#9391](https://github.com/matrix-org/synapse/issues/9391))
-- Fix a long-standing bug when upgrading a room: "TypeError: '>' not supported between instances of 'NoneType' and 'int'". ([\#9395](https://github.com/matrix-org/synapse/issues/9395))
-- Reduce the amount of memory used when generating the URL preview of a file that is larger than the `max_spider_size`. ([\#9421](https://github.com/matrix-org/synapse/issues/9421))
-- Fix a long-standing bug in the deduplication of old presence, resulting in no deduplication. ([\#9425](https://github.com/matrix-org/synapse/issues/9425))
-- The `ui_auth.session_timeout` config option can now be specified in terms of number of seconds/minutes/etc/. Contributed by Rishabh Arya. ([\#9426](https://github.com/matrix-org/synapse/issues/9426))
-- Fix a bug introduced in v1.27.0: "TypeError: int() argument must be a string, a bytes-like object or a number, not 'NoneType." related to the user directory. ([\#9428](https://github.com/matrix-org/synapse/issues/9428))
-
-
-Updates to the Docker image
----------------------------
-
-- Drop support for ARMv7 in Docker images. ([\#9433](https://github.com/matrix-org/synapse/issues/9433))
-
-
-Improved Documentation
-----------------------
-
-- Reorganize CHANGELOG.md. ([\#9281](https://github.com/matrix-org/synapse/issues/9281))
-- Add note to `auto_join_rooms` config option explaining existing rooms must be publicly joinable. ([\#9291](https://github.com/matrix-org/synapse/issues/9291))
-- Correct name of Synapse's service file in TURN howto. ([\#9308](https://github.com/matrix-org/synapse/issues/9308))
-- Fix the braces in the `oidc_providers` section of the sample config. ([\#9317](https://github.com/matrix-org/synapse/issues/9317))
-- Update installation instructions on Fedora. ([\#9322](https://github.com/matrix-org/synapse/issues/9322))
-- Add HTTP/2 support to the nginx example configuration. Contributed by David Vo. ([\#9390](https://github.com/matrix-org/synapse/issues/9390))
-- Update docs for using Gitea as OpenID provider. ([\#9404](https://github.com/matrix-org/synapse/issues/9404))
-- Document that pusher instances are shardable. ([\#9407](https://github.com/matrix-org/synapse/issues/9407))
-- Fix erroneous documentation from v1.27.0 about updating the SAML2 callback URL. ([\#9434](https://github.com/matrix-org/synapse/issues/9434))
-
-
-Deprecations and Removals
--------------------------
-
-- Deprecate old admin API `GET /_synapse/admin/v1/users/<user_id>`. ([\#9429](https://github.com/matrix-org/synapse/issues/9429))
-
-
-Internal Changes
-----------------
-
-- Fix 'object name reserved for internal use' errors with recent versions of SQLite. ([\#9003](https://github.com/matrix-org/synapse/issues/9003))
-- Add experimental support for running Synapse with PyPy. ([\#9123](https://github.com/matrix-org/synapse/issues/9123))
-- Deny access to additional IP addresses by default. ([\#9240](https://github.com/matrix-org/synapse/issues/9240))
-- Update the `Cursor` type hints to better match PEP 249. ([\#9299](https://github.com/matrix-org/synapse/issues/9299))
-- Add debug logging for SRV lookups. Contributed by @Bubu. ([\#9305](https://github.com/matrix-org/synapse/issues/9305))
-- Improve logging for OIDC login flow. ([\#9307](https://github.com/matrix-org/synapse/issues/9307))
-- Share the code for handling required attributes between the CAS and SAML handlers. ([\#9326](https://github.com/matrix-org/synapse/issues/9326))
-- Clean up the code to load the metadata for OpenID Connect identity providers. ([\#9362](https://github.com/matrix-org/synapse/issues/9362))
-- Convert tests to use `HomeserverTestCase`. ([\#9377](https://github.com/matrix-org/synapse/issues/9377), [\#9396](https://github.com/matrix-org/synapse/issues/9396))
-- Update the version of black used to 20.8b1. ([\#9381](https://github.com/matrix-org/synapse/issues/9381))
-- Allow OIDC config to override discovered values. ([\#9384](https://github.com/matrix-org/synapse/issues/9384))
-- Remove some dead code from the acceptance of room invites path. ([\#9394](https://github.com/matrix-org/synapse/issues/9394))
-- Clean up an unused method in the presence handler code. ([\#9408](https://github.com/matrix-org/synapse/issues/9408))
-
-
 Synapse 1.27.0 (2021-02-16)
 ===========================
 
 Note that this release includes a change in Synapse to use Redis as a cache ─ as well as a pub/sub mechanism ─ if Redis support is enabled for workers. No action is needed by server administrators, and we do not expect resource usage of the Redis instance to change dramatically.
 
-This release also changes the callback URI for OpenID Connect (OIDC) and SAML2 identity providers. If your server is configured to use single sign-on via an OIDC/OAuth2 or SAML2 IdP, you may need to make configuration changes. Please review [UPGRADE.rst](UPGRADE.rst) for more details on these changes.
+This release also changes the callback URI for OpenID Connect (OIDC) identity providers. If your server is configured to use single sign-on via an OIDC/OAuth2 IdP, you may need to make configuration changes. Please review [UPGRADE.rst](UPGRADE.rst) for more details on these changes.
 
 This release also changes escaping of variables in the HTML templates for SSO or email notifications. If you have customised these templates, please review [UPGRADE.rst](UPGRADE.rst) for more details on these changes.
 

CONTRIBUTING.md

@@ -1,31 +1,4 @@
-Welcome to Synapse
-
-This document aims to get you started with contributing to this repo! 
-
-- [1. Who can contribute to Synapse?](#1-who-can-contribute-to-synapse)
-- [2. What do I need?](#2-what-do-i-need)
-- [3. Get the source.](#3-get-the-source)
-- [4. Install the dependencies](#4-install-the-dependencies)
-  * [Under Unix (macOS, Linux, BSD, ...)](#under-unix-macos-linux-bsd-)
-  * [Under Windows](#under-windows)
-- [5. Get in touch.](#5-get-in-touch)
-- [6. Pick an issue.](#6-pick-an-issue)
-- [7. Turn coffee and documentation into code and documentation!](#7-turn-coffee-and-documentation-into-code-and-documentation)
-- [8. Test, test, test!](#8-test-test-test)
-  * [Run the linters.](#run-the-linters)
-  * [Run the unit tests.](#run-the-unit-tests)
-  * [Run the integration tests.](#run-the-integration-tests)
-- [9. Submit your patch.](#9-submit-your-patch)
-  * [Changelog](#changelog)
-    + [How do I know what to call the changelog file before I create the PR?](#how-do-i-know-what-to-call-the-changelog-file-before-i-create-the-pr)
-    + [Debian changelog](#debian-changelog)
-  * [Sign off](#sign-off)
-- [10. Turn feedback into better code.](#10-turn-feedback-into-better-code)
-- [11. Find a new issue.](#11-find-a-new-issue)
-- [Notes for maintainers on merging PRs etc](#notes-for-maintainers-on-merging-prs-etc)
-- [Conclusion](#conclusion)
-
-# 1. Who can contribute to Synapse?
+# Contributing code to Synapse
 
 Everyone is welcome to contribute code to [matrix.org
 projects](https://github.com/matrix-org), provided that they are willing to
@@ -36,179 +9,70 @@ license the code under the same terms as the project's overall 'outbound'
 license - in our case, this is almost always Apache Software License v2 (see
 [LICENSE](LICENSE)).
 
-# 2. What do I need?
-
-The code of Synapse is written in Python 3. To do pretty much anything, you'll need [a recent version of Python 3](https://wiki.python.org/moin/BeginnersGuide/Download).
-
-The source code of Synapse is hosted on GitHub. You will also need [a recent version of git](https://github.com/git-guides/install-git).
-
-For some tests, you will need [a recent version of Docker](https://docs.docker.com/get-docker/).
-
-
-# 3. Get the source.
+## How to contribute
 
 The preferred and easiest way to contribute changes is to fork the relevant
-project on GitHub, and then [create a pull request](
+project on github, and then [create a pull request](
 https://help.github.com/articles/using-pull-requests/) to ask us to pull your
 changes into our repo.
 
-Please base your changes on the `develop` branch.
+Some other points to follow:
 
-```sh
-git clone git@github.com:YOUR_GITHUB_USER_NAME/synapse.git
-git checkout develop
-```
+ * Please base your changes on the `develop` branch.
 
-If you need help getting started with git, this is beyond the scope of the document, but you
-can find many good git tutorials on the web.
+ * Please follow the [code style requirements](#code-style).
 
-# 4. Install the dependencies
+ * Please include a [changelog entry](#changelog) with each PR.
 
-## Under Unix (macOS, Linux, BSD, ...)
+ * Please [sign off](#sign-off) your contribution.
 
-Once you have installed Python 3 and added the source, please open a terminal and
-setup a *virtualenv*, as follows:
+ * Please keep an eye on the pull request for feedback from the [continuous
+   integration system](#continuous-integration-and-testing) and try to fix any
+   errors that come up.
 
-```sh
-cd path/where/you/have/cloned/the/repository
-python3 -m venv ./env
-source ./env/bin/activate
-pip install -e ".[all,lint,mypy,test]"
-pip install tox
-```
+ * If you need to [update your PR](#updating-your-pull-request), just add new
+   commits to your branch rather than rebasing.
 
-This will install the developer dependencies for the project.
-
-## Under Windows
-
-TBD
-
-
-# 5. Get in touch.
-
-Join our developer community on Matrix: #synapse-dev:matrix.org !
-
-
-# 6. Pick an issue.
-
-Fix your favorite problem or perhaps find a [Good First Issue](https://github.com/matrix-org/synapse/issues?q=is%3Aopen+is%3Aissue+label%3A%22Good+First+Issue%22)
-to work on.
-
-
-# 7. Turn coffee and documentation into code and documentation!
+## Code style
 
 Synapse's code style is documented [here](docs/code_style.md). Please follow
 it, including the conventions for the [sample configuration
 file](docs/code_style.md#configuration-file-format).
 
-There is a growing amount of documentation located in the [docs](docs)
-directory. This documentation is intended primarily for sysadmins running their
-own Synapse instance, as well as developers interacting externally with
-Synapse. [docs/dev](docs/dev) exists primarily to house documentation for
-Synapse developers. [docs/admin_api](docs/admin_api) houses documentation
-regarding Synapse's Admin API, which is used mostly by sysadmins and external
-service developers.
+Many of the conventions are enforced by scripts which are run as part of the
+[continuous integration system](#continuous-integration-and-testing). To help
+check if you have followed the code style, you can run `scripts-dev/lint.sh`
+locally. You'll need python 3.6 or later, and to install a number of tools:
 
-If you add new files added to either of these folders, please use [GitHub-Flavoured
-Markdown](https://guides.github.com/features/mastering-markdown/).
+```
+# Install the dependencies
+pip install -e ".[lint,mypy]"
 
-Some documentation also exists in [Synapse's GitHub
-Wiki](https://github.com/matrix-org/synapse/wiki), although this is primarily
-contributed to by community authors.
-
-
-# 8. Test, test, test!
-<a name="test-test-test"></a>
-
-While you're developing and before submitting a patch, you'll
-want to test your code.
-
-## Run the linters.
-
-The linters look at your code and do two things:
-
-- ensure that your code follows the coding style adopted by the project;
-- catch a number of errors in your code.
-
-They're pretty fast, don't hesitate!
-
-```sh
-source ./env/bin/activate
+# Run the linter script
 ./scripts-dev/lint.sh
 ```
 
-Note that this script *will modify your files* to fix styling errors.
-Make sure that you have saved all your files.
+**Note that the script does not just test/check, but also reformats code, so you
+may wish to ensure any new code is committed first**.
 
-If you wish to restrict the linters to only the files changed since the last commit
-(much faster!), you can instead run:
+By default, this script checks all files and can take some time; if you alter
+only certain files, you might wish to specify paths as arguments to reduce the
+run-time:
 
-```sh
-source ./env/bin/activate
-./scripts-dev/lint.sh -d
 ```
-
-Or if you know exactly which files you wish to lint, you can instead run:
-
-```sh
-source ./env/bin/activate
 ./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder
 ```
 
-## Run the unit tests.
+You can also provide the `-d` option, which will lint the files that have been
+changed since the last git commit. This will often be significantly faster than
+linting the whole codebase.
 
-The unit tests run parts of Synapse, including your changes, to see if anything
-was broken. They are slower than the linters but will typically catch more errors.
-
-```sh
-source ./env/bin/activate
-trial tests
-```
-
-If you wish to only run *some* unit tests, you may specify
-another module instead of `tests` - or a test class or a method:
-
-```sh
-source ./env/bin/activate
-trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite
-```
-
-If your tests fail, you may wish to look at the logs:
-
-```sh
-less _trial_temp/test.log
-```
-
-## Run the integration tests.
-
-The integration tests are a more comprehensive suite of tests. They
-run a full version of Synapse, including your changes, to check if
-anything was broken. They are slower than the unit tests but will
-typically catch more errors.
-
-The following command will let you run the integration test with the most common
-configuration:
-
-```sh
-$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:py37
-```
-
-This configuration should generally cover  your needs. For more details about other configurations, see [documentation in the SyTest repo](https://github.com/matrix-org/sytest/blob/develop/docker/README.md).
-
-
-# 9. Submit your patch.
-
-Once you're happy with your patch, it's time to prepare a Pull Request.
-
-To prepare a Pull Request, please:
-
-1. verify that [all the tests pass](#test-test-test), including the coding style;
-2. [sign off](#sign-off) your contribution;
-3. `git push` your commit to your fork of Synapse;
-4. on GitHub, [create the Pull Request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request);
-5. add a [changelog entry](#changelog) and push it to your Pull Request;
-6. for most contributors, that's all - however, if you are a member of the organization `matrix-org`, on GitHub, please request a review from `matrix.org / Synapse Core`.
+Before pushing new changes, ensure they don't produce linting errors. Commit any
+files that were corrected.
 
+Please ensure your changes match the cosmetic style of the existing project,
+and **never** mix cosmetic and functional changes in the same commit, as it
+makes it horribly hard to review otherwise.
 
 ## Changelog
 
@@ -292,6 +156,24 @@ directory, you will need both a regular newsfragment *and* an entry in the
 debian changelog. (Though typically such changes should be submitted as two
 separate pull requests.)
 
+## Documentation
+
+There is a growing amount of documentation located in the [docs](docs)
+directory. This documentation is intended primarily for sysadmins running their
+own Synapse instance, as well as developers interacting externally with
+Synapse. [docs/dev](docs/dev) exists primarily to house documentation for
+Synapse developers. [docs/admin_api](docs/admin_api) houses documentation
+regarding Synapse's Admin API, which is used mostly by sysadmins and external
+service developers.
+
+New files added to both folders should be written in [Github-Flavoured
+Markdown](https://guides.github.com/features/mastering-markdown/), and attempts
+should be made to migrate existing documents to markdown where possible.
+
+Some documentation also exists in [Synapse's Github
+Wiki](https://github.com/matrix-org/synapse/wiki), although this is primarily
+contributed to by community authors.
+
 ## Sign off
 
 In order to have a concrete record that your contribution is intentional
@@ -358,36 +240,47 @@ Git allows you to add this signoff automatically when using the `-s`
 flag to `git commit`, which uses the name and email set in your
 `user.name` and `user.email` git configs.
 
+## Continuous integration and testing
 
-# 10. Turn feedback into better code.
+[Buildkite](https://buildkite.com/matrix-dot-org/synapse) will automatically
+run a series of checks and tests against any PR which is opened against the
+project; if your change breaks the build, this will be shown in GitHub, with
+links to the build results. If your build fails, please try to fix the errors
+and update your branch.
 
-Once the Pull Request is opened, you will see a few things:
+To run unit tests in a local development environment, you can use:
 
-1. our automated CI (Continuous Integration) pipeline will run (again) the linters, the unit tests, the integration tests and more;
-2. one or more of the developers will take a look at your Pull Request and offer feedback.
+- ``tox -e py35`` (requires tox to be installed by ``pip install tox``)
+  for SQLite-backed Synapse on Python 3.5.
+- ``tox -e py36`` for SQLite-backed Synapse on Python 3.6.
+- ``tox -e py36-postgres`` for PostgreSQL-backed Synapse on Python 3.6
+  (requires a running local PostgreSQL with access to create databases).
+- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 3.5
+  (requires Docker). Entirely self-contained, recommended if you don't want to
+  set up PostgreSQL yourself.
 
-From this point, you should:
+Docker images are available for running the integration tests (SyTest) locally,
+see the [documentation in the SyTest repo](
+https://github.com/matrix-org/sytest/blob/develop/docker/README.md) for more
+information.
 
-1. Look at the results of the CI pipeline.
-   - If there is any error, fix the error.
-2. If a developer has requested changes, make these changes and let us know if it is ready for a developer to review again.
-3. Create a new commit with the changes.
-   - Please do NOT overwrite the history. New commits make the reviewer's life easier.
-   - Push this commits to your Pull Request.
-4. Back to 1.
+## Updating your pull request
 
-Once both the CI and the developers are happy, the patch will be merged into Synapse and released shortly!
+If you decide to make changes to your pull request - perhaps to address issues
+raised in a review, or to fix problems highlighted by [continuous
+integration](#continuous-integration-and-testing) - just add new commits to your
+branch, and push to GitHub. The pull request will automatically be updated.
 
-# 11. Find a new issue.
+Please **avoid** rebasing your branch, especially once the PR has been
+reviewed: doing so makes it very difficult for a reviewer to see what has
+changed since a previous review.
 
-By now, you know the drill!
-
-# Notes for maintainers on merging PRs etc
+## Notes for maintainers on merging PRs etc
 
 There are some notes for those with commit access to the project on how we
 manage git [here](docs/dev/git.md).
 
-# Conclusion
+## Conclusion
 
 That's it! Matrix is a very open and collaborative project as you might expect
 given our obsession with open communication. If we're going to successfully

UPGRADE.rst

@@ -88,21 +88,20 @@ for example:
 Upgrading to v1.27.0
 ====================
 
-Changes to callback URI for OAuth2 / OpenID Connect and SAML2
--------------------------------------------------------------
+Changes to callback URI for OAuth2 / OpenID Connect
+---------------------------------------------------
 
-This version changes the URI used for callbacks from OAuth2 and SAML2 identity providers:
+This version changes the URI used for callbacks from OAuth2 identity providers. If
+your server is configured for single sign-on via an OpenID Connect or OAuth2 identity
+provider, you will need to add ``[synapse public baseurl]/_synapse/client/oidc/callback``
+to the list of permitted "redirect URIs" at the identity provider.
 
-* If your server is configured for single sign-on via an OpenID Connect or OAuth2 identity
-  provider, you will need to add ``[synapse public baseurl]/_synapse/client/oidc/callback``
-  to the list of permitted "redirect URIs" at the identity provider.
+See `docs/openid.md <docs/openid.md>`_ for more information on setting up OpenID
+Connect.
 
-  See `docs/openid.md <docs/openid.md>`_ for more information on setting up OpenID
-  Connect.
-
-* If your server is configured for single sign-on via a SAML2 identity provider, you will
-  need to add ``[synapse public baseurl]/_synapse/client/saml2/authn_response`` as a permitted
-  "ACS location" (also known as "allowed callback URLs") at the identity provider.
+(Note: a similar change is being made for SAML2; in this case the old URI
+``[synapse public baseurl]/_matrix/saml2`` is being deprecated, but will continue to
+work, so no immediate changes are required for existing installations.)
 
 Changes to HTML templates
 -------------------------

changelog.d/9003.misc

@@ -0,0 +1 @@
+Fix 'object name reserved for internal use' errors with recent versions of SQLite.

changelog.d/9123.misc

@@ -0,0 +1 @@
+Add experimental support for running Synapse with PyPy.

changelog.d/9150.feature

@@ -0,0 +1 @@
+New API /_synapse/admin/rooms/{roomId}/context/{eventId}.

changelog.d/9240.misc

@@ -0,0 +1 @@
+Deny access to additional IP addresses by default.

changelog.d/9257.bugfix

@@ -0,0 +1 @@
+Fix long-standing bug where sending email push would fail for rooms that the server had since left.

changelog.d/9291.doc

@@ -0,0 +1 @@
+Add note to `auto_join_rooms` config option explaining existing rooms must be publicly joinable.

changelog.d/9296.bugfix

@@ -0,0 +1 @@
+Fix bug in Synapse 1.27.0rc1 which meant the "session expired" error page during SSO registration was badly formatted.

changelog.d/9299.misc

@@ -0,0 +1 @@
+Update the `Cursor` type hints to better match PEP 249.

changelog.d/9300.feature

@@ -0,0 +1 @@
+Further improvements to the user experience of registration via single sign-on.

changelog.d/9301.feature

@@ -0,0 +1 @@
+Further improvements to the user experience of registration via single sign-on.

changelog.d/9305.misc

@@ -0,0 +1 @@
+Add debug logging for SRV lookups. Contributed by @Bubu.

changelog.d/9307.misc

@@ -0,0 +1 @@
+Improve logging for OIDC login flow.

changelog.d/9308.doc

@@ -0,0 +1 @@
+Correct name of Synapse's service file in TURN howto.

changelog.d/9311.feature

@@ -0,0 +1 @@
+Add hook to spam checker modules that allow checking file uploads and remote downloads.

changelog.d/9317.doc

@@ -0,0 +1 @@
+Fix the braces in the `oidc_providers` section of the sample config.

changelog.d/9321.bugfix

@@ -0,0 +1 @@
+Assert a maximum length for the `client_secret` parameter for spec compliance.

changelog.d/9322.doc

@@ -0,0 +1 @@
+Update installation instructions on Fedora.

changelog.d/9326.misc

@@ -0,0 +1 @@
+Share the code for handling required attributes between the CAS and SAML handlers.

changelog.d/9333.bugfix

@@ -0,0 +1 @@
+Fix additional errors when previewing URLs: "AttributeError 'NoneType' object has no attribute 'xpath'" and "ValueError: Unicode strings with encoding declaration are not supported. Please use bytes input or XML fragments without declaration.".

changelog.d/9361.bugfix

@@ -0,0 +1 @@
+Fix a bug causing Synapse to impose the wrong type constraints on fields when processing responses from appservices to `/_matrix/app/v1/thirdparty/user/{protocol}`.

changelog.d/9362.misc

@@ -0,0 +1 @@
+Clean up the code to load the metadata for OpenID Connect identity providers.

changelog.d/9376.feature

@@ -0,0 +1 @@
+Add support for receiving OpenID Connect authentication responses via form `POST`s rather than `GET`s.

changelog.d/9377.misc

@@ -0,0 +1 @@
+Convert tests to use `HomeserverTestCase`.

changelog.d/9381.misc

@@ -0,0 +1 @@
+Update the version of black used to 20.8b1.

changelog.d/9384.misc

@@ -0,0 +1 @@
+Allow OIDC config to override discovered values.

changelog.d/9391.bugfix

@@ -0,0 +1 @@
+Fix bug where Synapse would occaisonally stop reconnecting after the connection was lost.

changelog.d/9394.misc

@@ -0,0 +1 @@
+Remove some dead code from the acceptance of room invites path.

changelog.d/9395.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug when upgrading a room: "TypeError: '>' not supported between instances of 'NoneType' and 'int'".

changelog.d/9396.misc

@@ -0,0 +1 @@
+Convert tests to use `HomeserverTestCase`.

changelog.d/9404.doc

@@ -0,0 +1 @@
+Update docs for using Gitea as OpenID provider.

changelog.d/9407.doc

@@ -0,0 +1 @@
+Document that pusher instances are shardable.

changelog.d/9423.bugfix

@@ -0,0 +1 @@
+Fix building docker images for 32-bit ARM.

debian/changelog

@@ -1,9 +1,3 @@
-matrix-synapse-py3 (1.28.0) stable; urgency=medium
-
-  * New synapse release 1.28.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 25 Feb 2021 10:21:57 +0000
-
 matrix-synapse-py3 (1.27.0) stable; urgency=medium
 
   [ Dan Callahan ]

docker/Dockerfile

@@ -12,11 +12,13 @@
 #
 
 ARG PYTHON_VERSION=3.8
+ARG BASE_IMAGE=docker.io/python:${PYTHON_VERSION}-slim
+ARG CARGO_NET_OFFLINE=false
 
 ###
 ### Stage 0: builder
 ###
-FROM docker.io/python:${PYTHON_VERSION}-slim as builder
+FROM ${BASE_IMAGE} as builder
 
 # install the OS build deps
 RUN apt-get update && apt-get install -y \
@@ -32,9 +34,16 @@ RUN apt-get update && apt-get install -y \
     zlib1g-dev \
  && rm -rf /var/lib/apt/lists/*
 
+ENV CARGO_NET_OFFLINE=${CARGO_NET_OFFLINE}
+
 # Build dependencies that are not available as wheels, to speed up rebuilds
 RUN pip install --prefix="/install" --no-warn-script-location \
-        cryptography \
+        lxml
+
+RUN pip install --prefix="/install" --no-warn-script-location \
+        cryptography
+
+RUN pip install --prefix="/install" --no-warn-script-location \
         frozendict \
         jaeger-client \
         opentracing \

docker/Dockerfile-cargo-cache

@@ -0,0 +1,21 @@
+# A docker file that caches the cargo index for the cryptography deps. This is
+# mainly useful for multi-arch builds where fetching the index from the internet
+# fails for 32bit archs built on 64 bit platforms.
+
+ARG PYTHON_VERSION=3.8
+
+FROM --platform=$BUILDPLATFORM docker.io/python:${PYTHON_VERSION}-slim as builder
+
+RUN apt-get update && apt-get install -y \
+    rustc \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip download --no-binary cryptography --no-deps cryptography
+
+RUN tar -xf cryptography*.tar.gz --wildcards cryptography*/src/rust/
+
+RUN cd cryptography*/src/rust && cargo fetch
+
+FROM docker.io/python:${PYTHON_VERSION}-slim
+
+COPY --from=builder /root/.cargo /root/.cargo

docs/admin_api/user_admin_api.rst

@@ -29,9 +29,8 @@ It returns a JSON body like the following:
             }
         ],
         "avatar_url": "<avatar_url>",
-        "admin": 0,
-        "deactivated": 0,
-        "shadow_banned": 0,
+        "admin": false,
+        "deactivated": false,
         "password_hash": "$2b$12$p9B4GkqYdRTPGD",
         "creation_ts": 1560432506,
         "appservice_id": null,
@@ -151,7 +150,6 @@ A JSON body is returned with the following shape:
                 "admin": 0,
                 "user_type": null,
                 "deactivated": 0,
-                "shadow_banned": 0,
                 "displayname": "<User One>",
                 "avatar_url": null
             }, {
@@ -160,7 +158,6 @@ A JSON body is returned with the following shape:
                 "admin": 1,
                 "user_type": null,
                 "deactivated": 0,
-                "shadow_banned": 0,
                 "displayname": "<User Two>",
                 "avatar_url": "<avatar_url>"
             }
@@ -265,7 +262,7 @@ The following actions are performed when deactivating an user:
 - Reject all pending invites
 - Remove all account validity information related to the user
 
-The following additional actions are performed during deactivation if ``erase``
+The following additional actions are performed during deactivation if``erase``
 is set to ``true``:
 
 - Remove the user's display name

docs/reverse_proxy.md

@@ -40,12 +40,12 @@ the reverse proxy and the homeserver.
 
 ```
 server {
-    listen 443 ssl http2;
-    listen [::]:443 ssl http2;
+    listen 443 ssl;
+    listen [::]:443 ssl;
 
     # For the federation port
-    listen 8448 ssl http2 default_server;
-    listen [::]:8448 ssl http2 default_server;
+    listen 8448 ssl default_server;
+    listen [::]:8448 ssl default_server;
 
     server_name matrix.example.com;
 

docs/sample_config.yaml

@@ -2228,8 +2228,8 @@ password_config:
       #require_uppercase: true
 
 ui_auth:
-    # The amount of time to allow a user-interactive authentication session
-    # to be active.
+    # The number of milliseconds to allow a user-interactive authentication
+    # session to be active.
     #
     # This defaults to 0, meaning the user is queried for their credentials
     # before every action, but this can be overridden to allow a single
@@ -2240,7 +2240,7 @@ ui_auth:
     # Uncomment below to allow for credential validation to last for 15
     # seconds.
     #
-    #session_timeout: "15s"
+    #session_timeout: 15000
 
 
 # Configuration for sending emails from Synapse.

docs/workers.md

@@ -276,8 +276,7 @@ using):
 
 Ensure that all SSO logins go to a single process.
 For multiple workers not handling the SSO endpoints properly, see
-[#7530](https://github.com/matrix-org/synapse/issues/7530) and 
-[#9427](https://github.com/matrix-org/synapse/issues/9427).
+[#7530](https://github.com/matrix-org/synapse/issues/7530).
 
 Note that a HTTP listener with `client` and `federation` resources must be
 configured in the `worker_listeners` option in the worker config.

mypy.ini

@@ -23,7 +23,6 @@ files =
   synapse/events/validator.py,
   synapse/events/spamcheck.py,
   synapse/federation,
-  synapse/groups,
   synapse/handlers,
   synapse/http/client.py,
   synapse/http/federation/matrix_federation_agent.py,

synapse/__init__.py

@@ -48,7 +48,7 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.28.0"
+__version__ = "1.27.0"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when

synapse/api/constants.py

@@ -27,11 +27,6 @@ MAX_ALIAS_LENGTH = 255
 # the maximum length for a user id is 255 characters
 MAX_USERID_LENGTH = 255
 
-# The maximum length for a group id is 255 characters
-MAX_GROUPID_LENGTH = 255
-MAX_GROUP_CATEGORYID_LENGTH = 255
-MAX_GROUP_ROLEID_LENGTH = 255
-
 
 class Membership:
 

synapse/config/auth.py

@@ -37,9 +37,7 @@ class AuthConfig(Config):
 
         # User-interactive authentication
         ui_auth = config.get("ui_auth") or {}
-        self.ui_auth_session_timeout = self.parse_duration(
-            ui_auth.get("session_timeout", 0)
-        )
+        self.ui_auth_session_timeout = ui_auth.get("session_timeout", 0)
 
     def generate_config_section(self, config_dir_path, server_name, **kwargs):
         return """\
@@ -95,8 +93,8 @@ class AuthConfig(Config):
               #require_uppercase: true
 
         ui_auth:
-            # The amount of time to allow a user-interactive authentication session
-            # to be active.
+            # The number of milliseconds to allow a user-interactive authentication
+            # session to be active.
             #
             # This defaults to 0, meaning the user is queried for their credentials
             # before every action, but this can be overridden to allow a single
@@ -107,5 +105,5 @@ class AuthConfig(Config):
             # Uncomment below to allow for credential validation to last for 15
             # seconds.
             #
-            #session_timeout: "15s"
+            #session_timeout: 15000
         """

synapse/federation/transport/server.py

@@ -21,7 +21,6 @@ import re
 from typing import Optional, Tuple, Type
 
 import synapse
-from synapse.api.constants import MAX_GROUP_CATEGORYID_LENGTH, MAX_GROUP_ROLEID_LENGTH
 from synapse.api.errors import Codes, FederationDeniedError, SynapseError
 from synapse.api.room_versions import RoomVersions
 from synapse.api.urls import (
@@ -1119,17 +1118,7 @@ class FederationGroupsSummaryRoomsServlet(BaseFederationServlet):
             raise SynapseError(403, "requester_user_id doesn't match origin")
 
         if category_id == "":
-            raise SynapseError(
-                400, "category_id cannot be empty string", Codes.INVALID_PARAM
-            )
-
-        if len(category_id) > MAX_GROUP_CATEGORYID_LENGTH:
-            raise SynapseError(
-                400,
-                "category_id may not be longer than %s characters"
-                % (MAX_GROUP_CATEGORYID_LENGTH,),
-                Codes.INVALID_PARAM,
-            )
+            raise SynapseError(400, "category_id cannot be empty string")
 
         resp = await self.handler.update_group_summary_room(
             group_id,
@@ -1195,14 +1184,6 @@ class FederationGroupsCategoryServlet(BaseFederationServlet):
         if category_id == "":
             raise SynapseError(400, "category_id cannot be empty string")
 
-        if len(category_id) > MAX_GROUP_CATEGORYID_LENGTH:
-            raise SynapseError(
-                400,
-                "category_id may not be longer than %s characters"
-                % (MAX_GROUP_CATEGORYID_LENGTH,),
-                Codes.INVALID_PARAM,
-            )
-
         resp = await self.handler.upsert_group_category(
             group_id, requester_user_id, category_id, content
         )
@@ -1259,17 +1240,7 @@ class FederationGroupsRoleServlet(BaseFederationServlet):
             raise SynapseError(403, "requester_user_id doesn't match origin")
 
         if role_id == "":
-            raise SynapseError(
-                400, "role_id cannot be empty string", Codes.INVALID_PARAM
-            )
-
-        if len(role_id) > MAX_GROUP_ROLEID_LENGTH:
-            raise SynapseError(
-                400,
-                "role_id may not be longer than %s characters"
-                % (MAX_GROUP_ROLEID_LENGTH,),
-                Codes.INVALID_PARAM,
-            )
+            raise SynapseError(400, "role_id cannot be empty string")
 
         resp = await self.handler.update_group_role(
             group_id, requester_user_id, role_id, content
@@ -1314,14 +1285,6 @@ class FederationGroupsSummaryUsersServlet(BaseFederationServlet):
         if role_id == "":
             raise SynapseError(400, "role_id cannot be empty string")
 
-        if len(role_id) > MAX_GROUP_ROLEID_LENGTH:
-            raise SynapseError(
-                400,
-                "role_id may not be longer than %s characters"
-                % (MAX_GROUP_ROLEID_LENGTH,),
-                Codes.INVALID_PARAM,
-            )
-
         resp = await self.handler.update_group_summary_user(
             group_id,
             requester_user_id,

synapse/groups/attestations.py

@@ -37,16 +37,13 @@ An attestation is a signed blob of json that looks like:
 
 import logging
 import random
-from typing import TYPE_CHECKING, Optional, Tuple
+from typing import Tuple
 
 from signedjson.sign import sign_json
 
 from synapse.api.errors import HttpResponseException, RequestSendFailed, SynapseError
 from synapse.metrics.background_process_metrics import run_as_background_process
-from synapse.types import JsonDict, get_domain_from_id
-
-if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+from synapse.types import get_domain_from_id
 
 logger = logging.getLogger(__name__)
 
@@ -66,19 +63,15 @@ UPDATE_ATTESTATION_TIME_MS = 1 * 24 * 60 * 60 * 1000
 class GroupAttestationSigning:
     """Creates and verifies group attestations."""
 
-    def __init__(self, hs: "HomeServer"):
+    def __init__(self, hs):
         self.keyring = hs.get_keyring()
         self.clock = hs.get_clock()
         self.server_name = hs.hostname
         self.signing_key = hs.signing_key
 
     async def verify_attestation(
-        self,
-        attestation: JsonDict,
-        group_id: str,
-        user_id: str,
-        server_name: Optional[str] = None,
-    ) -> None:
+        self, attestation, group_id, user_id, server_name=None
+    ):
         """Verifies that the given attestation matches the given parameters.
 
         An optional server_name can be supplied to explicitly set which server's
@@ -107,18 +100,16 @@ class GroupAttestationSigning:
         if valid_until_ms < now:
             raise SynapseError(400, "Attestation expired")
 
-        assert server_name is not None
         await self.keyring.verify_json_for_server(
             server_name, attestation, now, "Group attestation"
         )
 
-    def create_attestation(self, group_id: str, user_id: str) -> JsonDict:
+    def create_attestation(self, group_id, user_id):
         """Create an attestation for the group_id and user_id with default
         validity length.
         """
-        validity_period = DEFAULT_ATTESTATION_LENGTH_MS * random.uniform(
-            *DEFAULT_ATTESTATION_JITTER
-        )
+        validity_period = DEFAULT_ATTESTATION_LENGTH_MS
+        validity_period *= random.uniform(*DEFAULT_ATTESTATION_JITTER)
         valid_until_ms = int(self.clock.time_msec() + validity_period)
 
         return sign_json(
@@ -135,7 +126,7 @@ class GroupAttestationSigning:
 class GroupAttestionRenewer:
     """Responsible for sending and receiving attestation updates."""
 
-    def __init__(self, hs: "HomeServer"):
+    def __init__(self, hs):
         self.clock = hs.get_clock()
         self.store = hs.get_datastore()
         self.assestations = hs.get_groups_attestation_signing()
@@ -148,9 +139,7 @@ class GroupAttestionRenewer:
                 self._start_renew_attestations, 30 * 60 * 1000
             )
 
-    async def on_renew_attestation(
-        self, group_id: str, user_id: str, content: JsonDict
-    ) -> JsonDict:
+    async def on_renew_attestation(self, group_id, user_id, content):
         """When a remote updates an attestation"""
         attestation = content["attestation"]
 
@@ -165,10 +154,10 @@ class GroupAttestionRenewer:
 
         return {}
 
-    def _start_renew_attestations(self) -> None:
+    def _start_renew_attestations(self):
         return run_as_background_process("renew_attestations", self._renew_attestations)
 
-    async def _renew_attestations(self) -> None:
+    async def _renew_attestations(self):
         """Called periodically to check if we need to update any of our attestations"""
 
         now = self.clock.time_msec()
@@ -177,7 +166,7 @@ class GroupAttestionRenewer:
             now + UPDATE_ATTESTATION_TIME_MS
         )
 
-        async def _renew_attestation(group_user: Tuple[str, str]) -> None:
+        async def _renew_attestation(group_user: Tuple[str, str]):
             group_id, user_id = group_user
             try:
                 if not self.is_mine_id(group_id):

synapse/groups/groups_server.py

@@ -16,17 +16,12 @@
 # limitations under the License.
 
 import logging
-from typing import TYPE_CHECKING, Optional
 
 from synapse.api.errors import Codes, SynapseError
-from synapse.handlers.groups_local import GroupsLocalHandler
 from synapse.handlers.profile import MAX_AVATAR_URL_LEN, MAX_DISPLAYNAME_LEN
-from synapse.types import GroupID, JsonDict, RoomID, UserID, get_domain_from_id
+from synapse.types import GroupID, RoomID, UserID, get_domain_from_id
 from synapse.util.async_helpers import concurrently_execute
 
-if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
-
 logger = logging.getLogger(__name__)
 
 
@@ -44,7 +39,7 @@ MAX_LONG_DESC_LEN = 10000
 
 
 class GroupsServerWorkerHandler:
-    def __init__(self, hs: "HomeServer"):
+    def __init__(self, hs):
         self.hs = hs
         self.store = hs.get_datastore()
         self.room_list_handler = hs.get_room_list_handler()
@@ -59,21 +54,16 @@ class GroupsServerWorkerHandler:
         self.profile_handler = hs.get_profile_handler()
 
     async def check_group_is_ours(
-        self,
-        group_id: str,
-        requester_user_id: str,
-        and_exists: bool = False,
-        and_is_admin: Optional[str] = None,
-    ) -> Optional[dict]:
+        self, group_id, requester_user_id, and_exists=False, and_is_admin=None
+    ):
         """Check that the group is ours, and optionally if it exists.
 
         If group does exist then return group.
 
         Args:
-            group_id: The group ID to check.
-            requester_user_id: The user ID of the requester.
-            and_exists: whether to also check if group exists
-            and_is_admin: whether to also check if given str is a user_id
+            group_id (str)
+            and_exists (bool): whether to also check if group exists
+            and_is_admin (str): whether to also check if given str is a user_id
                 that is an admin
         """
         if not self.is_mine_id(group_id):
@@ -96,9 +86,7 @@ class GroupsServerWorkerHandler:
 
         return group
 
-    async def get_group_summary(
-        self, group_id: str, requester_user_id: str
-    ) -> JsonDict:
+    async def get_group_summary(self, group_id, requester_user_id):
         """Get the summary for a group as seen by requester_user_id.
 
         The group summary consists of the profile of the room, and a curated
@@ -131,8 +119,6 @@ class GroupsServerWorkerHandler:
             entry = await self.room_list_handler.generate_room_entry(
                 room_id, len(joined_users), with_alias=False, allow_private=True
             )
-            if entry is None:
-                continue
             entry = dict(entry)  # so we don't change what's cached
             entry.pop("room_id", None)
 
@@ -140,22 +126,22 @@ class GroupsServerWorkerHandler:
 
         rooms.sort(key=lambda e: e.get("order", 0))
 
-        for user in users:
-            user_id = user["user_id"]
+        for entry in users:
+            user_id = entry["user_id"]
 
             if not self.is_mine_id(requester_user_id):
                 attestation = await self.store.get_remote_attestation(group_id, user_id)
                 if not attestation:
                     continue
 
-                user["attestation"] = attestation
+                entry["attestation"] = attestation
             else:
-                user["attestation"] = self.attestations.create_attestation(
+                entry["attestation"] = self.attestations.create_attestation(
                     group_id, user_id
                 )
 
             user_profile = await self.profile_handler.get_profile_from_cache(user_id)
-            user.update(user_profile)
+            entry.update(user_profile)
 
         users.sort(key=lambda e: e.get("order", 0))
 
@@ -178,43 +164,40 @@ class GroupsServerWorkerHandler:
             "user": membership_info,
         }
 
-    async def get_group_categories(
-        self, group_id: str, requester_user_id: str
-    ) -> JsonDict:
+    async def get_group_categories(self, group_id, requester_user_id):
         """Get all categories in a group (as seen by user)"""
         await self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
 
         categories = await self.store.get_group_categories(group_id=group_id)
         return {"categories": categories}
 
-    async def get_group_category(
-        self, group_id: str, requester_user_id: str, category_id: str
-    ) -> JsonDict:
+    async def get_group_category(self, group_id, requester_user_id, category_id):
         """Get a specific category in a group (as seen by user)"""
         await self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
 
-        return await self.store.get_group_category(
+        res = await self.store.get_group_category(
             group_id=group_id, category_id=category_id
         )
 
-    async def get_group_roles(self, group_id: str, requester_user_id: str) -> JsonDict:
+        logger.info("group %s", res)
+
+        return res
+
+    async def get_group_roles(self, group_id, requester_user_id):
         """Get all roles in a group (as seen by user)"""
         await self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
 
         roles = await self.store.get_group_roles(group_id=group_id)
         return {"roles": roles}
 
-    async def get_group_role(
-        self, group_id: str, requester_user_id: str, role_id: str
-    ) -> JsonDict:
+    async def get_group_role(self, group_id, requester_user_id, role_id):
         """Get a specific role in a group (as seen by user)"""
         await self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
 
-        return await self.store.get_group_role(group_id=group_id, role_id=role_id)
+        res = await self.store.get_group_role(group_id=group_id, role_id=role_id)
+        return res
 
-    async def get_group_profile(
-        self, group_id: str, requester_user_id: str
-    ) -> JsonDict:
+    async def get_group_profile(self, group_id, requester_user_id):
         """Get the group profile as seen by requester_user_id"""
 
         await self.check_group_is_ours(group_id, requester_user_id)
@@ -236,9 +219,7 @@ class GroupsServerWorkerHandler:
         else:
             raise SynapseError(404, "Unknown group")
 
-    async def get_users_in_group(
-        self, group_id: str, requester_user_id: str
-    ) -> JsonDict:
+    async def get_users_in_group(self, group_id, requester_user_id):
         """Get the users in group as seen by requester_user_id.
 
         The ordering is arbitrary at the moment
@@ -287,9 +268,7 @@ class GroupsServerWorkerHandler:
 
         return {"chunk": chunk, "total_user_count_estimate": len(user_results)}
 
-    async def get_invited_users_in_group(
-        self, group_id: str, requester_user_id: str
-    ) -> JsonDict:
+    async def get_invited_users_in_group(self, group_id, requester_user_id):
         """Get the users that have been invited to a group as seen by requester_user_id.
 
         The ordering is arbitrary at the moment
@@ -319,9 +298,7 @@ class GroupsServerWorkerHandler:
 
         return {"chunk": user_profiles, "total_user_count_estimate": len(invited_users)}
 
-    async def get_rooms_in_group(
-        self, group_id: str, requester_user_id: str
-    ) -> JsonDict:
+    async def get_rooms_in_group(self, group_id, requester_user_id):
         """Get the rooms in group as seen by requester_user_id
 
         This returns rooms in order of decreasing number of joined users
@@ -359,20 +336,15 @@ class GroupsServerWorkerHandler:
 
 
 class GroupsServerHandler(GroupsServerWorkerHandler):
-    def __init__(self, hs: "HomeServer"):
+    def __init__(self, hs):
         super().__init__(hs)
 
         # Ensure attestations get renewed
         hs.get_groups_attestation_renewer()
 
     async def update_group_summary_room(
-        self,
-        group_id: str,
-        requester_user_id: str,
-        room_id: str,
-        category_id: str,
-        content: JsonDict,
-    ) -> JsonDict:
+        self, group_id, requester_user_id, room_id, category_id, content
+    ):
         """Add/update a room to the group summary"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
@@ -395,8 +367,8 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
         return {}
 
     async def delete_group_summary_room(
-        self, group_id: str, requester_user_id: str, room_id: str, category_id: str
-    ) -> JsonDict:
+        self, group_id, requester_user_id, room_id, category_id
+    ):
         """Remove a room from the summary"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
@@ -408,9 +380,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
 
         return {}
 
-    async def set_group_join_policy(
-        self, group_id: str, requester_user_id: str, content: JsonDict
-    ) -> JsonDict:
+    async def set_group_join_policy(self, group_id, requester_user_id, content):
         """Sets the group join policy.
 
         Currently supported policies are:
@@ -430,8 +400,8 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
         return {}
 
     async def update_group_category(
-        self, group_id: str, requester_user_id: str, category_id: str, content: JsonDict
-    ) -> JsonDict:
+        self, group_id, requester_user_id, category_id, content
+    ):
         """Add/Update a group category"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
@@ -449,9 +419,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
 
         return {}
 
-    async def delete_group_category(
-        self, group_id: str, requester_user_id: str, category_id: str
-    ) -> JsonDict:
+    async def delete_group_category(self, group_id, requester_user_id, category_id):
         """Delete a group category"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
@@ -463,9 +431,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
 
         return {}
 
-    async def update_group_role(
-        self, group_id: str, requester_user_id: str, role_id: str, content: JsonDict
-    ) -> JsonDict:
+    async def update_group_role(self, group_id, requester_user_id, role_id, content):
         """Add/update a role in a group"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
@@ -481,9 +447,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
 
         return {}
 
-    async def delete_group_role(
-        self, group_id: str, requester_user_id: str, role_id: str
-    ) -> JsonDict:
+    async def delete_group_role(self, group_id, requester_user_id, role_id):
         """Remove role from group"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
@@ -494,13 +458,8 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
         return {}
 
     async def update_group_summary_user(
-        self,
-        group_id: str,
-        requester_user_id: str,
-        user_id: str,
-        role_id: str,
-        content: JsonDict,
-    ) -> JsonDict:
+        self, group_id, requester_user_id, user_id, role_id, content
+    ):
         """Add/update a users entry in the group summary"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
@@ -521,8 +480,8 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
         return {}
 
     async def delete_group_summary_user(
-        self, group_id: str, requester_user_id: str, user_id: str, role_id: str
-    ) -> JsonDict:
+        self, group_id, requester_user_id, user_id, role_id
+    ):
         """Remove a user from the group summary"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
@@ -534,9 +493,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
 
         return {}
 
-    async def update_group_profile(
-        self, group_id: str, requester_user_id: str, content: JsonDict
-    ) -> None:
+    async def update_group_profile(self, group_id, requester_user_id, content):
         """Update the group profile"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
@@ -567,9 +524,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
 
         await self.store.update_group_profile(group_id, profile)
 
-    async def add_room_to_group(
-        self, group_id: str, requester_user_id: str, room_id: str, content: JsonDict
-    ) -> JsonDict:
+    async def add_room_to_group(self, group_id, requester_user_id, room_id, content):
         """Add room to group"""
         RoomID.from_string(room_id)  # Ensure valid room id
 
@@ -584,13 +539,8 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
         return {}
 
     async def update_room_in_group(
-        self,
-        group_id: str,
-        requester_user_id: str,
-        room_id: str,
-        config_key: str,
-        content: JsonDict,
-    ) -> JsonDict:
+        self, group_id, requester_user_id, room_id, config_key, content
+    ):
         """Update room in group"""
         RoomID.from_string(room_id)  # Ensure valid room id
 
@@ -609,9 +559,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
 
         return {}
 
-    async def remove_room_from_group(
-        self, group_id: str, requester_user_id: str, room_id: str
-    ) -> JsonDict:
+    async def remove_room_from_group(self, group_id, requester_user_id, room_id):
         """Remove room from group"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
@@ -621,16 +569,12 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
 
         return {}
 
-    async def invite_to_group(
-        self, group_id: str, user_id: str, requester_user_id: str, content: JsonDict
-    ) -> JsonDict:
+    async def invite_to_group(self, group_id, user_id, requester_user_id, content):
         """Invite user to group"""
 
         group = await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
         )
-        if not group:
-            raise SynapseError(400, "Group does not exist", errcode=Codes.BAD_STATE)
 
         # TODO: Check if user knocked
 
@@ -653,9 +597,6 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
 
         if self.hs.is_mine_id(user_id):
             groups_local = self.hs.get_groups_local_handler()
-            assert isinstance(
-                groups_local, GroupsLocalHandler
-            ), "Workers cannot invites users to groups."
             res = await groups_local.on_invite(group_id, user_id, content)
             local_attestation = None
         else:
@@ -691,7 +632,6 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
                 local_attestation=local_attestation,
                 remote_attestation=remote_attestation,
             )
-            return {"state": "join"}
         elif res["state"] == "invite":
             await self.store.add_group_invite(group_id, user_id)
             return {"state": "invite"}
@@ -700,17 +640,13 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
         else:
             raise SynapseError(502, "Unknown state returned by HS")
 
-    async def _add_user(
-        self, group_id: str, user_id: str, content: JsonDict
-    ) -> Optional[JsonDict]:
+    async def _add_user(self, group_id, user_id, content):
         """Add a user to a group based on a content dict.
 
         See accept_invite, join_group.
         """
         if not self.hs.is_mine_id(user_id):
-            local_attestation = self.attestations.create_attestation(
-                group_id, user_id
-            )  # type: Optional[JsonDict]
+            local_attestation = self.attestations.create_attestation(group_id, user_id)
 
             remote_attestation = content["attestation"]
 
@@ -734,9 +670,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
 
         return local_attestation
 
-    async def accept_invite(
-        self, group_id: str, requester_user_id: str, content: JsonDict
-    ) -> JsonDict:
+    async def accept_invite(self, group_id, requester_user_id, content):
         """User tries to accept an invite to the group.
 
         This is different from them asking to join, and so should error if no
@@ -755,9 +689,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
 
         return {"state": "join", "attestation": local_attestation}
 
-    async def join_group(
-        self, group_id: str, requester_user_id: str, content: JsonDict
-    ) -> JsonDict:
+    async def join_group(self, group_id, requester_user_id, content):
         """User tries to join the group.
 
         This will error if the group requires an invite/knock to join
@@ -766,8 +698,6 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
         group_info = await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True
         )
-        if not group_info:
-            raise SynapseError(404, "Group does not exist", errcode=Codes.NOT_FOUND)
         if group_info["join_policy"] != "open":
             raise SynapseError(403, "Group is not publicly joinable")
 
@@ -775,9 +705,25 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
 
         return {"state": "join", "attestation": local_attestation}
 
+    async def knock(self, group_id, requester_user_id, content):
+        """A user requests becoming a member of the group"""
+        await self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
+
+        raise NotImplementedError()
+
+    async def accept_knock(self, group_id, requester_user_id, content):
+        """Accept a users knock to the room.
+
+        Errors if the user hasn't knocked, rather than inviting them.
+        """
+
+        await self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
+
+        raise NotImplementedError()
+
     async def remove_user_from_group(
-        self, group_id: str, user_id: str, requester_user_id: str, content: JsonDict
-    ) -> JsonDict:
+        self, group_id, user_id, requester_user_id, content
+    ):
         """Remove a user from the group; either a user is leaving or an admin
         kicked them.
         """
@@ -799,9 +745,6 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
         if is_kick:
             if self.hs.is_mine_id(user_id):
                 groups_local = self.hs.get_groups_local_handler()
-                assert isinstance(
-                    groups_local, GroupsLocalHandler
-                ), "Workers cannot remove users from groups."
                 await groups_local.user_removed_from_group(group_id, user_id, {})
             else:
                 await self.transport_client.remove_user_from_group_notification(
@@ -818,15 +761,14 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
 
         return {}
 
-    async def create_group(
-        self, group_id: str, requester_user_id: str, content: JsonDict
-    ) -> JsonDict:
+    async def create_group(self, group_id, requester_user_id, content):
+        group = await self.check_group_is_ours(group_id, requester_user_id)
+
         logger.info("Attempting to create group with ID: %r", group_id)
 
         # parsing the id into a GroupID validates it.
         group_id_obj = GroupID.from_string(group_id)
 
-        group = await self.check_group_is_ours(group_id, requester_user_id)
         if group:
             raise SynapseError(400, "Group already exists")
 
@@ -871,7 +813,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
 
             local_attestation = self.attestations.create_attestation(
                 group_id, requester_user_id
-            )  # type: Optional[JsonDict]
+            )
         else:
             local_attestation = None
             remote_attestation = None
@@ -894,14 +836,15 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
 
         return {"group_id": group_id}
 
-    async def delete_group(self, group_id: str, requester_user_id: str) -> None:
+    async def delete_group(self, group_id, requester_user_id):
         """Deletes a group, kicking out all current members.
 
         Only group admins or server admins can call this request
 
         Args:
-            group_id: The group ID to delete.
-            requester_user_id: The user requesting to delete the group.
+            group_id (str)
+            request_user_id (str)
+
         """
 
         await self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
@@ -924,9 +867,6 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
         async def _kick_user_from_group(user_id):
             if self.hs.is_mine_id(user_id):
                 groups_local = self.hs.get_groups_local_handler()
-                assert isinstance(
-                    groups_local, GroupsLocalHandler
-                ), "Workers cannot kick users from groups."
                 await groups_local.user_removed_from_group(group_id, user_id, {})
             else:
                 await self.transport_client.remove_user_from_group_notification(
@@ -958,7 +898,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
         await self.store.delete_group(group_id)
 
 
-def _parse_join_policy_from_contents(content: JsonDict) -> Optional[str]:
+def _parse_join_policy_from_contents(content):
     """Given a content for a request, return the specified join policy or None"""
 
     join_policy_dict = content.get("m.join_policy")
@@ -968,7 +908,7 @@ def _parse_join_policy_from_contents(content: JsonDict) -> Optional[str]:
         return None
 
 
-def _parse_join_policy_dict(join_policy_dict: JsonDict) -> str:
+def _parse_join_policy_dict(join_policy_dict):
     """Given a dict for the "m.join_policy" config return the join policy specified"""
     join_policy_type = join_policy_dict.get("type")
     if not join_policy_type:
@@ -979,7 +919,7 @@ def _parse_join_policy_dict(join_policy_dict: JsonDict) -> str:
     return join_policy_type
 
 
-def _parse_visibility_from_contents(content: JsonDict) -> bool:
+def _parse_visibility_from_contents(content):
     """Given a content for a request parse out whether the entity should be
     public or not
     """
@@ -993,7 +933,7 @@ def _parse_visibility_from_contents(content: JsonDict) -> bool:
     return is_public
 
 
-def _parse_visibility_dict(visibility: JsonDict) -> bool:
+def _parse_visibility_dict(visibility):
     """Given a dict for the "m.visibility" config return if the entity should
     be public or not
     """

synapse/handlers/presence.py

@@ -349,13 +349,10 @@ class PresenceHandler(BasePresenceHandler):
                 [self.user_to_current_state[user_id] for user_id in unpersisted]
             )
 
-    async def _update_states(self, new_states: Iterable[UserPresenceState]) -> None:
+    async def _update_states(self, new_states):
         """Updates presence of users. Sets the appropriate timeouts. Pokes
         the notifier and federation if and only if the changed presence state
         should be sent to clients/servers.
-
-        Args:
-            new_states: The new user presence state updates to process.
         """
         now = self.clock.time_msec()
 
@@ -371,7 +368,7 @@ class PresenceHandler(BasePresenceHandler):
             new_states_dict = {}
             for new_state in new_states:
                 new_states_dict[new_state.user_id] = new_state
-            new_states = new_states_dict.values()
+            new_state = new_states_dict.values()
 
             for new_state in new_states:
                 user_id = new_state.user_id
@@ -660,6 +657,17 @@ class PresenceHandler(BasePresenceHandler):
 
         self._push_to_remotes(states)
 
+    async def notify_for_states(self, state, stream_id):
+        parties = await get_interested_parties(self.store, [state])
+        room_ids_to_states, users_to_states = parties
+
+        self.notifier.on_new_event(
+            "presence_key",
+            stream_id,
+            rooms=room_ids_to_states.keys(),
+            users=[UserID.from_string(u) for u in users_to_states],
+        )
+
     def _push_to_remotes(self, states):
         """Sends state updates to remote servers.
 

synapse/handlers/user_directory.py

@@ -143,10 +143,6 @@ class UserDirectoryHandler(StateDeltasHandler):
         if self.pos is None:
             self.pos = await self.store.get_user_directory_stream_pos()
 
-        # If still None then the initial background update hasn't happened yet.
-        if self.pos is None:
-            return None
-
         # Loop round handling deltas until we're up to date
         while True:
             with Measure(self.clock, "user_dir_delta"):

synapse/http/client.py

@@ -56,7 +56,7 @@ from twisted.web.client import (
 )
 from twisted.web.http import PotentialDataLoss
 from twisted.web.http_headers import Headers
-from twisted.web.iweb import UNKNOWN_LENGTH, IAgent, IBodyProducer, IResponse
+from twisted.web.iweb import IAgent, IBodyProducer, IResponse
 
 from synapse.api.errors import Codes, HttpResponseException, SynapseError
 from synapse.http import QuieterFileBodyProducer, RequestTimedOutError, redact_uri
@@ -408,9 +408,6 @@ class SimpleHttpClient:
                     agent=self.agent,
                     data=body_producer,
                     headers=headers,
-                    # Avoid buffering the body in treq since we do not reuse
-                    # response bodies.
-                    unbuffered=True,
                     **self._extra_treq_args,
                 )  # type: defer.Deferred
 
@@ -705,6 +702,18 @@ class SimpleHttpClient:
 
         resp_headers = dict(response.headers.getAllRawHeaders())
 
+        if (
+            b"Content-Length" in resp_headers
+            and max_size
+            and int(resp_headers[b"Content-Length"][0]) > max_size
+        ):
+            logger.warning("Requested URL is too large > %r bytes" % (max_size,))
+            raise SynapseError(
+                502,
+                "Requested file is too large > %r bytes" % (max_size,),
+                Codes.TOO_LARGE,
+            )
+
         if response.code > 299:
             logger.warning("Got %d when downloading %s" % (response.code, url))
             raise SynapseError(502, "Got error %d" % (response.code,), Codes.UNKNOWN)
@@ -771,9 +780,7 @@ class _ReadBodyWithMaxSizeProtocol(protocol.Protocol):
         # in the meantime.
         if self.max_size is not None and self.length >= self.max_size:
             self.deferred.errback(BodyExceededMaxSize())
-            # Close the connection (forcefully) since all the data will get
-            # discarded anyway.
-            self.transport.abortConnection()
+            self.transport.loseConnection()
 
     def connectionLost(self, reason: Failure) -> None:
         # If the maximum size was already exceeded, there's nothing to do.
@@ -807,11 +814,6 @@ def read_body_with_max_size(
     Returns:
         A Deferred which resolves to the length of the read body.
     """
-    # If the Content-Length header gives a size larger than the maximum allowed
-    # size, do not bother downloading the body.
-    if max_size is not None and response.length != UNKNOWN_LENGTH:
-        if response.length > max_size:
-            return defer.fail(BodyExceededMaxSize())
 
     d = defer.Deferred()
     response.deliverBody(_ReadBodyWithMaxSizeProtocol(stream, d, max_size))

synapse/rest/client/v2_alpha/groups.py

@@ -16,16 +16,11 @@
 
 import logging
 from functools import wraps
-from typing import TYPE_CHECKING, Optional, Tuple
+from typing import TYPE_CHECKING, Tuple
 
 from twisted.web.http import Request
 
-from synapse.api.constants import (
-    MAX_GROUP_CATEGORYID_LENGTH,
-    MAX_GROUP_ROLEID_LENGTH,
-    MAX_GROUPID_LENGTH,
-)
-from synapse.api.errors import Codes, SynapseError
+from synapse.api.errors import SynapseError
 from synapse.handlers.groups_local import GroupsLocalHandler
 from synapse.http.servlet import (
     RestServlet,
@@ -89,9 +84,7 @@ class GroupServlet(RestServlet):
         assert_params_in_dict(
             content, ("name", "avatar_url", "short_description", "long_description")
         )
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot create group profiles."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         await self.groups_handler.update_group_profile(
             group_id, requester_user_id, content
         )
@@ -144,26 +137,13 @@ class GroupSummaryRoomsCatServlet(RestServlet):
 
     @_validate_group_id
     async def on_PUT(
-        self, request: Request, group_id: str, category_id: Optional[str], room_id: str
+        self, request: Request, group_id: str, category_id: str, room_id: str
     ):
         requester = await self.auth.get_user_by_req(request)
         requester_user_id = requester.user.to_string()
 
-        if category_id == "":
-            raise SynapseError(400, "category_id cannot be empty", Codes.INVALID_PARAM)
-
-        if category_id and len(category_id) > MAX_GROUP_CATEGORYID_LENGTH:
-            raise SynapseError(
-                400,
-                "category_id may not be longer than %s characters"
-                % (MAX_GROUP_CATEGORYID_LENGTH,),
-                Codes.INVALID_PARAM,
-            )
-
         content = parse_json_object_from_request(request)
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot modify group summaries."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         resp = await self.groups_handler.update_group_summary_room(
             group_id,
             requester_user_id,
@@ -181,9 +161,7 @@ class GroupSummaryRoomsCatServlet(RestServlet):
         requester = await self.auth.get_user_by_req(request)
         requester_user_id = requester.user.to_string()
 
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot modify group profiles."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         resp = await self.groups_handler.delete_group_summary_room(
             group_id, requester_user_id, room_id=room_id, category_id=category_id
         )
@@ -224,21 +202,8 @@ class GroupCategoryServlet(RestServlet):
         requester = await self.auth.get_user_by_req(request)
         requester_user_id = requester.user.to_string()
 
-        if not category_id:
-            raise SynapseError(400, "category_id cannot be empty", Codes.INVALID_PARAM)
-
-        if len(category_id) > MAX_GROUP_CATEGORYID_LENGTH:
-            raise SynapseError(
-                400,
-                "category_id may not be longer than %s characters"
-                % (MAX_GROUP_CATEGORYID_LENGTH,),
-                Codes.INVALID_PARAM,
-            )
-
         content = parse_json_object_from_request(request)
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot modify group categories."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         resp = await self.groups_handler.update_group_category(
             group_id, requester_user_id, category_id=category_id, content=content
         )
@@ -252,9 +217,7 @@ class GroupCategoryServlet(RestServlet):
         requester = await self.auth.get_user_by_req(request)
         requester_user_id = requester.user.to_string()
 
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot modify group categories."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         resp = await self.groups_handler.delete_group_category(
             group_id, requester_user_id, category_id=category_id
         )
@@ -316,21 +279,8 @@ class GroupRoleServlet(RestServlet):
         requester = await self.auth.get_user_by_req(request)
         requester_user_id = requester.user.to_string()
 
-        if not role_id:
-            raise SynapseError(400, "role_id cannot be empty", Codes.INVALID_PARAM)
-
-        if len(role_id) > MAX_GROUP_ROLEID_LENGTH:
-            raise SynapseError(
-                400,
-                "role_id may not be longer than %s characters"
-                % (MAX_GROUP_ROLEID_LENGTH,),
-                Codes.INVALID_PARAM,
-            )
-
         content = parse_json_object_from_request(request)
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot modify group roles."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         resp = await self.groups_handler.update_group_role(
             group_id, requester_user_id, role_id=role_id, content=content
         )
@@ -344,9 +294,7 @@ class GroupRoleServlet(RestServlet):
         requester = await self.auth.get_user_by_req(request)
         requester_user_id = requester.user.to_string()
 
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot modify group roles."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         resp = await self.groups_handler.delete_group_role(
             group_id, requester_user_id, role_id=role_id
         )
@@ -399,26 +347,13 @@ class GroupSummaryUsersRoleServlet(RestServlet):
 
     @_validate_group_id
     async def on_PUT(
-        self, request: Request, group_id: str, role_id: Optional[str], user_id: str
+        self, request: Request, group_id: str, role_id: str, user_id: str
     ) -> Tuple[int, JsonDict]:
         requester = await self.auth.get_user_by_req(request)
         requester_user_id = requester.user.to_string()
 
-        if role_id == "":
-            raise SynapseError(400, "role_id cannot be empty", Codes.INVALID_PARAM)
-
-        if role_id and len(role_id) > MAX_GROUP_ROLEID_LENGTH:
-            raise SynapseError(
-                400,
-                "role_id may not be longer than %s characters"
-                % (MAX_GROUP_ROLEID_LENGTH,),
-                Codes.INVALID_PARAM,
-            )
-
         content = parse_json_object_from_request(request)
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot modify group summaries."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         resp = await self.groups_handler.update_group_summary_user(
             group_id,
             requester_user_id,
@@ -436,9 +371,7 @@ class GroupSummaryUsersRoleServlet(RestServlet):
         requester = await self.auth.get_user_by_req(request)
         requester_user_id = requester.user.to_string()
 
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot modify group summaries."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         resp = await self.groups_handler.delete_group_summary_user(
             group_id, requester_user_id, user_id=user_id, role_id=role_id
         )
@@ -532,9 +465,7 @@ class GroupSettingJoinPolicyServlet(RestServlet):
 
         content = parse_json_object_from_request(request)
 
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot modify group join policy."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         result = await self.groups_handler.set_group_join_policy(
             group_id, requester_user_id, content
         )
@@ -563,19 +494,7 @@ class GroupCreateServlet(RestServlet):
         localpart = content.pop("localpart")
         group_id = GroupID(localpart, self.server_name).to_string()
 
-        if not localpart:
-            raise SynapseError(400, "Group ID cannot be empty", Codes.INVALID_PARAM)
-
-        if len(group_id) > MAX_GROUPID_LENGTH:
-            raise SynapseError(
-                400,
-                "Group ID may not be longer than %s characters" % (MAX_GROUPID_LENGTH,),
-                Codes.INVALID_PARAM,
-            )
-
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot create groups."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         result = await self.groups_handler.create_group(
             group_id, requester_user_id, content
         )
@@ -604,9 +523,7 @@ class GroupAdminRoomsServlet(RestServlet):
         requester_user_id = requester.user.to_string()
 
         content = parse_json_object_from_request(request)
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot modify rooms in a group."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         result = await self.groups_handler.add_room_to_group(
             group_id, requester_user_id, room_id, content
         )
@@ -620,9 +537,7 @@ class GroupAdminRoomsServlet(RestServlet):
         requester = await self.auth.get_user_by_req(request)
         requester_user_id = requester.user.to_string()
 
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot modify group categories."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         result = await self.groups_handler.remove_room_from_group(
             group_id, requester_user_id, room_id
         )
@@ -652,9 +567,7 @@ class GroupAdminRoomsConfigServlet(RestServlet):
         requester_user_id = requester.user.to_string()
 
         content = parse_json_object_from_request(request)
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot modify group categories."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         result = await self.groups_handler.update_room_in_group(
             group_id, requester_user_id, room_id, config_key, content
         )
@@ -684,9 +597,7 @@ class GroupAdminUsersInviteServlet(RestServlet):
 
         content = parse_json_object_from_request(request)
         config = content.get("config", {})
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot invite users to a group."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         result = await self.groups_handler.invite(
             group_id, user_id, requester_user_id, config
         )
@@ -713,9 +624,7 @@ class GroupAdminUsersKickServlet(RestServlet):
         requester_user_id = requester.user.to_string()
 
         content = parse_json_object_from_request(request)
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot kick users from a group."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         result = await self.groups_handler.remove_user_from_group(
             group_id, user_id, requester_user_id, content
         )
@@ -740,9 +649,7 @@ class GroupSelfLeaveServlet(RestServlet):
         requester_user_id = requester.user.to_string()
 
         content = parse_json_object_from_request(request)
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot leave a group for a users."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         result = await self.groups_handler.remove_user_from_group(
             group_id, requester_user_id, requester_user_id, content
         )
@@ -767,9 +674,7 @@ class GroupSelfJoinServlet(RestServlet):
         requester_user_id = requester.user.to_string()
 
         content = parse_json_object_from_request(request)
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot join a user to a group."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         result = await self.groups_handler.join_group(
             group_id, requester_user_id, content
         )
@@ -794,9 +699,7 @@ class GroupSelfAcceptInviteServlet(RestServlet):
         requester_user_id = requester.user.to_string()
 
         content = parse_json_object_from_request(request)
-        assert isinstance(
-            self.groups_handler, GroupsLocalHandler
-        ), "Workers cannot accept an invite to a group."
+        assert isinstance(self.groups_handler, GroupsLocalHandler)
         result = await self.groups_handler.accept_invite(
             group_id, requester_user_id, content
         )

synapse/rest/synapse/client/__init__.py

@@ -58,7 +58,6 @@ def build_synapse_client_resource_tree(hs: "HomeServer") -> Mapping[str, Resourc
         resources["/_synapse/client/saml2"] = res
 
         # This is also mounted under '/_matrix' for backwards-compatibility.
-        # To be removed in Synapse v1.32.0.
         resources["/_matrix/saml2"] = res
 
     return resources

synapse/storage/databases/main/__init__.py

@@ -340,7 +340,7 @@ class DataStore(
             count = txn.fetchone()[0]
 
             sql = (
-                "SELECT name, user_type, is_guest, admin, deactivated, shadow_banned, displayname, avatar_url "
+                "SELECT name, user_type, is_guest, admin, deactivated, displayname, avatar_url "
                 + sql_base
                 + " ORDER BY u.name LIMIT ? OFFSET ?"
             )

synapse/storage/databases/main/group_server.py

@@ -14,9 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Any, Dict, List, Optional, Tuple
-
-from typing_extensions import TypedDict
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 from synapse.api.errors import SynapseError
 from synapse.storage._base import SQLBaseStore, db_to_json
@@ -28,9 +26,6 @@ from synapse.util import json_encoder
 _DEFAULT_CATEGORY_ID = ""
 _DEFAULT_ROLE_ID = ""
 
-# A room in a group.
-_RoomInGroup = TypedDict("_RoomInGroup", {"room_id": str, "is_public": bool})
-
 
 class GroupServerWorkerStore(SQLBaseStore):
     async def get_group(self, group_id: str) -> Optional[Dict[str, Any]]:
@@ -77,7 +72,7 @@ class GroupServerWorkerStore(SQLBaseStore):
 
     async def get_rooms_in_group(
         self, group_id: str, include_private: bool = False
-    ) -> List[_RoomInGroup]:
+    ) -> List[Dict[str, Union[str, bool]]]:
         """Retrieve the rooms that belong to a given group. Does not return rooms that
         lack members.
 

synapse/storage/databases/main/registration.py

@@ -113,7 +113,6 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore):
                 "creation_ts",
                 "user_type",
                 "deactivated",
-                "shadow_banned",
             ],
             allow_none=True,
             desc="get_user_by_id",
@@ -373,25 +372,23 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore):
         """
 
         def set_shadow_banned_txn(txn):
-            user_id = user.to_string()
             self.db_pool.simple_update_one_txn(
                 txn,
                 table="users",
-                keyvalues={"name": user_id},
+                keyvalues={"name": user.to_string()},
                 updatevalues={"shadow_banned": shadow_banned},
             )
             # In order for this to apply immediately, clear the cache for this user.
             tokens = self.db_pool.simple_select_onecol_txn(
                 txn,
                 table="access_tokens",
-                keyvalues={"user_id": user_id},
+                keyvalues={"user_id": user.to_string()},
                 retcol="token",
             )
             for token in tokens:
                 self._invalidate_cache_and_stream(
                     txn, self.get_user_by_access_token, (token,)
                 )
-            self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,))
 
         await self.db_pool.runInteraction("set_shadow_banned", set_shadow_banned_txn)
 

synapse/storage/databases/main/user_directory.py

@@ -707,13 +707,7 @@ class UserDirectoryStore(UserDirectoryBackgroundUpdateStore):
 
         return {row["room_id"] for row in rows}
 
-    async def get_user_directory_stream_pos(self) -> Optional[int]:
-        """
-        Get the stream ID of the user directory stream.
-
-        Returns:
-            The stream token or None if the initial background update hasn't happened yet.
-        """
+    async def get_user_directory_stream_pos(self) -> int:
         return await self.db_pool.simple_select_one_onecol(
             table="user_directory_stream_pos",
             keyvalues={},

tests/http/test_client.py

@@ -18,7 +18,6 @@ from mock import Mock
 
 from twisted.python.failure import Failure
 from twisted.web.client import ResponseDone
-from twisted.web.iweb import UNKNOWN_LENGTH
 
 from synapse.http.client import BodyExceededMaxSize, read_body_with_max_size
 
@@ -28,12 +27,12 @@ from tests.unittest import TestCase
 class ReadBodyWithMaxSizeTests(TestCase):
     def setUp(self):
         """Start reading the body, returns the response, result and proto"""
-        response = Mock(length=UNKNOWN_LENGTH)
+        self.response = Mock()
         self.result = BytesIO()
-        self.deferred = read_body_with_max_size(response, self.result, 6)
+        self.deferred = read_body_with_max_size(self.response, self.result, 6)
 
         # Fish the protocol out of the response.
-        self.protocol = response.deliverBody.call_args[0][0]
+        self.protocol = self.response.deliverBody.call_args[0][0]
         self.protocol.transport = Mock()
 
     def _cleanup_error(self):
@@ -89,7 +88,7 @@ class ReadBodyWithMaxSizeTests(TestCase):
         self.protocol.dataReceived(b"1234567890")
         self.assertIsInstance(self.deferred.result, Failure)
         self.assertIsInstance(self.deferred.result.value, BodyExceededMaxSize)
-        self.protocol.transport.abortConnection.assert_called_once()
+        self.protocol.transport.loseConnection.assert_called_once()
 
         # More data might have come in.
         self.protocol.dataReceived(b"1234567890")

tests/rest/admin/test_user.py

@@ -769,7 +769,6 @@ class UsersListTestCase(unittest.HomeserverTestCase):
             self.assertIn("admin", u)
             self.assertIn("user_type", u)
             self.assertIn("deactivated", u)
-            self.assertIn("shadow_banned", u)
             self.assertIn("displayname", u)
             self.assertIn("avatar_url", u)
 
@@ -1147,7 +1146,6 @@ class UserRestTestCase(unittest.HomeserverTestCase):
         self.assertEqual(False, channel.json_body["admin"])
         self.assertEqual(False, channel.json_body["is_guest"])
         self.assertEqual(False, channel.json_body["deactivated"])
-        self.assertEqual(False, channel.json_body["shadow_banned"])
         self.assertEqual("mxc://fibble/wibble", channel.json_body["avatar_url"])
 
     @override_config(

tests/rest/client/v2_alpha/test_auth.py

@@ -343,7 +343,7 @@ class UIAuthTests(unittest.HomeserverTestCase):
             },
         )
 
-    @unittest.override_config({"ui_auth": {"session_timeout": "5s"}})
+    @unittest.override_config({"ui_auth": {"session_timeout": 5 * 1000}})
     def test_can_reuse_session(self):
         """
         The session can be reused if configured.

tests/storage/test_registration.py

@@ -52,7 +52,6 @@ class RegistrationStoreTestCase(unittest.TestCase):
                 "creation_ts": 1000,
                 "user_type": None,
                 "deactivated": 0,
-                "shadow_banned": 0,
             },
             (yield defer.ensureDeferred(self.store.get_user_by_id(self.user_id))),
         )