tests.utils passes disallow-untyped-defs

`time_ms` returns an int like the proper Synapse `Clock`

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -1,72 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-
----
-
-<!--
-
-**THIS IS NOT A SUPPORT CHANNEL!**
-**IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**,
-please ask in **#synapse:matrix.org** (using a matrix.org account if necessary)
-
-If you want to report a security issue, please see https://matrix.org/security-disclosure-policy/
-
-This is a bug report template. By following the instructions below and
-filling out the sections with your information, you will help the us to get all
-the necessary data to fix your issue.
-
-You can also preview your report before submitting it. You may remove sections
-that aren't relevant to your particular case.
-
-Text between <!-- and --​> marks will be invisible in the report.
-
--->
-
-### Description
-
-<!-- Describe here the problem that you are experiencing -->
-
-### Steps to reproduce
-
-- list the steps
-- that reproduce the bug
-- using hyphens as bullet points
-
-<!--
-Describe how what happens differs from what you expected.
-
-If you can identify any relevant log snippets from _homeserver.log_, please include
-those (please be careful to remove any personal or private data). Please surround them with
-``` (three backticks, on a line on their own), so that they are formatted legibly.
--->
-
-### Version information
-
-<!-- IMPORTANT: please answer the following questions, to help us narrow down the problem -->
-
-<!-- Was this issue identified on matrix.org or another homeserver? -->
-- **Homeserver**:
-
-If not matrix.org:
-
-<!--
- What version of Synapse is running?
-
-You can find the Synapse version with this command:
-
-$ curl http://localhost:8008/_synapse/admin/v1/server_version
-
-(You may need to replace `localhost:8008` if Synapse is not configured to
-listen on that port.)
--->
-- **Version**:
-
-- **Install method**:
-<!-- examples: package manager/git clone/pip  -->
-
-- **Platform**:
-<!--
-Tell us about the environment in which your homeserver is operating
-distro, hardware, if it's running in a vm/container, etc.
--->

.github/ISSUE_TEMPLATE/BUG_REPORT.yml

@@ -0,0 +1,103 @@
+name: Bug report
+description: Create a report to help us improve
+body:
+  - type: markdown
+    attributes:
+      value: |
+        **THIS IS NOT A SUPPORT CHANNEL!**
+        **IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**, please ask in **[#synapse:matrix.org](https://matrix.to/#/#synapse:matrix.org)** (using a matrix.org account if necessary).
+
+        If you want to report a security issue, please see https://matrix.org/security-disclosure-policy/
+
+        This is a bug report form. By following the instructions below and completing the sections with your information, you will help the us to get all the necessary data to fix your issue.
+
+        You can also preview your report before submitting it.
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: Describe the problem that you are experiencing
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction_steps
+    attributes:
+      label: Steps to reproduce
+      description: |
+        Describe the series of steps that leads you to the problem.
+
+        Describe how what happens differs from what you expected.
+      placeholder: Tell us what you see!
+      value: |
+        - list the steps
+        - that reproduce the bug
+        - using hyphens as bullet points
+    validations:
+      required: true
+  - type: markdown
+    attributes:
+      value: |
+        ---
+
+        **IMPORTANT**: please answer the following questions, to help us narrow down the problem.
+  - type: input
+    id: homeserver
+    attributes:
+      label: Homeserver
+      description: Which homeserver was this issue identified on? (matrix.org, another homeserver, etc)
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: Synapse Version
+      description: |
+        What version of Synapse is this homeserver running?
+
+        You can find the Synapse version by visiting https://yourserver.example.com/_matrix/federation/v1/version
+
+        or with this command:
+
+        ```
+        $ curl http://localhost:8008/_synapse/admin/v1/server_version
+        ```
+
+        (You may need to replace `localhost:8008` if Synapse is not configured to listen on that port.)
+    validations:
+      required: true
+  - type: dropdown
+    id: install_method
+    attributes:
+      label: Installation Method
+      options:
+        - Docker (matrixdotorg/synapse)
+        - Debian packages from packages.matrix.org
+        - pip (from PyPI)
+        - Other (please mention below)
+  - type: textarea
+    id: platform
+    attributes:
+      label: Platform
+      description: |
+        Tell us about the environment in which your homeserver is operating...
+        e.g. distro, hardware, if it's running in a vm/container, etc.
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: |
+        Please copy and paste any relevant log output, ideally at INFO or DEBUG log level.
+        This will be automatically formatted into code, so there is no need for backticks.
+
+        Please be careful to remove any personal or private data.
+
+        **Bug reports are usually very difficult to diagnose without logging.**
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: anything_else
+    attributes:
+      label: Anything else that would be useful to know?

.github/workflows/tests.yml

@@ -310,6 +310,16 @@ jobs:
     needs: linting-done
     runs-on: ubuntu-latest
 
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arrangement: monolith
+            database: SQLite
+
+          - arrangement: monolith
+            database: Postgres
+
     steps:
       # The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
       # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
@@ -337,7 +347,7 @@ jobs:
 
       - run: |
           set -o pipefail
-          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
+          POSTGRES=${{ (matrix.database == 'Postgres') && 1 }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
         shell: bash
         name: Run Complement Tests
 

CHANGES.md

@@ -1,3 +1,91 @@
+Synapse 1.61.0rc1 (2022-06-07)
+==============================
+
+This release removes support for the non-standard feature known both as 'groups' and as 'communities', which have been superseded by *Spaces*.
+
+Features
+--------
+
+- Add new `media_retention` options to the homeserver config for routinely cleaning up non-recently accessed media. ([\#12732](https://github.com/matrix-org/synapse/issues/12732), [\#12972](https://github.com/matrix-org/synapse/issues/12972), [\#12977](https://github.com/matrix-org/synapse/issues/12977))
+- Experimental support for [MSC3772](https://github.com/matrix-org/matrix-spec-proposals/pull/3772): Push rule for mutually related events. ([\#12740](https://github.com/matrix-org/synapse/issues/12740), [\#12859](https://github.com/matrix-org/synapse/issues/12859))
+- Update to the `check_event_for_spam` module callback: Deprecate the current callback signature, replace it with a new signature that is both less ambiguous (replacing booleans with explicit allow/block) and more powerful (ability to return explicit error codes). ([\#12808](https://github.com/matrix-org/synapse/issues/12808))
+- Add storage and module API methods to get monthly active users (and their corresponding appservices) within an optionally specified time range. ([\#12838](https://github.com/matrix-org/synapse/issues/12838), [\#12917](https://github.com/matrix-org/synapse/issues/12917))
+- Support the new error code `ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED` from [MSC3823](https://github.com/matrix-org/matrix-spec-proposals/pull/3823). ([\#12845](https://github.com/matrix-org/synapse/issues/12845), [\#12923](https://github.com/matrix-org/synapse/issues/12923))
+- Add a configurable background job to delete stale devices. ([\#12855](https://github.com/matrix-org/synapse/issues/12855))
+- Improve URL previews for pages with empty elements. ([\#12951](https://github.com/matrix-org/synapse/issues/12951))
+- Allow updating a user's password using the admin API without logging out their devices. Contributed by @jcgruenhage. ([\#12952](https://github.com/matrix-org/synapse/issues/12952))
+
+
+Bugfixes
+--------
+
+- Always send an `access_token` in `/thirdparty/` requests to appservices, as required by the [Application Service API specification](https://spec.matrix.org/v1.1/application-service-api/#third-party-networks). ([\#12746](https://github.com/matrix-org/synapse/issues/12746))
+- Implement [MSC3816](https://github.com/matrix-org/matrix-spec-proposals/pull/3816): sending the root event in a thread should count as having 'participated' in it. ([\#12766](https://github.com/matrix-org/synapse/issues/12766))
+- Delete events from the `federation_inbound_events_staging` table when a room is purged through the admin API. ([\#12784](https://github.com/matrix-org/synapse/issues/12784))
+- Fix a bug where we did not correctly handle invalid device list updates over federation. Contributed by Carl Bordum Hansen. ([\#12829](https://github.com/matrix-org/synapse/issues/12829))
+- Fix a bug which allowed multiple async operations to access database locks concurrently. Contributed by @sumnerevans @ Beeper. ([\#12832](https://github.com/matrix-org/synapse/issues/12832))
+- Fix an issue introduced in Synapse 0.34 where the `/notifications` endpoint would only return notifications if a user registered at least one pusher. Contributed by Famedly. ([\#12840](https://github.com/matrix-org/synapse/issues/12840))
+- Fix a bug where servers using a Postgres database would fail to backfill from an insertion event when MSC2716 is enabled (`experimental_features.msc2716_enabled`). ([\#12843](https://github.com/matrix-org/synapse/issues/12843))
+- Fix [MSC3787](https://github.com/matrix-org/matrix-spec-proposals/pull/3787) rooms being omitted from room directory, room summary and space hierarchy responses. ([\#12858](https://github.com/matrix-org/synapse/issues/12858))
+- Fix a bug introduced in Synapse 1.54.0 which could sometimes cause exceptions when handling federated traffic. ([\#12877](https://github.com/matrix-org/synapse/issues/12877))
+- Fix a bug introduced in Synapse 1.59.0 which caused room deletion to fail with a foreign key violation error. ([\#12889](https://github.com/matrix-org/synapse/issues/12889))
+- Fix a long-standing bug which caused the `/messages` endpoint to return an incorrect `end` attribute when there were no more events. Contributed by @Vetchu. ([\#12903](https://github.com/matrix-org/synapse/issues/12903))
+- Fix a bug introduced in Synapse 1.58.0 where `/sync` would fail if the most recent event in a room was a redaction of an event that has since been purged. ([\#12905](https://github.com/matrix-org/synapse/issues/12905))
+- Fix a potential memory leak when generating thumbnails. ([\#12932](https://github.com/matrix-org/synapse/issues/12932))
+- Fix a long-standing bug where a URL preview would break if the image failed to download. ([\#12950](https://github.com/matrix-org/synapse/issues/12950))
+
+
+Improved Documentation
+----------------------
+
+- Fix typographical errors in documentation. ([\#12863](https://github.com/matrix-org/synapse/issues/12863))
+- Fix documentation incorrectly stating the `sendToDevice` endpoint can be directed at generic workers. Contributed by Nick @ Beeper. ([\#12867](https://github.com/matrix-org/synapse/issues/12867))
+
+
+Deprecations and Removals
+-------------------------
+
+- Remove support for the non-standard groups/communities feature from Synapse. ([\#12553](https://github.com/matrix-org/synapse/issues/12553), [\#12558](https://github.com/matrix-org/synapse/issues/12558), [\#12563](https://github.com/matrix-org/synapse/issues/12563), [\#12895](https://github.com/matrix-org/synapse/issues/12895), [\#12897](https://github.com/matrix-org/synapse/issues/12897), [\#12899](https://github.com/matrix-org/synapse/issues/12899), [\#12900](https://github.com/matrix-org/synapse/issues/12900), [\#12936](https://github.com/matrix-org/synapse/issues/12936), [\#12966](https://github.com/matrix-org/synapse/issues/12966))
+- Remove contributed `kick_users.py` script. This is broken under Python 3, and is not added to the environment when `pip install`ing Synapse. ([\#12908](https://github.com/matrix-org/synapse/issues/12908))
+- Remove `contrib/jitsimeetbridge`. This was an unused experiment that hasn't been meaningfully changed since 2014. ([\#12909](https://github.com/matrix-org/synapse/issues/12909))
+- Remove unused `contrib/experiements/cursesio.py` script, which fails to run under Python 3. ([\#12910](https://github.com/matrix-org/synapse/issues/12910))
+- Remove unused `contrib/experiements/test_messaging.py` script. This fails to run on Python 3. ([\#12911](https://github.com/matrix-org/synapse/issues/12911))
+
+
+Internal Changes
+----------------
+
+- Test Synapse against Complement with workers. ([\#12810](https://github.com/matrix-org/synapse/issues/12810), [\#12933](https://github.com/matrix-org/synapse/issues/12933))
+- Reduce the amount of state we pull from the DB. ([\#12811](https://github.com/matrix-org/synapse/issues/12811), [\#12964](https://github.com/matrix-org/synapse/issues/12964))
+- Try other homeservers when re-syncing state for rooms with partial state. ([\#12812](https://github.com/matrix-org/synapse/issues/12812))
+- Resume state re-syncing for rooms with partial state after a Synapse restart. ([\#12813](https://github.com/matrix-org/synapse/issues/12813))
+- Remove Mutual Rooms' ([MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666)) endpoint dependency on the User Directory. ([\#12836](https://github.com/matrix-org/synapse/issues/12836))
+- Experimental: expand `check_event_for_spam` with ability to return additional fields. This enables spam-checker implementations to experiment with mechanisms to give users more information about why they are blocked and whether any action is needed from them to be unblocked. ([\#12846](https://github.com/matrix-org/synapse/issues/12846))
+- Remove `dont_notify` from the `.m.rule.room.server_acl` rule. ([\#12849](https://github.com/matrix-org/synapse/issues/12849))
+- Remove the unstable `/hierarchy` endpoint from [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946). ([\#12851](https://github.com/matrix-org/synapse/issues/12851))
+- Pull out less state when handling gaps in room DAG. ([\#12852](https://github.com/matrix-org/synapse/issues/12852), [\#12904](https://github.com/matrix-org/synapse/issues/12904))
+- Clean-up the push rules datastore. ([\#12856](https://github.com/matrix-org/synapse/issues/12856))
+- Correct a type annotation in the URL preview source code. ([\#12860](https://github.com/matrix-org/synapse/issues/12860))
+- Update `pyjwt` dependency to [2.4.0](https://github.com/jpadilla/pyjwt/releases/tag/2.4.0). ([\#12865](https://github.com/matrix-org/synapse/issues/12865))
+- Enable the `/account/whoami` endpoint on synapse worker processes. Contributed by Nick @ Beeper. ([\#12866](https://github.com/matrix-org/synapse/issues/12866))
+- Enable the `batch_send` endpoint on synapse worker processes. Contributed by Nick @ Beeper. ([\#12868](https://github.com/matrix-org/synapse/issues/12868))
+- Don't generate empty AS transactions when the AS is flagged as down. Contributed by Nick @ Beeper. ([\#12869](https://github.com/matrix-org/synapse/issues/12869))
+- Fix up the variable `state_store` naming. ([\#12871](https://github.com/matrix-org/synapse/issues/12871))
+- Faster room joins: when querying the current state of the room, wait for state to be populated. ([\#12872](https://github.com/matrix-org/synapse/issues/12872))
+- Avoid running queries which will never result in deletions. ([\#12879](https://github.com/matrix-org/synapse/issues/12879))
+- Use constants for EDU types. ([\#12884](https://github.com/matrix-org/synapse/issues/12884))
+- Reduce database load of `/sync` when presence is enabled. ([\#12885](https://github.com/matrix-org/synapse/issues/12885))
+- Refactor `have_seen_events` to reduce memory consumed when processing federation traffic. ([\#12886](https://github.com/matrix-org/synapse/issues/12886))
+- Refactor receipt linearization code. ([\#12888](https://github.com/matrix-org/synapse/issues/12888))
+- Add type annotations to `synapse.logging.opentracing`. ([\#12894](https://github.com/matrix-org/synapse/issues/12894))
+- Remove PyNaCl occurrences directly used in Synapse code. ([\#12902](https://github.com/matrix-org/synapse/issues/12902))
+- Bump types-jsonschema from 4.4.1 to 4.4.6. ([\#12912](https://github.com/matrix-org/synapse/issues/12912))
+- Rename storage classes. ([\#12913](https://github.com/matrix-org/synapse/issues/12913))
+- Preparation for database schema simplifications: stop reading from `event_edges.room_id`. ([\#12914](https://github.com/matrix-org/synapse/issues/12914))
+- Check if we are in a virtual environment before overriding the `PYTHONPATH` environment variable in the demo script. ([\#12916](https://github.com/matrix-org/synapse/issues/12916))
+- Improve the logging when signature checks on events fail. ([\#12925](https://github.com/matrix-org/synapse/issues/12925))
+
+
 Synapse 1.60.0 (2022-05-31)
 ===========================
 

changelog.d/12553.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12558.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12563.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12674.misc

@@ -0,0 +1 @@
+Add tests for cancellation of `GET /rooms/$room_id/members` and `GET /rooms/$room_id/state` requests.

changelog.d/12732.feature

@@ -1 +0,0 @@
-Add new `media_retention` options to the homeserver config for routinely cleaning up non-recently accessed media.

changelog.d/12737.doc

@@ -0,0 +1 @@
+Add documentation for how to configure Synapse with Workers using Docker Compose. Includes example worker config and docker-compose.yaml. Contributed by @Thumbscrew.

changelog.d/12738.misc

@@ -0,0 +1 @@
+Report login failures due to unknown third party identifiers in the same way as failures due to invalid passwords. This prevents an attacker from using the error response to determine if the identifier exists. Contributed by Daniel Aloni.

changelog.d/12740.feature

@@ -1 +0,0 @@
-Experimental support for [MSC3772](https://github.com/matrix-org/matrix-spec-proposals/pull/3772): Push rule for mutually related events.

changelog.d/12746.bugfix

@@ -1 +0,0 @@
-Always send an `access_token` in `/thirdparty/` requests to appservices, as required by the [Matrix specification](https://spec.matrix.org/v1.1/application-service-api/#third-party-networks).

changelog.d/12766.bugfix

@@ -1 +0,0 @@
-Implement [MSC3816](https://github.com/matrix-org/matrix-spec-proposals/pull/3816): sending the root event in a thread should count as "participated" in it.

changelog.d/12784.bugfix

@@ -1 +0,0 @@
-Delete events from the `federation_inbound_events_staging` table when a room is purged through the admin API.

changelog.d/12808.feature

@@ -1 +0,0 @@
-Update to `check_event_for_spam`. Deprecate the current callback signature, replace it with a new signature that is both less ambiguous (replacing booleans with explicit allow/block) and more powerful (ability to return explicit error codes).

changelog.d/12810.misc

@@ -1 +0,0 @@
-Test Synapse against Complement with workers.

changelog.d/12812.misc

@@ -1 +0,0 @@
-Try other homeservers when re-syncing state for rooms with partial state.

changelog.d/12813.misc

@@ -1 +0,0 @@
-Resume state re-syncing for rooms with partial state after a Synapse restart.

changelog.d/12829.bugfix

@@ -1 +0,0 @@
-Fix a bug where we did not correctly handle invalid device list updates over federation. Contributed by Carl Bordum Hansen.

changelog.d/12832.bugfix

@@ -1 +0,0 @@
-Fixed a bug which allowed multiple async operations to access database locks concurrently. Contributed by @sumnerevans @ Beeper.

changelog.d/12836.misc

@@ -1 +0,0 @@
-Remove Mutual Rooms ([MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666)) endpoint dependency on the User Directory.

changelog.d/12838.feature

@@ -1 +0,0 @@
-Add storage and module API methods to get monthly active users (and their corresponding appservices) within an optionally specified time range.

changelog.d/12840.bugfix

@@ -1 +0,0 @@
-Fix an issue introduced in Synapse 0.34 where the `/notifications` endpoint would only return notifications if a user registered at least one pusher. Contributed by Famedly.

changelog.d/12843.bugfix

@@ -1 +0,0 @@
-Fix bug where servers using a Postgres database would fail to backfill from an insertion event when MSC2716 is enabled (`experimental_features.msc2716_enabled`).

changelog.d/12845.feature

@@ -1 +0,0 @@
-Support the new error code "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" from [MSC3823](https://github.com/matrix-org/matrix-spec-proposals/pull/3823).

changelog.d/12846.misc

@@ -1 +0,0 @@
-Experimental: expand `check_event_for_spam` with ability to return additional fields. This enables spam-checker implementations to experiment with mechanisms to give users more information about why they are blocked and whether any action is needed from them to be unblocked.

changelog.d/12849.misc

@@ -1 +0,0 @@
-Remove `dont_notify` from the `.m.rule.room.server_acl` rule.

changelog.d/12851.misc

@@ -1 +0,0 @@
-Remove the unstable `/hierarchy` endpoint from [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).

changelog.d/12852.misc

@@ -1 +0,0 @@
-Pull out less state when handling gaps in room DAG.

changelog.d/12855.feature

@@ -1 +0,0 @@
-Add a configurable background job to delete stale devices.

changelog.d/12856.misc

@@ -1 +0,0 @@
-Clean-up the push rules datastore.

changelog.d/12858.bugfix

@@ -1 +0,0 @@
-Fix [MSC3787](https://github.com/matrix-org/matrix-spec-proposals/pull/3787) rooms being omitted from room directory, room summary and space hierarchy responses.

changelog.d/12859.feature

@@ -1 +0,0 @@
-Experimental support for [MSC3772](https://github.com/matrix-org/matrix-spec-proposals/pull/3772): Push rule for mutually related events.

changelog.d/12860.misc

@@ -1 +0,0 @@
-Correct a type annotation in the URL preview source code.

changelog.d/12863.doc

@@ -1 +0,0 @@
-Fix typos in documentation.

changelog.d/12865.misc

@@ -1 +0,0 @@
-Update `pyjwt` dependency to [2.4.0](https://github.com/jpadilla/pyjwt/releases/tag/2.4.0).

changelog.d/12866.misc

@@ -1 +0,0 @@
-Enable the `/account/whoami` endpoint on synapse worker processes. Contributed by Nick @ Beeper.

changelog.d/12867.doc

@@ -1 +0,0 @@
-Fix documentation incorrectly stating the `sendToDevice` endpoint can be directed at generic workers. Contributed by Nick @ Beeper.

changelog.d/12868.misc

@@ -1 +0,0 @@
-Enable the `batch_send` endpoint on synapse worker processes. Contributed by Nick @ Beeper.

changelog.d/12869.misc

@@ -1 +0,0 @@
-Don't generate empty AS transactions when the AS is flagged as down. Contributed by Nick @ Beeper.

changelog.d/12871.misc

@@ -1 +0,0 @@
-Fix up the variable `state_store` naming.

changelog.d/12872.misc

@@ -1 +0,0 @@
-Faster room joins: when querying the current state of the room, wait for state to be populated.

changelog.d/12877.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.54 which could sometimes cause exceptions when handling federated traffic.

changelog.d/12879.misc

@@ -1 +0,0 @@
-Avoid running queries which will never result in deletions.

changelog.d/12881.misc

@@ -0,0 +1 @@
+Merge the Complement testing Docker images into a single, multi-purpose image.

changelog.d/12884.misc

@@ -1 +0,0 @@
-Use constants for EDU types.

changelog.d/12885.misc

@@ -1 +0,0 @@
-Reduce database load of `/sync` when presence is enabled.

changelog.d/12886.misc

@@ -1 +0,0 @@
-Refactor `have_seen_events` to reduce memory consumed when processing federation traffic.

changelog.d/12888.misc

@@ -1 +0,0 @@
-Refactor receipt linearization code.

changelog.d/12889.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.59.0 which caused room deletion to fail with a foreign key violation.

changelog.d/12894.misc

@@ -1 +0,0 @@
-Add type annotations to `synapse.logging.opentracing`.

changelog.d/12895.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12897.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12899.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12900.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12902.misc

@@ -1 +0,0 @@
-Remove PyNaCl occurrences directly used in Synapse code.

changelog.d/12903.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug which caused the `/messages` endpoint to return an incorrect `end` attribute when there were no more events. Contributed by @Vetchu.

changelog.d/12904.misc

@@ -1 +0,0 @@
-Pull out less state when handling gaps in room DAG.

changelog.d/12905.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.58.0 where `/sync` would fail if the most recent event in a room was a redaction of an event that has since been purged.

changelog.d/12908.removal

@@ -1 +0,0 @@
-Remove contributed `kick_users.py` script. This is broken under Python 3, and is not added to the environment when `pip install`ing Synapse.

changelog.d/12909.removal

@@ -1 +0,0 @@
-Remove `contrib/jitsimeetbridge`. This was an unused experiment that hasn't been meaningfully changed since 2014.

changelog.d/12910.removal

@@ -1 +0,0 @@
-Remove unused `contrib/experiements/cursesio.py` script, which fails to run under Python 3.

changelog.d/12911.removal

@@ -1 +0,0 @@
-Remove unused `contrib/experiements/test_messaging.py` script. This fails to run on Python 3.

changelog.d/12912.misc

@@ -1 +0,0 @@
-Bump types-jsonschema from 4.4.1 to 4.4.6.

changelog.d/12913.misc

@@ -1 +0,0 @@
-Rename storage classes.

changelog.d/12914.misc

@@ -1 +0,0 @@
-Preparation for database schema simplifications: stop reading from `event_edges.room_id`.

changelog.d/12916.misc

@@ -1 +0,0 @@
-Check if we are in a virtual environment before overriding the `PYTHONPATH` environment variable in the demo script.

changelog.d/12917.feature

@@ -1 +0,0 @@
-Add storage and module API methods to get monthly active users (and their corresponding appservices) within an optionally specified time range.

changelog.d/12923.feature

@@ -1 +0,0 @@
-Support the new error code "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" from [MSC3823](https://github.com/matrix-org/matrix-spec-proposals/pull/3823).

changelog.d/12925.misc

@@ -1 +0,0 @@
-Improve the logging when signature checks on events fail.

changelog.d/12929.misc

@@ -0,0 +1 @@
+Clean up the test code for client disconnection.

changelog.d/12932.bugfix

@@ -1 +0,0 @@
-Fix potential memory leak when generating thumbnails.

changelog.d/12933.misc

@@ -1 +0,0 @@
-Test Synapse against Complement with workers.

changelog.d/12936.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12950.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug where a URL preview would break if the image failed to download.

changelog.d/12951.feature

@@ -1 +0,0 @@
-Improve URL previews for pages with empty elements.

changelog.d/12952.feature

@@ -1 +0,0 @@
-Allow updating a user's password using the admin API without logging out their devices. Contributed by @jcgruenhage.

changelog.d/12957.misc

@@ -0,0 +1 @@
+Use lower isolation level when inserting read receipts to avoid serialization errors. Contributed by Nick @ Beeper.

changelog.d/12964.misc

@@ -1 +0,0 @@
-Reduce the amount of state we pull from the DB.

changelog.d/12965.misc

@@ -0,0 +1 @@
+Enable testing against PostgreSQL databases in Complement CI.

changelog.d/12966.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12969.misc

@@ -0,0 +1 @@
+Fix an inaccurate comment.

changelog.d/12970.misc

@@ -0,0 +1 @@
+Remove the `delete_device` method and always call `delete_devices`.

changelog.d/12973.bugfix

@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.58 where Synapse would not report full version information when installed from a git checkout. This is a best-effort affair and not guaranteed to be stable.

changelog.d/12979.bugfix

@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.60 where Synapse would fail to start if the `sqlite3` module was not available.

changelog.d/12982.misc

@@ -0,0 +1 @@
+Use a GitHub form for issues rather than a hard-to-read, easy-to-ignore template.

changelog.d/12984.misc

@@ -0,0 +1 @@
+Move [MSC3715](https://github.com/matrix-org/matrix-spec-proposals/pull/3715) behind an experimental config flag.

changelog.d/12985.misc

@@ -0,0 +1 @@
+Add type annotations to `tests.state.test_v2`.

changelog.d/12990.misc

@@ -0,0 +1 @@
+Fix documentation for running complement tests.

changelog.d/12991.bugfix

@@ -0,0 +1,2 @@
+Fix a bug where non-standard information was required when requesting the `/hierarchy` API over federation. Introduced 
+in Synapse v1.41.0.

changelog.d/13004.misc

@@ -0,0 +1 @@
+Faster joins: add issue links to the TODO comments in the code.

changelog.d/13013.misc

@@ -0,0 +1 @@
+Modernize the `contrib/graph/` scripts.

changelog.d/13022.doc

@@ -0,0 +1 @@
+Ensure the [Poetry cheat sheet](https://matrix-org.github.io/synapse/develop/development/dependencies.html) is available in the online documentation.

changelog.d/13028.misc

@@ -0,0 +1 @@
+Add type annotations to `tests.utils`.

contrib/docker_compose_workers/README.md

@@ -0,0 +1,125 @@
+# Setting up Synapse with Workers using Docker Compose
+
+This directory describes how deploy and manage Synapse and workers via [Docker Compose](https://docs.docker.com/compose/).
+
+Example worker configuration files can be found [here](workers).
+
+All examples and snippets assume that your Synapse service is called `synapse` in your Docker Compose file.
+
+An example Docker Compose file can be found [here](docker-compose.yaml).
+
+## Worker Service Examples in Docker Compose
+
+In order to start the Synapse container as a worker, you must specify an `entrypoint` that loads both the `homeserver.yaml` and the configuration for the worker (`synapse-generic-worker-1.yaml` in the example below). You must also include the worker type in the environment variable `SYNAPSE_WORKER` or alternatively pass `-m synapse.app.generic_worker` as part of the `entrypoint` after `"/start.py", "run"`).
+
+### Generic Worker Example
+
+```yaml
+synapse-generic-worker-1:
+  image: matrixdotorg/synapse:latest
+  container_name: synapse-generic-worker-1
+  restart: unless-stopped
+  entrypoint: ["/start.py", "run", "--config-path=/data/homeserver.yaml", "--config-path=/data/workers/synapse-generic-worker-1.yaml"]
+  healthcheck:
+    test: ["CMD-SHELL", "curl -fSs http://localhost:8081/health || exit 1"]
+    start_period: "5s"
+    interval: "15s"
+    timeout: "5s"
+  volumes:
+    - ${VOLUME_PATH}/data:/data:rw # Replace VOLUME_PATH with the path to your Synapse volume
+  environment:
+    SYNAPSE_WORKER: synapse.app.generic_worker
+  # Expose port if required so your reverse proxy can send requests to this worker
+  # Port configuration will depend on how the http listener is defined in the worker configuration file
+  ports:
+    - 8081:8081
+  depends_on:
+    - synapse
+```
+
+### Federation Sender Example
+
+Please note: The federation sender does not receive REST API calls so no exposed ports are required.
+
+```yaml
+synapse-federation-sender-1:
+  image: matrixdotorg/synapse:latest
+  container_name: synapse-federation-sender-1
+  restart: unless-stopped
+  entrypoint: ["/start.py", "run", "--config-path=/data/homeserver.yaml", "--config-path=/data/workers/synapse-federation-sender-1.yaml"]
+  healthcheck:
+    disable: true
+  volumes:
+    - ${VOLUME_PATH}/data:/data:rw # Replace VOLUME_PATH with the path to your Synapse volume
+  environment:
+    SYNAPSE_WORKER: synapse.app.federation_sender
+  depends_on:
+    - synapse
+```
+
+## `homeserver.yaml` Configuration
+
+### Enable Redis
+
+Locate the `redis` section of your `homeserver.yaml` and enable and configure it:
+
+```yaml
+redis:
+  enabled: true
+  host: redis
+  port: 6379
+  # password: <secret_password>  
+```
+
+This assumes that your Redis service is called `redis` in your Docker Compose file.
+
+### Add a replication Listener
+
+Locate the `listeners` section of your `homeserver.yaml` and add the following replication listener:
+
+```yaml
+listeners:
+  # Other listeners
+
+  - port: 9093
+    type: http
+    resources:
+      - names: [replication]
+```
+
+This listener is used by the workers for replication and is referred to in worker config files using the following settings:
+
+```yaml
+worker_replication_host: synapse
+worker_replication_http_port: 9093
+```
+
+### Add Workers to `instance_map`
+
+Locate the `instance_map` section of your `homeserver.yaml` and populate it with your workers:
+
+```yaml
+instance_map:
+  synapse-generic-worker-1:        # The worker_name setting in your worker configuration file
+    host: synapse-generic-worker-1 # The name of the worker service in your Docker Compose file
+    port: 8034                     # The port assigned to the replication listener in your worker config file
+  synapse-federation-sender-1:
+    host: synapse-federation-sender-1
+    port: 8034
+```
+
+### Configure Federation Senders
+
+This section is applicable if you are using Federation senders (synapse.app.federation_sender). Locate the `send_federation` and `federation_sender_instances` settings in your `homeserver.yaml` and configure them:
+
+```yaml
+# This will disable federation sending on the main Synapse instance
+send_federation: false
+
+federation_sender_instances:
+  - synapse-federation-sender-1 # The worker_name setting in your federation sender worker configuration file
+```
+
+## Other Worker types
+
+Using the concepts shown here it is possible to create other worker types in Docker Compose. See the [Workers](https://matrix-org.github.io/synapse/latest/workers.html#available-worker-applications) documentation for a list of available workers.

contrib/docker_compose_workers/docker-compose.yaml

@@ -0,0 +1,77 @@
+networks:
+  backend:
+
+services:
+  postgres:
+    image: postgres:latest
+    restart: unless-stopped
+    volumes:
+      - ${VOLUME_PATH}/var/lib/postgresql/data:/var/lib/postgresql/data:rw
+    networks:
+      - backend
+    environment:
+      POSTGRES_DB: synapse
+      POSTGRES_USER: synapse_user
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_INITDB_ARGS: --encoding=UTF8 --locale=C
+
+  redis:
+    image: redis:latest
+    restart: unless-stopped
+    networks:
+      - backend
+
+  synapse:
+    image: matrixdotorg/synapse:latest
+    container_name: synapse
+    restart: unless-stopped
+    volumes:
+      - ${VOLUME_PATH}/data:/data:rw
+    ports:
+      - 8008:8008
+    networks:
+      - backend
+    environment:
+      SYNAPSE_CONFIG_DIR: /data
+      SYNAPSE_CONFIG_PATH: /data/homeserver.yaml
+    depends_on:
+      - postgres
+
+  synapse-generic-worker-1:
+    image: matrixdotorg/synapse:latest
+    container_name: synapse-generic-worker-1
+    restart: unless-stopped
+    entrypoint: ["/start.py", "run", "--config-path=/data/homeserver.yaml", "--config-path=/data/workers/synapse-generic-worker-1.yaml"]
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fSs http://localhost:8081/health || exit 1"]
+      start_period: "5s"
+      interval: "15s"
+      timeout: "5s"
+    networks:
+      - backend
+    volumes:
+      - ${VOLUME_PATH}/data:/data:rw # Replace VOLUME_PATH with the path to your Synapse volume
+    environment:
+      SYNAPSE_WORKER: synapse.app.generic_worker
+    # Expose port if required so your reverse proxy can send requests to this worker
+    # Port configuration will depend on how the http listener is defined in the worker configuration file
+    ports:
+      - 8081:8081
+    depends_on:
+      - synapse
+
+  synapse-federation-sender-1:
+    image: matrixdotorg/synapse:latest
+    container_name: synapse-federation-sender-1
+    restart: unless-stopped
+    entrypoint: ["/start.py", "run", "--config-path=/data/homeserver.yaml", "--config-path=/data/workers/synapse-federation-sender-1.yaml"]
+    healthcheck:
+      disable: true
+    networks:
+      - backend
+    volumes:
+      - ${VOLUME_PATH}/data:/data:rw # Replace VOLUME_PATH with the path to your Synapse volume
+    environment:
+      SYNAPSE_WORKER: synapse.app.federation_sender
+    depends_on:
+      - synapse

contrib/docker_compose_workers/workers/synapse-federation-sender-1.yaml

@@ -0,0 +1,14 @@
+worker_app: synapse.app.federation_sender
+worker_name: synapse-federation-sender-1
+
+# The replication listener on the main synapse process.
+worker_replication_host: synapse
+worker_replication_http_port: 9093
+
+worker_listeners:
+  - type: http
+    port: 8034
+    resources:
+      - names: [replication]
+
+worker_log_config: /data/federation_sender.log.config

contrib/docker_compose_workers/workers/synapse-generic-worker-1.yaml

@@ -0,0 +1,19 @@
+worker_app: synapse.app.generic_worker
+worker_name: synapse-generic-worker-1
+
+# The replication listener on the main synapse process.
+worker_replication_host: synapse
+worker_replication_http_port: 9093
+
+worker_listeners:
+  - type: http
+    port: 8034
+    resources:
+      - names: [replication]
+  - type: http
+    port: 8081
+    x_forwarded: true
+    resources:
+      - names: [client, federation]
+
+worker_log_config: /data/worker.log.config