Fix running background tasks on BG worker

WORKING workers setup template
STASH
2022-03-18 13:53:55 +00:00 · 2022-03-17 10:42:28 +00:00 · 2022-03-16 14:02:58 +00:00 · 2022-03-11 11:36:47 +00:00 · 2022-02-24 17:10:23 +00:00
341 changed files with 5683 additions and 8035 deletions
@@ -21,7 +21,7 @@ python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
 echo "--- Prepare test database"

 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
-update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates

 # Run the export-data command on the sqlite test database
 python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
@@ -41,7 +41,7 @@ fi

 # Port the SQLite databse to postgres so we can check command works against postgres
 echo "+++ Port SQLite3 databse to postgres"
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

 # Run the export-data command on postgres database
 python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
@@ -25,19 +25,17 @@ python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
 echo "--- Prepare test database"

 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
-update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates

 # Create the PostgreSQL database.
 .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"

 echo "+++ Run synapse_port_db against test database"
-# TODO: this invocation of synapse_port_db (and others below) used to be prepended with `coverage run`,
-# but coverage seems unable to find the entrypoints installed by `pip install -e .`.
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

 # We should be able to run twice against the same database.
 echo "+++ Run synapse_port_db a second time"
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

 #####

@@ -48,7 +46,7 @@ echo "--- Prepare empty SQLite database"
 # we do this by deleting the sqlite db, and then doing the same again.
 rm .ci/test_db.db

-update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates

 # re-create the PostgreSQL database.
 .ci/scripts/postgres_exec.py \
@@ -56,4 +54,4 @@ update_synapse_database --database-config .ci/sqlite-config.yaml --run-backgroun
  "CREATE DATABASE synapse"

 echo "+++ Run synapse_port_db against empty database"
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
@@ -3,9 +3,11 @@

 # things to include
 !docker
+!scripts
 !synapse
 !MANIFEST.in
 !README.rst
 !setup.py
+!synctl

 **/__pycache__
@@ -31,7 +31,7 @@ jobs:
          # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
          dists='["debian:sid"]'
          if [[ $GITHUB_REF == refs/tags/* ]]; then
-              dists=$(scripts-dev/build_debian_packages.py --show-dists-json)
+              dists=$(scripts-dev/build_debian_packages --show-dists-json)
          fi
          echo "::set-output name=distros::$dists"
    # map the step outputs to job outputs
@@ -74,7 +74,7 @@ jobs:
        # see https://github.com/docker/build-push-action/issues/252
        # for the cache magic here
        run: |
-          ./src/scripts-dev/build_debian_packages.py \
+          ./src/scripts-dev/build_debian_packages \
            --docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \
            --docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
            --docker-build-arg=--progress=plain \
@@ -112,8 +112,7 @@ jobs:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          files: |
-            Sdist/*
-            Wheel/*
+            python-dist/*
            debs.tar.xz
          # if it's not already published, keep the release as a draft.
          draft: true
@@ -10,20 +10,12 @@ concurrency:
  cancel-in-progress: true

 jobs:
-  check-sampleconfig:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install -e .
-      - run: scripts-dev/generate_sample_config.sh --check
-      - run: scripts-dev/config-lint.sh
-
  lint:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        toxenv:
+          - "check-sampleconfig"
          - "check_codestyle"
          - "check_isort"
          - "mypy"
@@ -51,15 +43,15 @@ jobs:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0
      - uses: actions/setup-python@v2
-      - run: "pip install 'towncrier>=18.6.0rc1'"
-      - run: scripts-dev/check-newsfragment.sh
+      - run: pip install tox
+      - run: scripts-dev/check-newsfragment
        env:
          PULL_REQUEST_NUMBER: ${{ github.event.number }}

  # Dummy step to gate other tests on without repeating the whole list
  linting-done:
    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    needs: [lint, lint-crlf, lint-newsfile, check-sampleconfig]
+    needs: [lint, lint-crlf, lint-newsfile]
    runs-on: ubuntu-latest
    steps:
      - run: "true"
@@ -377,7 +369,7 @@ jobs:
      # Run Complement
      - run: |
          set -o pipefail
-          go test -v -json -p 1 -tags synapse_blacklist,msc2403,msc2716,msc3030 ./tests/... 2>&1 | gotestfmt
+          go test -v -json -p 1 -tags synapse_blacklist,msc2403 ./tests/... 2>&1 | gotestfmt
        shell: bash
        name: Run Complement Tests
        env:
@@ -388,22 +380,34 @@ jobs:
  tests-done:
    if: ${{ always() }}
    needs:
-      - check-sampleconfig
      - lint
      - lint-crlf
      - lint-newsfile
      - trial
      - trial-olddeps
      - sytest
-      - export-data
      - portdb
      - complement
    runs-on: ubuntu-latest
    steps:
-      - uses: matrix-org/done-action@v2
-        with:
-          needs: ${{ toJSON(needs) }}
+      - name: Set build result
+        env:
+          NEEDS_CONTEXT: ${{ toJSON(needs) }}
+        # the `jq` incantation dumps out a series of "<job> <result>" lines.
+        # we set it to an intermediate variable to avoid a pipe, which makes it
+        # hard to set $rc.
+        run: |
+          rc=0
+          results=$(jq -r 'to_entries[] | [.key,.value.result] | join(" ")' <<< $NEEDS_CONTEXT)
+          while read job result ; do
+              # The newsfile lint may be skipped on non PR builds
+              if [ $result == "skipped" ] && [ $job == "lint-newsfile" ]; then
+                continue
+              fi

-          # The newsfile lint may be skipped on non PR builds
-          skippable:
-            lint-newsfile
+              if [ "$result" != "success" ]; then
+                  echo "::set-failed ::Job $job returned $result"
+                  rc=1
+              fi
+          done <<< $results
+          exit $rc
@@ -1,228 +1,3 @@
-Synapse 1.55.1 (2022-03-24)
-===========================
-
-This is a patch release that fixes an incompatibility with version 3.1.0 of the [Jinja](https://pypi.org/project/Jinja2/) library, released on March 24th, 2022. Deployments of Synapse using the `matrixdotorg/synapse` Docker image or Debian packages from packages.matrix.org are not affected.
-
-Internal Changes
----------------
-
- Remove uses of the long-deprecated `jinja2.Markup` which would prevent Synapse from starting with Jinja 3.1.0 or above installed. ([\#12289](https://github.com/matrix-org/synapse/issues/12289))
-
-
-Synapse 1.55.0 (2022-03-22)
-===========================
-
-This release removes a workaround introduced in Synapse 1.50.0 for Mjolnir compatibility. **This breaks compatibility with Mjolnir 1.3.1 and earlier. ([\#11700](https://github.com/matrix-org/synapse/issues/11700))**; Mjolnir users should upgrade Mjolnir before upgrading Synapse to this version.
-
-This release also moves the location of the `synctl` script; see the [upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#synctl-script-has-been-moved) for more details.
-
-
-Internal Changes
----------------
-
- Tweak copy for default Single Sign-On account details template to better adhere to mobile app store guidelines. ([\#12265](https://github.com/matrix-org/synapse/issues/12265), [\#12260](https://github.com/matrix-org/synapse/issues/12260))
-
-
-Synapse 1.55.0rc1 (2022-03-15)
-==============================
-
-Features
--------
-
- Add third-party rules callbacks `check_can_shutdown_room` and `check_can_deactivate_user`. ([\#12028](https://github.com/matrix-org/synapse/issues/12028))
- Improve performance of logging in for large accounts. ([\#12132](https://github.com/matrix-org/synapse/issues/12132))
- Add experimental env var `SYNAPSE_ASYNC_IO_REACTOR` that causes Synapse to use the asyncio reactor for Twisted. ([\#12135](https://github.com/matrix-org/synapse/issues/12135))
- Support the stable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440): threads. ([\#12151](https://github.com/matrix-org/synapse/issues/12151))
- Add a new Jinja2 template filter to extract the local part of an email address. ([\#12212](https://github.com/matrix-org/synapse/issues/12212))
-
-
-Bugfixes
--------
-
- Use the proper serialization format for bundled thread aggregations. The bug has existed since Synapse v1.48.0. ([\#12090](https://github.com/matrix-org/synapse/issues/12090))
- Fix a long-standing bug when redacting events with relations. ([\#12113](https://github.com/matrix-org/synapse/issues/12113), [\#12121](https://github.com/matrix-org/synapse/issues/12121), [\#12130](https://github.com/matrix-org/synapse/issues/12130), [\#12189](https://github.com/matrix-org/synapse/issues/12189))
- Fix a bug introduced in Synapse 1.7.2 whereby background updates are never run with the default background batch size. ([\#12157](https://github.com/matrix-org/synapse/issues/12157))
- Fix a bug where non-standard information was returned from the `/hierarchy` API. Introduced in Synapse v1.41.0. ([\#12175](https://github.com/matrix-org/synapse/issues/12175))
- Fix a bug introduced in Synapse 1.54.0 that broke background updates on sqlite homeservers while search was disabled. ([\#12215](https://github.com/matrix-org/synapse/issues/12215))
- Fix a long-standing bug when a `filter` argument with `event_fields` which did not include the `unsigned` field could result in a 500 error on `/sync`. ([\#12234](https://github.com/matrix-org/synapse/issues/12234))
-
-
-Improved Documentation
----------------------
-
- Fix complexity checking config example in [Resource Constrained Devices](https://matrix-org.github.io/synapse/v1.54/other/running_synapse_on_single_board_computers.html) docs page. ([\#11998](https://github.com/matrix-org/synapse/issues/11998))
- Improve documentation for demo scripts. ([\#12143](https://github.com/matrix-org/synapse/issues/12143))
- Updates to the Room DAG concepts development document. ([\#12179](https://github.com/matrix-org/synapse/issues/12179))
- Document that the `typing`, `to_device`, `account_data`, `receipts`, and `presence` stream writer can only be used on a single worker. ([\#12196](https://github.com/matrix-org/synapse/issues/12196))
- Document that contributors can sign off privately by email. ([\#12204](https://github.com/matrix-org/synapse/issues/12204))
-
-
-Deprecations and Removals
-------------------------
-
- **Remove workaround introduced in Synapse 1.50.0 for Mjolnir compatibility. Breaks compatibility with Mjolnir 1.3.1 and earlier. ([\#11700](https://github.com/matrix-org/synapse/issues/11700))**
- **`synctl` has been moved into into `synapse._scripts` and is exposed as an entry point; see [upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#synctl-script-has-been-moved). ([\#12140](https://github.com/matrix-org/synapse/issues/12140))
- Remove backwards compatibilty with pagination tokens from the `/relations` and `/aggregations` endpoints generated from Synapse < v1.52.0. ([\#12138](https://github.com/matrix-org/synapse/issues/12138))
- The groups/communities feature in Synapse has been deprecated. ([\#12200](https://github.com/matrix-org/synapse/issues/12200))
-
-
-Internal Changes
----------------
-
- Simplify the `ApplicationService` class' set of public methods related to interest checking. ([\#11915](https://github.com/matrix-org/synapse/issues/11915))
- Add config settings for background update parameters. ([\#11980](https://github.com/matrix-org/synapse/issues/11980))
- Correct type hints for txredis. ([\#12042](https://github.com/matrix-org/synapse/issues/12042))
- Limit the size of `aggregation_key` on annotations. ([\#12101](https://github.com/matrix-org/synapse/issues/12101))
- Add type hints to tests files. ([\#12108](https://github.com/matrix-org/synapse/issues/12108), [\#12146](https://github.com/matrix-org/synapse/issues/12146), [\#12207](https://github.com/matrix-org/synapse/issues/12207), [\#12208](https://github.com/matrix-org/synapse/issues/12208))
- Move scripts to Synapse package and expose as setuptools entry points. ([\#12118](https://github.com/matrix-org/synapse/issues/12118))
- Add support for cancellation to `ReadWriteLock`. ([\#12120](https://github.com/matrix-org/synapse/issues/12120))
- Fix data validation to compare to lists, not sequences. ([\#12128](https://github.com/matrix-org/synapse/issues/12128))
- Fix CI not attaching source distributions and wheels to the GitHub releases. ([\#12131](https://github.com/matrix-org/synapse/issues/12131))
- Remove unused mocks from `test_typing`. ([\#12136](https://github.com/matrix-org/synapse/issues/12136))
- Give `scripts-dev` scripts suffixes for neater CI config. ([\#12137](https://github.com/matrix-org/synapse/issues/12137))
- Move the snapcraft configuration file to `contrib`. ([\#12142](https://github.com/matrix-org/synapse/issues/12142))
- Enable [MSC3030](https://github.com/matrix-org/matrix-doc/pull/3030) Complement tests in CI. ([\#12144](https://github.com/matrix-org/synapse/issues/12144))
- Enable [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) Complement tests in CI. ([\#12145](https://github.com/matrix-org/synapse/issues/12145))
- Add test for `ObservableDeferred`'s cancellation behaviour. ([\#12149](https://github.com/matrix-org/synapse/issues/12149))
- Use `ParamSpec` in type hints for `synapse.logging.context`. ([\#12150](https://github.com/matrix-org/synapse/issues/12150))
- Prune unused jobs from `tox` config. ([\#12152](https://github.com/matrix-org/synapse/issues/12152))
- Move CI checks out of tox, to facilitate a move to using poetry. ([\#12153](https://github.com/matrix-org/synapse/issues/12153))
- Avoid generating state groups for local out-of-band leaves. ([\#12154](https://github.com/matrix-org/synapse/issues/12154))
- Avoid trying to calculate the state at outlier events. ([\#12155](https://github.com/matrix-org/synapse/issues/12155), [\#12173](https://github.com/matrix-org/synapse/issues/12173), [\#12202](https://github.com/matrix-org/synapse/issues/12202))
- Fix some type annotations. ([\#12156](https://github.com/matrix-org/synapse/issues/12156))
- Add type hints for `ObservableDeferred` attributes. ([\#12159](https://github.com/matrix-org/synapse/issues/12159))
- Use a prebuilt Action for the `tests-done` CI job. ([\#12161](https://github.com/matrix-org/synapse/issues/12161))
- Reduce number of DB queries made during processing of `/sync`. ([\#12163](https://github.com/matrix-org/synapse/issues/12163))
- Add `delay_cancellation` utility function, which behaves like `stop_cancellation` but waits until the original `Deferred` resolves before raising a `CancelledError`. ([\#12180](https://github.com/matrix-org/synapse/issues/12180))
- Retry HTTP replication failures, this should prevent 502's when restarting stateful workers (main, event persisters, stream writers). Contributed by Nick @ Beeper. ([\#12182](https://github.com/matrix-org/synapse/issues/12182))
- Add cancellation support to `@cached` and `@cachedList` decorators. ([\#12183](https://github.com/matrix-org/synapse/issues/12183))
- Remove unused variables. ([\#12187](https://github.com/matrix-org/synapse/issues/12187))
- Add combined test for HTTP pusher and push rule. Contributed by Nick @ Beeper. ([\#12188](https://github.com/matrix-org/synapse/issues/12188))
- Rename `HomeServer.get_tcp_replication` to `get_replication_command_handler`. ([\#12192](https://github.com/matrix-org/synapse/issues/12192))
- Remove some dead code. ([\#12197](https://github.com/matrix-org/synapse/issues/12197))
- Fix a misleading comment in the function `check_event_for_spam`. ([\#12203](https://github.com/matrix-org/synapse/issues/12203))
- Remove unnecessary `pass` statements. ([\#12206](https://github.com/matrix-org/synapse/issues/12206))
- Update the SSO username picker template to comply with SIWA guidelines. ([\#12210](https://github.com/matrix-org/synapse/issues/12210))
- Improve code documentation for the typing stream over replication. ([\#12211](https://github.com/matrix-org/synapse/issues/12211))
-
-
-Synapse 1.54.0 (2022-03-08)
-===========================
-
-Please note that this will be the last release of Synapse that is compatible with Mjolnir 1.3.1 and earlier.
-Administrators of servers which have the Mjolnir module installed are advised to upgrade Mjolnir to version 1.3.2 or later.
-
-
-Bugfixes
--------
-
- Fix a bug introduced in Synapse 1.54.0rc1 preventing the new module callbacks introduced in this release from being registered by modules. ([\#12141](https://github.com/matrix-org/synapse/issues/12141))
- Fix a bug introduced in Synapse 1.54.0rc1 where runtime dependency version checks would mistakenly check development dependencies if they were present and would not accept pre-release versions of dependencies. ([\#12129](https://github.com/matrix-org/synapse/issues/12129), [\#12177](https://github.com/matrix-org/synapse/issues/12177))
-
-
-Internal Changes
----------------
-
- Update release script to insert the previous version when writing "No significant changes" line in the changelog. ([\#12127](https://github.com/matrix-org/synapse/issues/12127))
- Relax the version guard for "packaging" added in [\#12088](https://github.com/matrix-org/synapse/issues/12088). ([\#12166](https://github.com/matrix-org/synapse/issues/12166))
-
-
-Synapse 1.54.0rc1 (2022-03-02)
-==============================
-
-
-Features
--------
-
- Add support for [MSC3202](https://github.com/matrix-org/matrix-doc/pull/3202): sending one-time key counts and fallback key usage states to Application Services. ([\#11617](https://github.com/matrix-org/synapse/issues/11617))
- Improve the generated URL previews for some web pages. Contributed by @AndrewRyanChama. ([\#11985](https://github.com/matrix-org/synapse/issues/11985))
- Track cache invalidations in Prometheus metrics, as already happens for cache eviction based on size or time. ([\#12000](https://github.com/matrix-org/synapse/issues/12000))
- Implement experimental support for [MSC3720](https://github.com/matrix-org/matrix-doc/pull/3720) (account status endpoints). ([\#12001](https://github.com/matrix-org/synapse/issues/12001), [\#12067](https://github.com/matrix-org/synapse/issues/12067))
- Enable modules to set a custom display name when registering a user. ([\#12009](https://github.com/matrix-org/synapse/issues/12009))
- Advertise Matrix 1.1 and 1.2 support on `/_matrix/client/versions`. ([\#12020](https://github.com/matrix-org/synapse/issues/12020), ([\#12022](https://github.com/matrix-org/synapse/issues/12022))
- Support only the stable identifier for [MSC3069](https://github.com/matrix-org/matrix-doc/pull/3069)'s `is_guest` on `/_matrix/client/v3/account/whoami`. ([\#12021](https://github.com/matrix-org/synapse/issues/12021))
- Use room version 9 as the default room version (per [MSC3589](https://github.com/matrix-org/matrix-doc/pull/3589)). ([\#12058](https://github.com/matrix-org/synapse/issues/12058))
- Add module callbacks to react to user deactivation status changes (i.e. deactivations and reactivations) and profile updates. ([\#12062](https://github.com/matrix-org/synapse/issues/12062))
-
-
-Bugfixes
--------
-
- Fix a bug introduced in Synapse 1.48.0 where an edit of the latest event in a thread would not be properly applied to the thread summary. ([\#11992](https://github.com/matrix-org/synapse/issues/11992))
- Fix long-standing bug where the `get_rooms_for_user` cache was not correctly invalidated for remote users when the server left a room. ([\#11999](https://github.com/matrix-org/synapse/issues/11999))
- Fix a 500 error with Postgres when looking backwards with the [MSC3030](https://github.com/matrix-org/matrix-doc/pull/3030) `/timestamp_to_event?dir=b` endpoint. ([\#12024](https://github.com/matrix-org/synapse/issues/12024))
- Properly fix a long-standing bug where wrong data could be inserted into the `event_search` table when using SQLite. This could block running `synapse_port_db` with an `argument of type 'int' is not iterable` error. This bug was partially fixed by a change in Synapse 1.44.0. ([\#12037](https://github.com/matrix-org/synapse/issues/12037))
- Fix slow performance of `/logout` in some cases where refresh tokens are in use. The slowness existed since the initial implementation of refresh tokens in version 1.38.0. ([\#12056](https://github.com/matrix-org/synapse/issues/12056))
- Fix a long-standing bug where Synapse would make additional failing requests over federation for missing data. ([\#12077](https://github.com/matrix-org/synapse/issues/12077))
- Fix occasional `Unhandled error in Deferred` error message. ([\#12089](https://github.com/matrix-org/synapse/issues/12089))
- Fix a bug introduced in Synapse 1.51.0 where incoming federation transactions containing at least one EDU would be dropped if debug logging was enabled for `synapse.8631_debug`. ([\#12098](https://github.com/matrix-org/synapse/issues/12098))
- Fix a long-standing bug which could cause push notifications to malfunction if `use_frozen_dicts` was set in the configuration. ([\#12100](https://github.com/matrix-org/synapse/issues/12100))
- Fix an extremely rare, long-standing bug in `ReadWriteLock` that would cause an error when a newly unblocked writer completes instantly. ([\#12105](https://github.com/matrix-org/synapse/issues/12105))
- Make a `POST` to `/rooms/<room_id>/receipt/m.read/<event_id>` only trigger a push notification if the count of unread messages is different to the one in the last successfully sent push. This reduces server load and load on the receiving device. ([\#11835](https://github.com/matrix-org/synapse/issues/11835))
-
-
-Updates to the Docker image
---------------------------
-
- The Docker image no longer automatically creates a temporary volume at `/data`. This is not expected to affect normal usage. ([\#11997](https://github.com/matrix-org/synapse/issues/11997))
- Use Python 3.9 in Docker images by default. ([\#12112](https://github.com/matrix-org/synapse/issues/12112))
-
-
-Improved Documentation
----------------------
-
- Document support for the `to_device`, `account_data`, `receipts`, and `presence` stream writers for workers. ([\#11599](https://github.com/matrix-org/synapse/issues/11599))
- Explain the meaning of spam checker callbacks' return values. ([\#12003](https://github.com/matrix-org/synapse/issues/12003))
- Clarify information about external Identity Provider IDs. ([\#12004](https://github.com/matrix-org/synapse/issues/12004))
-
-
-Deprecations and Removals
-------------------------
-
- Deprecate using `synctl` with the config option `synctl_cache_factor` and print a warning if a user still uses this option. ([\#11865](https://github.com/matrix-org/synapse/issues/11865))
- Remove support for the legacy structured logging configuration (please see the the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#legacy-structured-logging-configuration-removal) if you are using `structured: true` in the Synapse configuration). ([\#12008](https://github.com/matrix-org/synapse/issues/12008))
- Drop support for [MSC3283](https://github.com/matrix-org/matrix-doc/pull/3283) unstable flags now that the stable flags are supported. ([\#12018](https://github.com/matrix-org/synapse/issues/12018))
- Remove the unstable `/spaces` endpoint from [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946). ([\#12073](https://github.com/matrix-org/synapse/issues/12073))
-
-
-Internal Changes
----------------
-
- Make the `get_room_version` method use `get_room_version_id` to benefit from caching. ([\#11808](https://github.com/matrix-org/synapse/issues/11808))
- Remove unnecessary condition on knock -> leave auth rule check. ([\#11900](https://github.com/matrix-org/synapse/issues/11900))
- Add tests for device list changes between local users. ([\#11972](https://github.com/matrix-org/synapse/issues/11972))
- Optimise calculating `device_list` changes in `/sync`. ([\#11974](https://github.com/matrix-org/synapse/issues/11974))
- Add missing type hints to storage classes. ([\#11984](https://github.com/matrix-org/synapse/issues/11984))
- Refactor the search code for improved readability. ([\#11991](https://github.com/matrix-org/synapse/issues/11991))
- Move common deduplication code down into `_auth_and_persist_outliers`. ([\#11994](https://github.com/matrix-org/synapse/issues/11994))
- Limit concurrent joins from applications services. ([\#11996](https://github.com/matrix-org/synapse/issues/11996))
- Preparation for faster-room-join work: when parsing the `send_join` response, get the `m.room.create` event from `state`, not `auth_chain`. ([\#12005](https://github.com/matrix-org/synapse/issues/12005), [\#12039](https://github.com/matrix-org/synapse/issues/12039))
- Preparation for faster-room-join work: parse MSC3706 fields in send_join response. ([\#12011](https://github.com/matrix-org/synapse/issues/12011))
- Preparation for faster-room-join work: persist information on which events and rooms have partial state to the database. ([\#12012](https://github.com/matrix-org/synapse/issues/12012))
- Preparation for faster-room-join work: Support for calling `/federation/v1/state` on a remote server. ([\#12013](https://github.com/matrix-org/synapse/issues/12013))
- Configure `tox` to use `venv` rather than `virtualenv`. ([\#12015](https://github.com/matrix-org/synapse/issues/12015))
- Fix bug in `StateFilter.return_expanded()` and add some tests. ([\#12016](https://github.com/matrix-org/synapse/issues/12016))
- Use Matrix v1.1 endpoints (`/_matrix/client/v3/auth/...`) in fallback auth HTML forms. ([\#12019](https://github.com/matrix-org/synapse/issues/12019))
- Update the `olddeps` CI job to use an old version of `markupsafe`. ([\#12025](https://github.com/matrix-org/synapse/issues/12025))
- Upgrade Mypy to version 0.931. ([\#12030](https://github.com/matrix-org/synapse/issues/12030))
- Remove legacy `HomeServer.get_datastore()`. ([\#12031](https://github.com/matrix-org/synapse/issues/12031), [\#12070](https://github.com/matrix-org/synapse/issues/12070))
- Minor typing fixes. ([\#12034](https://github.com/matrix-org/synapse/issues/12034), [\#12069](https://github.com/matrix-org/synapse/issues/12069))
- After joining a room, create a dedicated logcontext to process the queued events. ([\#12041](https://github.com/matrix-org/synapse/issues/12041))
- Tidy up GitHub Actions config which builds distributions for PyPI. ([\#12051](https://github.com/matrix-org/synapse/issues/12051))
- Move configuration out of `setup.cfg`. ([\#12052](https://github.com/matrix-org/synapse/issues/12052), [\#12059](https://github.com/matrix-org/synapse/issues/12059))
- Fix error message when a worker process fails to talk to another worker process. ([\#12060](https://github.com/matrix-org/synapse/issues/12060))
- Fix using the `complement.sh` script without specifying a directory or a branch. Contributed by Nico on behalf of Famedly. ([\#12063](https://github.com/matrix-org/synapse/issues/12063))
- Add type hints to `tests/rest/client`. ([\#12066](https://github.com/matrix-org/synapse/issues/12066), [\#12072](https://github.com/matrix-org/synapse/issues/12072), [\#12084](https://github.com/matrix-org/synapse/issues/12084), [\#12094](https://github.com/matrix-org/synapse/issues/12094))
- Add some logging to `/sync` to try and track down #11916. ([\#12068](https://github.com/matrix-org/synapse/issues/12068))
- Inspect application dependencies using `importlib.metadata` or its backport. ([\#12088](https://github.com/matrix-org/synapse/issues/12088))
- Use `assertEqual` instead of the deprecated `assertEquals` in test code. ([\#12092](https://github.com/matrix-org/synapse/issues/12092))
- Move experimental support for [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440) to `/versions`. ([\#12099](https://github.com/matrix-org/synapse/issues/12099))
- Add `stop_cancellation` utility function to stop `Deferred`s from being cancelled. ([\#12106](https://github.com/matrix-org/synapse/issues/12106))
- Improve exception handling for concurrent execution. ([\#12109](https://github.com/matrix-org/synapse/issues/12109))
- Advertise support for Python 3.10 in packaging files. ([\#12111](https://github.com/matrix-org/synapse/issues/12111))
- Move CI checks out of tox, to facilitate a move to using poetry. ([\#12119](https://github.com/matrix-org/synapse/issues/12119))
-
-
 Synapse 1.53.0 (2022-02-22)
 ===========================

@@ -1,3 +1,4 @@
+include synctl
 include LICENSE
 include VERSION
 include *.rst
@@ -16,6 +17,7 @@ recursive-include synapse/storage *.txt
 recursive-include synapse/storage *.md

 recursive-include docs *
+recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
 recursive-include tests *.py
@@ -51,4 +53,5 @@ prune contrib
 prune debian
 prune demo/etc
 prune docker
+prune snap
 prune stubs
@@ -312,9 +312,6 @@ We recommend using the demo which starts 3 federated instances running on ports

 (to stop, you can use `./demo/stop.sh`)

-See the [demo documentation](https://matrix-org.github.io/synapse/develop/development/demo.html)
-for more information.
-
 If you just want to start a single instance of the app and run it directly::

    # Create the homeserver.yaml config once
@@ -0,0 +1 @@
+Deduplicate in-flight requests in `_get_state_for_groups`.
@@ -0,0 +1 @@
+Deduplicate in-flight requests in `_get_state_for_groups`.
@@ -0,0 +1 @@
+Make a `POST` to `/rooms/<room_id>/receipt/m.read/<event_id>` only trigger a push notification if the count of unread messages is different to the one in the last successfully sent push.
@@ -0,0 +1 @@
+Remove unnecessary condition on knock->leave auth rule check.
@@ -0,0 +1 @@
+Add tests for device list changes between local users.
@@ -0,0 +1 @@
+Optimise calculating device_list changes in `/sync`.
@@ -0,0 +1 @@
+Add missing type hints to storage classes.
@@ -0,0 +1 @@
+Fetch images when previewing Twitter URLs. Contributed by @AndrewRyanChama.
@@ -0,0 +1 @@
+Refactor the search code for improved readability.
@@ -0,0 +1 @@
+Fix a bug introduced in Synapse v1.48.0 where an edit of the latest event in a thread would not be properly applied to the thread summary.
@@ -0,0 +1 @@
+Move common deduplication code down into `_auth_and_persist_outliers`.
@@ -0,0 +1 @@
+Limit concurrent joins from applications services.
@@ -0,0 +1 @@
+The docker image no longer automatically creates a temporary volume at `/data`. This is not expected to affect normal usage.
@@ -0,0 +1 @@
+Fix long standing bug where `get_rooms_for_user` was not correctly invalidated for remote users when the server left a room.
@@ -0,0 +1 @@
+Track cache invalidations in Prometheus metrics, as already happens for cache eviction based on size or time.
@@ -0,0 +1 @@
+Implement experimental support for [MSC3720](https://github.com/matrix-org/matrix-doc/pull/3720) (account status endpoints).
@@ -0,0 +1 @@
+Explain the meaning of spam checker callbacks' return values.
@@ -0,0 +1 @@
+Clarify information about external Identity Provider IDs.
@@ -0,0 +1 @@
+Preparation for faster-room-join work: when parsing the `send_join` response, get the `m.room.create` event from `state`, not `auth_chain`.
@@ -0,0 +1 @@
+Remove support for the legacy structured logging configuration (please see the the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#legacy-structured-logging-configuration-removal) if you are using `structured: true` in the Synapse configuration).
@@ -0,0 +1 @@
+Enable modules to set a custom display name when registering a user.
@@ -0,0 +1 @@
+Preparation for faster-room-join work: parse msc3706 fields in send_join response.
@@ -0,0 +1 @@
+Preparation for faster-room-join work: Support for calling `/federation/v1/state` on a remote server.
@@ -0,0 +1 @@
+Configure `tox` to use `venv` rather than `virtualenv`.
@@ -0,0 +1 @@
+Fix bug in `StateFilter.return_expanded()` and add some tests.
@@ -0,0 +1 @@
+Drop support for [MSC3283](https://github.com/matrix-org/matrix-doc/pull/3283) unstable flags now that the stable flags are supported.
@@ -0,0 +1 @@
+Use Matrix v1.1 endpoints (`/_matrix/client/v3/auth/...`) in fallback auth HTML forms.
@@ -0,0 +1 @@
+Advertise Matrix 1.1 support on `/_matrix/client/versions`.
@@ -0,0 +1 @@
+Support only the stable identifier for [MSC3069](https://github.com/matrix-org/matrix-doc/pull/3069)'s `is_guest` on `/_matrix/client/v3/account/whoami`.
@@ -0,0 +1 @@
+Advertise Matrix 1.2 support on `/_matrix/client/versions`.
@@ -0,0 +1 @@
+Fix 500 error with Postgres when looking backwards with the [MSC3030](https://github.com/matrix-org/matrix-doc/pull/3030) `/timestamp_to_event?dir=b` endpoint.
@@ -0,0 +1 @@
+Update the `olddeps` CI job to use an old version of `markupsafe`.
@@ -0,0 +1 @@
+Upgrade mypy to version 0.931.
@@ -0,0 +1 @@
+Remove legacy `HomeServer.get_datastore()`.
@@ -0,0 +1 @@
+Deduplicate in-flight requests in `_get_state_for_groups`.
@@ -0,0 +1 @@
+Minor typing fixes.
@@ -0,0 +1 @@
+Preparation for faster-room-join work: when parsing the `send_join` response, get the `m.room.create` event from `state`, not `auth_chain`.
@@ -0,0 +1 @@
+After joining a room, create a dedicated logcontext to process the queued events.
@@ -0,0 +1 @@
+Tidy up GitHub Actions config which builds distributions for PyPI.
@@ -0,0 +1 @@
+Move configuration out of `setup.cfg`.
@@ -0,0 +1 @@
+Fix slow performance of `/logout` in some cases where refresh tokens are in use. The slowness existed since the initial implementation of refresh tokens.
@@ -0,0 +1 @@
+Use room version 9 as the default room version (per [MSC3589](https://github.com/matrix-org/matrix-doc/pull/3589)).
@@ -0,0 +1 @@
+Move configuration out of `setup.cfg`.
@@ -0,0 +1 @@
+Fix error message when a worker process fails to talk to another worker process.
@@ -0,0 +1 @@
+Fix using the complement.sh script without specifying a dir or a branch. Contributed by Nico on behalf of Famedly.
@@ -1,33 +1,3 @@
-matrix-synapse-py3 (1.55.1) stable; urgency=medium
-
-  * New synapse release 1.55.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 24 Mar 2022 17:44:23 +0000
-
-matrix-synapse-py3 (1.55.0) stable; urgency=medium
-
-  * New synapse release 1.55.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 22 Mar 2022 13:59:26 +0000
-
-matrix-synapse-py3 (1.55.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.55.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 15 Mar 2022 10:59:31 +0000
-
-matrix-synapse-py3 (1.54.0) stable; urgency=medium
-
-  * New synapse release 1.54.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 08 Mar 2022 10:54:52 +0000
-
-matrix-synapse-py3 (1.54.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.54.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 02 Mar 2022 10:43:22 +0000
-
 matrix-synapse-py3 (1.53.0) stable; urgency=medium

  * New synapse release 1.53.0.
@@ -1,4 +1,7 @@
-# Ignore all the temporary files from the demo servers.
-8080/
-8081/
-8082/
+*.db
+*.log
+*.log.*
+*.pid
+
+/media_store.*
+/etc
@@ -0,0 +1,26 @@
+DO NOT USE THESE DEMO SERVERS IN PRODUCTION
+
+Requires you to have done:
+    python setup.py develop
+
+
+The demo start.sh will start three synapse servers on ports 8080, 8081 and 8082, with host names localhost:$port. This can be easily changed to `hostname`:$port in start.sh if required.
+
+To enable the servers to communicate untrusted ssl certs are used. In order to do this the servers do not check the certs
+and are configured in a highly insecure way. Do not use these configuration files in production.
+
+stop.sh will stop the synapse servers and the webclient.
+
+clean.sh will delete the databases and log files.
+
+To start a completely new set of servers, run:
+
+    ./demo/stop.sh; ./demo/clean.sh && ./demo/start.sh
+
+
+Logs and sqlitedb will be stored in demo/808{0,1,2}.{log,db}
+
+
+
+Also note that when joining a public room on a different HS via "#foo:bar.net", then you are (in the current impl) joining a room with room_id "foo". This means that it won't work if your HS already has a room with that name.
+
@@ -4,9 +4,6 @@ set -e

 DIR="$( cd "$( dirname "$0" )" && pwd )"

-# Ensure that the servers are stopped.
-$DIR/stop.sh
-
 PID_FILE="$DIR/servers.pid"

 if [ -f "$PID_FILE" ]; then
@@ -6,6 +6,8 @@ CWD=$(pwd)

 cd "$DIR/.." || exit

+mkdir -p demo/etc
+
 PYTHONPATH=$(readlink -f "$(pwd)")
 export PYTHONPATH

@@ -19,26 +21,22 @@ for port in 8080 8081 8082; do
    mkdir -p demo/$port
    pushd demo/$port || exit

-    # Generate the configuration for the homeserver at localhost:848x.
+    #rm $DIR/etc/$port.config
    python3 -m synapse.app.homeserver \
        --generate-config \
-        --server-name "localhost:$port" \
-        --config-path "$port.config" \
+        -H "localhost:$https_port" \
+        --config-path "$DIR/etc/$port.config" \
        --report-stats no

-    if ! grep -F "Customisation made by demo/start.sh" -q "$port.config"; then
-        # Generate TLS keys.
-        openssl req -x509 -newkey rsa:4096 \
-          -keyout "localhost:$port.tls.key" \
-          -out "localhost:$port.tls.crt" \
-          -days 365 -nodes -subj "/O=matrix"
+    if ! grep -F "Customisation made by demo/start.sh" -q "$DIR/etc/$port.config"; then
+        # Generate tls keys
+        openssl req -x509 -newkey rsa:4096 -keyout "$DIR/etc/localhost:$https_port.tls.key" -out "$DIR/etc/localhost:$https_port.tls.crt" -days 365 -nodes -subj "/O=matrix"

-        # Add customisations to the configuration.
+        # Regenerate configuration
        {
-            printf '\n\n# Customisation made by demo/start.sh\n\n'
+            printf '\n\n# Customisation made by demo/start.sh\n'
            echo "public_baseurl: http://localhost:$port/"
            echo 'enable_registration: true'
-            echo ''

 			# Warning, this heredoc depends on the interaction of tabs and spaces.
 			# Please don't accidentaly bork me with your fancy settings.
@@ -65,34 +63,38 @@ for port in 8080 8081 8082; do

            echo "${listeners}"

-            # Disable TLS for the servers
-            printf '\n\n# Disable TLS for the servers.'
+            # Disable tls for the servers
+            printf '\n\n# Disable tls on the servers.'
            echo '# DO NOT USE IN PRODUCTION'
            echo 'use_insecure_ssl_client_just_for_testing_do_not_use: true'
            echo 'federation_verify_certificates: false'

-            # Set paths for the TLS certificates.
-            echo "tls_certificate_path: \"$DIR/$port/localhost:$port.tls.crt\""
-            echo "tls_private_key_path: \"$DIR/$port/localhost:$port.tls.key\""
+            # Set tls paths
+            echo "tls_certificate_path: \"$DIR/etc/localhost:$https_port.tls.crt\""
+            echo "tls_private_key_path: \"$DIR/etc/localhost:$https_port.tls.key\""

            # Ignore keys from the trusted keys server
            echo '# Ignore keys from the trusted keys server'
            echo 'trusted_key_servers:'
            echo '  - server_name: "matrix.org"'
            echo '    accept_keys_insecurely: true'
-            echo ''

-			# Allow the servers to communicate over localhost.
-			allow_list=$(cat <<-ALLOW_LIST
-			# Allow the servers to communicate over localhost.
-			ip_range_whitelist:
-			  - '127.0.0.1/8'
-			  - '::1/128'
-			ALLOW_LIST
+			# Reduce the blacklist
+			blacklist=$(cat <<-BLACK
+			# Set the blacklist so that it doesn't include 127.0.0.1, ::1
+			federation_ip_range_blacklist:
+			  - '10.0.0.0/8'
+			  - '172.16.0.0/12'
+			  - '192.168.0.0/16'
+			  - '100.64.0.0/10'
+			  - '169.254.0.0/16'
+			  - 'fe80::/64'
+			  - 'fc00::/7'
+			BLACK
 			)

-            echo "${allow_list}"
-        } >> "$port.config"
+            echo "${blacklist}"
+        } >> "$DIR/etc/$port.config"
    fi

    # Check script parameters
@@ -139,18 +141,19 @@ for port in 8080 8081 8082; do
 			    burst_count: 1000
 			RC
 			)
-            echo "${ratelimiting}" >> "$port.config"
+            echo "${ratelimiting}" >> "$DIR/etc/$port.config"
        fi
    fi

-    # Always disable reporting of stats if the option is not there.
-    if ! grep -F "report_stats" -q  "$port.config" ; then
-        echo "report_stats: false" >> "$port.config"
+    if ! grep -F "full_twisted_stacktraces" -q  "$DIR/etc/$port.config"; then
+        echo "full_twisted_stacktraces: true" >> "$DIR/etc/$port.config"
+    fi
+    if ! grep -F "report_stats" -q  "$DIR/etc/$port.config" ; then
+        echo "report_stats: false" >> "$DIR/etc/$port.config"
    fi

-    # Run the homeserver in the background.
    python3 -m synapse.app.homeserver \
-        --config-path "$port.config" \
+        --config-path "$DIR/etc/$port.config" \
        -D \

    popd || exit
@@ -11,10 +11,10 @@
 # There is an optional PYTHON_VERSION build argument which sets the
 # version of python to build against: for example:
 #
-#    DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.10 .
+#    DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.9 .
 #

-ARG PYTHON_VERSION=3.9
+ARG PYTHON_VERSION=3.8

 ###
 ### Stage 0: builder
@@ -46,7 +46,8 @@ RUN \
    && rm -rf /var/lib/apt/lists/*

 # Copy just what we need to pip install
-COPY MANIFEST.in README.rst setup.py /synapse/
+COPY scripts /synapse/scripts/
+COPY MANIFEST.in README.rst setup.py synctl /synapse/
 COPY synapse/__init__.py /synapse/synapse/__init__.py
 COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py

@@ -82,7 +82,6 @@
  - [Release Cycle](development/releases.md)
  - [Git Usage](development/git.md)
  - [Testing]()
-    - [Demo scripts](development/demo.md)
  - [OpenTracing](opentracing.md)
  - [Database Schemas](development/database_schema.md)
  - [Experimental features](development/experimental_features.md)
@@ -172,6 +172,6 @@ frobber:
 ```

 Note that the sample configuration is generated from the synapse code
-and is maintained by a script, `scripts-dev/generate_sample_config.sh`.
+and is maintained by a script, `scripts-dev/generate_sample_config`.
 Making sure that the output from this script matches the desired format
 is left as an exercise for the reader!
@@ -458,17 +458,6 @@ Git allows you to add this signoff automatically when using the `-s`
 flag to `git commit`, which uses the name and email set in your
 `user.name` and `user.email` git configs.

-### Private Sign off
-
-If you would like to provide your legal name privately to the Matrix.org
-Foundation (instead of in a public commit or comment), you can do so
-by emailing your legal name and a link to the pull request to
-[dco@matrix.org](mailto:dco@matrix.org?subject=Private%20sign%20off).
-It helps to include "sign off" or similar in the subject line. You will then
-be instructed further.
-
-Once private sign off is complete, doing so for future contributions will not
-be required.

 # 10. Turn feedback into better code.

@@ -158,9 +158,9 @@ same as integers.
 There are three separate aspects to this:

 * Any new boolean column must be added to the `BOOLEAN_COLUMNS` list in
-   `synapse/_scripts/synapse_port_db.py`. This tells the port script to cast
-   the integer value from SQLite to a boolean before writing the value to the
-   postgres database.
+   `scripts/synapse_port_db`. This tells the port script to cast the integer
+   value from SQLite to a boolean before writing the value to the postgres
+   database.

 * Before SQLite 3.23, `TRUE` and `FALSE` were not recognised as constants by
   SQLite, and the `IS [NOT] TRUE`/`IS [NOT] FALSE` operators were not
@@ -1,41 +0,0 @@
-# Synapse demo setup
-
-**DO NOT USE THESE DEMO SERVERS IN PRODUCTION**
-
-Requires you to have a [Synapse development environment setup](https://matrix-org.github.io/synapse/develop/development/contributing_guide.html#4-install-the-dependencies).
-
-The demo setup allows running three federation Synapse servers, with server
-names `localhost:8080`, `localhost:8081`, and `localhost:8082`.
-
-You can access them via any Matrix client over HTTP at `localhost:8080`,
-`localhost:8081`, and `localhost:8082` or over HTTPS at `localhost:8480`,
-`localhost:8481`, and `localhost:8482`.
-
-To enable the servers to communicate, self-signed SSL certificates are generated
-and the servers are configured in a highly insecure way, including:
-
-* Not checking certificates over federation.
-* Not verifying keys.
-
-The servers are configured to store their data under `demo/8080`, `demo/8081`, and
-`demo/8082`. This includes configuration, logs, SQLite databases, and media.
-
-Note that when joining a public room on a different HS via "#foo:bar.net", then
-you are (in the current impl) joining a room with room_id "foo". This means that
-it won't work if your HS already has a room with that name.
-
-## Using the demo scripts
-
-There's three main scripts with straightforward purposes:
-
-* `start.sh` will start the Synapse servers, generating any missing configuration.
-  * This accepts a single parameter `--no-rate-limit` to "disable" rate limits
-    (they actually still exist, but are very high).
-* `stop.sh` will stop the Synapse servers.
-* `clean.sh` will delete the configuration, databases, log files, etc.
-
-To start a completely new set of servers, run:
-
-```sh
-./demo/stop.sh; ./demo/clean.sh && ./demo/start.sh
-```
@@ -30,57 +30,13 @@ rather than skipping any that arrived late; whereas if you're looking at a
 historical section of timeline (i.e. `/messages`), you want to see the best
 representation of the state of the room as others were seeing it at the time.

-## Outliers
-
-We mark an event as an `outlier` when we haven't figured out the state for the
-room at that point in the DAG yet. They are "floating" events that we haven't
-yet correlated to the DAG.
-
-Outliers typically arise when we fetch the auth chain or state for a given
-event. When that happens, we just grab the events in the state/auth chain,
-without calculating the state at those events, or backfilling their
-`prev_events`.
-
-So, typically, we won't have the `prev_events` of an `outlier` in the database,
-(though it's entirely possible that we *might* have them for some other
-reason). Other things that make outliers different from regular events:
-
- * We don't have state for them, so there should be no entry in
-   `event_to_state_groups` for an outlier. (In practice this isn't always
-   the case, though I'm not sure why: see https://github.com/matrix-org/synapse/issues/12201).
-
- * We don't record entries for them in the `event_edges`,
-   `event_forward_extremeties` or `event_backward_extremities` tables.
-
-Since outliers are not tied into the DAG, they do not normally form part of the
-timeline sent down to clients via `/sync` or `/messages`; however there is an
-exception:
-
-### Out-of-band membership events
-
-A special case of outlier events are some membership events for federated rooms
-that we aren't full members of. For example:
-
- * invites received over federation, before we join the room
- * *rejections* for said invites
- * knock events for rooms that we would like to join but have not yet joined.
-
-In all the above cases, we don't have the state for the room, which is why they
-are treated as outliers. They are a bit special though, in that they are
-proactively sent to clients via `/sync`.

 ## Forward extremity

-Most-recent-in-time events in the DAG which are not referenced by any other
-events' `prev_events` yet. (In this definition, outliers, rejected events, and
-soft-failed events don't count.)
+Most-recent-in-time events in the DAG which are not referenced by any other events' `prev_events` yet.

-The forward extremities of a room (or at least, a subset of them, if there are
-more than ten) are used as the `prev_events` when the next event is sent.
+The forward extremities of a room are used as the `prev_events` when the next event is sent.

-The "current state" of a room (ie: the state which would be used if we
-generated a new event) is, therefore, the resolution of the room states
-at each of the forward extremities.

 ## Backward extremity

@@ -88,14 +44,23 @@ The current marker of where we have backfilled up to and will generally be the
 `prev_events` of the oldest-in-time events we have in the DAG. This gives a starting point when
 backfilling history.

-Note that, unlike forward extremities, we typically don't have any backward
-extremity events themselves in the database - or, if we do, they will be "outliers" (see
-above). Either way, we don't expect to have the room state at a backward extremity.
+When we persist a non-outlier event, we clear it as a backward extremity and set
+all of its `prev_events` as the new backward extremities if they aren't already
+persisted in the `events` table.
+
+
+## Outliers
+
+We mark an event as an `outlier` when we haven't figured out the state for the
+room at that point in the DAG yet.
+
+We won't *necessarily* have the `prev_events` of an `outlier` in the database,
+but it's entirely possible that we *might*.
+
+For example, when we fetch the event auth chain or state for a given event, we
+mark all of those claimed auth events as outliers because we haven't done the
+state calculation ourself.

-When we persist a non-outlier event, if it was previously a backward extremity,
-we clear it as a backward extremity and set all of its `prev_events` as the new
-backward extremities if they aren't already persisted as non-outliers. This
-therefore keeps the backward extremities up-to-date.

 ## State groups

@@ -63,5 +63,4 @@ release of Synapse.

 If you want to get up and running quickly with a trio of homeservers in a
 private federation, there is a script in the `demo` directory. This is mainly
-useful just for development purposes. See
-[demo scripts](https://matrix-org.github.io/synapse/develop/development/demo.html).
+useful just for development purposes. See [demo/README](https://github.com/matrix-org/synapse/tree/develop/demo/).
@@ -148,105 +148,6 @@ deny an incoming event, see [`check_event_for_spam`](spam_checker_callbacks.md#c

 If multiple modules implement this callback, Synapse runs them all in order.

-### `check_can_shutdown_room`
-
-_First introduced in Synapse v1.55.0_
-
-```python
-async def check_can_shutdown_room(
-    user_id: str, room_id: str,
-) -> bool:
-```
-
-Called when an admin user requests the shutdown of a room. The module must return a
-boolean indicating whether the shutdown can go through. If the callback returns `False`,
-the shutdown will not proceed and the caller will see a `M_FORBIDDEN` error.
-
-If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
-
-### `check_can_deactivate_user`
-
-_First introduced in Synapse v1.55.0_
-
-```python
-async def check_can_deactivate_user(
-    user_id: str, by_admin: bool,
-) -> bool:
-```
-
-Called when the deactivation of a user is requested. User deactivation can be
-performed by an admin or the user themselves, so developers are encouraged to check the
-requester when implementing this callback. The module must return a
-boolean indicating whether the deactivation can go through. If the callback returns `False`,
-the deactivation will not proceed and the caller will see a `M_FORBIDDEN` error.
-
-The module is passed two parameters, `user_id` which is the ID of the user being deactivated, and `by_admin` which is `True` if the request is made by a serve admin, and `False` otherwise.
-
-If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
-
-
-### `on_profile_update`
-
-_First introduced in Synapse v1.54.0_
-
-```python
-async def on_profile_update(
-    user_id: str,
-    new_profile: "synapse.module_api.ProfileInfo",
-    by_admin: bool,
-    deactivation: bool,
-) -> None:
-```
-
-Called after updating a local user's profile. The update can be triggered either by the
-user themselves or a server admin. The update can also be triggered by a user being
-deactivated (in which case their display name is set to an empty string (`""`) and the
-avatar URL is set to `None`). The module is passed the Matrix ID of the user whose profile
-has been updated, their new profile, as well as a `by_admin` boolean that is `True` if the
-update was triggered by a server admin (and `False` otherwise), and a `deactivated`
-boolean that is `True` if the update is a result of the user being deactivated.
-
-Note that the `by_admin` boolean is also `True` if the profile change happens as a result
-of the user logging in through Single Sign-On, or if a server admin updates their own
-profile.
-
-Per-room profile changes do not trigger this callback to be called. Synapse administrators
-wishing this callback to be called on every profile change are encouraged to disable
-per-room profiles globally using the `allow_per_room_profiles` configuration setting in
-Synapse's configuration file.
-This callback is not called when registering a user, even when setting it through the
-[`get_displayname_for_registration`](https://matrix-org.github.io/synapse/latest/modules/password_auth_provider_callbacks.html#get_displayname_for_registration)
-module callback.
-
-If multiple modules implement this callback, Synapse runs them all in order.
-
-### `on_user_deactivation_status_changed`
-
-_First introduced in Synapse v1.54.0_
-
-```python
-async def on_user_deactivation_status_changed(
-    user_id: str, deactivated: bool, by_admin: bool
-) -> None:
-```
-
-Called after deactivating a local user, or reactivating them through the admin API. The
-deactivation can be triggered either by the user themselves or a server admin. The module
-is passed the Matrix ID of the user whose status is changed, as well as a `deactivated`
-boolean that is `True` if the user is being deactivated and `False` if they're being
-reactivated, and a `by_admin` boolean that is `True` if the deactivation was triggered by
-a server admin (and `False` otherwise). This latter `by_admin` boolean is always `True`
-if the user is being reactivated, as this operation can only be performed through the
-admin API.
-
-If multiple modules implement this callback, Synapse runs them all in order.
-
 ## Example

 The example below is a module that implements the third-party rules callback
@@ -31,29 +31,28 @@ Anything that requires modifying the device list [#7721](https://github.com/matr
 Put the below in a new file at /etc/matrix-synapse/conf.d/sbc.yaml to override the defaults in homeserver.yaml.

 ```
-# Disable presence tracking, which is currently fairly resource intensive
-# More info: https://github.com/matrix-org/synapse/issues/9478
+# Set to false to disable presence tracking on this homeserver.
 use_presence: false

-# Set a small complexity limit, preventing users from joining large rooms
-# which may be resource-intensive to remain a part of.
-#
-# Note that this will not prevent users from joining smaller rooms that
-# eventually become complex.
+# When this is enabled, the room "complexity" will be checked before a user
+# joins a new remote room. If it is above the complexity limit, the server will
+# disallow joining, or will instantly leave.
 limit_remote_rooms:
-  enabled: true
+  # Uncomment to enable room complexity checking.
+  #enabled: true
  complexity: 3.0

 # Database configuration
 database:
-  # Use postgres for the best performance
  name: psycopg2
  args:
    user: matrix-synapse
-    # Generate a long, secure password using a password manager
+    # Generate a long, secure one with a password manager
    password: hunter2
    database: matrix-synapse
    host: localhost
+    cp_min: 5
+    cp_max: 10
 ```

 Currently the complexity is measured by [current_state_events / 500](https://github.com/matrix-org/synapse/blob/v1.20.1/synapse/storage/databases/main/events_worker.py#L986). You can find join times and your most complex rooms like this:
@@ -153,9 +153,9 @@ database file (typically `homeserver.db`) to another location. Once the
 copy is complete, restart synapse. For instance:

 ```sh
-synctl stop
+./synctl stop
 cp homeserver.db homeserver.db.snapshot
-synctl start
+./synctl start
 ```

 Copy the old config file into a new config file:
@@ -192,10 +192,10 @@ Once that has completed, change the synapse config to point at the
 PostgreSQL database configuration file `homeserver-postgres.yaml`:

 ```sh
-synctl stop
+./synctl stop
 mv homeserver.yaml homeserver-old-sqlite.yaml
 mv homeserver-postgres.yaml homeserver.yaml
-synctl start
+./synctl start
 ```

 Synapse should now be running against PostgreSQL.
@@ -1947,14 +1947,8 @@ saml2_config:
 #
 #             localpart_template: Jinja2 template for the localpart of the MXID.
 #                 If this is not set, the user will be prompted to choose their
-#                 own username (see the documentation for the
-#                 'sso_auth_account_details.html' template). This template can
-#                 use the 'localpart_from_email' filter.
-#
-#             confirm_localpart: Whether to prompt the user to validate (or
-#                 change) the generated localpart (see the documentation for the
-#                 'sso_auth_account_details.html' template), instead of
-#                 registering the account right away.
+#                 own username (see 'sso_auth_account_details.html' in the 'sso'
+#                 section of this file).
 #
 #             display_name_template: Jinja2 template for the display name to set
 #                 on first login. If unset, no displayname will be set.
@@ -2735,35 +2729,3 @@ redis:
  # Optional password if configured on the Redis instance
  #
  #password: <secret_password>
-
-
-## Background Updates ##
-
-# Background updates are database updates that are run in the background in batches.
-# The duration, minimum batch size, default batch size, whether to sleep between batches and if so, how long to
-# sleep can all be configured. This is helpful to speed up or slow down the updates.
-#
-background_updates:
-    # How long in milliseconds to run a batch of background updates for. Defaults to 100. Uncomment and set
-    # a time to change the default.
-    #
-    #background_update_duration_ms: 500
-
-    # Whether to sleep between updates. Defaults to True. Uncomment to change the default.
-    #
-    #sleep_enabled: false
-
-    # If sleeping between updates, how long in milliseconds to sleep for. Defaults to 1000. Uncomment
-    # and set a duration to change the default.
-    #
-    #sleep_duration_ms: 300
-
-    # Minimum size a batch of background updates can be. Must be greater than 0. Defaults to 1. Uncomment and
-    # set a size to change the default.
-    #
-    #min_batch_size: 10
-
-    # The batch size to use for the first iteration of a new background update. The default is 100.
-    # Uncomment and set a size to change the default.
-    #
-    #default_batch_size: 50
@@ -36,13 +36,6 @@ Turns a `mxc://` URL for media content into an HTTP(S) one using the homeserver'

 Example: `message.sender_avatar_url|mxc_to_http(32,32)`

-```python
-localpart_from_email(address: str) -> str
-```
-
-Returns the local part of an email address (e.g. `alice` in `alice@example.com`).
-
-Example: `user.email_address|localpart_from_email`

 ## Email templates

@@ -183,11 +176,8 @@ Below are the templates Synapse will look for when generating pages related to S
             for the brand of the IdP
    * `user_attributes`: an object containing details about the user that
      we received from the IdP. May have the following attributes:
-        * `display_name`: the user's display name
-        * `emails`: a list of email addresses
-        * `localpart`: the local part of the Matrix user ID to register,
-          if `localpart_template` is set in the mapping provider configuration (empty
-          string if not)
+        * display_name: the user's display_name
+        * emails: a list of email addresses
  The template should render a form which submits the following fields:
    * `username`: the localpart of the user's chosen user id
 * `sso_new_user_consent.html`: HTML page allowing the user to consent to the
@@ -238,9 +238,8 @@ After updating the homeserver configuration, you must restart synapse:

  * If you use synctl:
    ```sh
-    # Depending on how Synapse is installed, synctl may already be on
-    # your PATH. If not, you may need to activate a virtual environment.
-    synctl restart
+    cd /where/you/run/synapse
+    ./synctl restart
    ```
  * If you use systemd:
    ```sh
@@ -47,7 +47,7 @@ this document.
    3.  Restart Synapse:

        ```bash
-        synctl restart
+        ./synctl restart
        ```

 To check whether your update was successful, you can check the running
@@ -85,49 +85,6 @@ process, for example:
    dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
    ```

-# Upgrading to v1.56.0
-
-## Groups/communities feature has been deprecated
-
-The non-standard groups/communities feature in Synapse has been deprecated and will
-be disabled by default in Synapse v1.58.0.
-
-You can test disabling it by adding the following to your homeserver configuration:
-
-```yaml
-experimental_features:
-  groups_enabled: false
-```
-
-# Upgrading to v1.55.0
-
-## `synctl` script has been moved
-
-The `synctl` script
-[has been made](https://github.com/matrix-org/synapse/pull/12140) an
-[entry point](https://packaging.python.org/en/latest/specifications/entry-points/)
-and no longer exists at the root of Synapse's source tree. If you wish to use
-`synctl` to manage your homeserver, you should invoke `synctl` directly, e.g. 
-`synctl start` instead of `./synctl start` or `/path/to/synctl start`. 
-
-You will need to ensure `synctl` is on your `PATH`.
-  - This is automatically the case when using
-    [Debian packages](https://packages.matrix.org/debian/) or
-    [docker images](https://hub.docker.com/r/matrixdotorg/synapse)
-    provided by Matrix.org.
-  - When installing from a wheel, sdist, or PyPI, a `synctl` executable is added 
-    to your Python installation's `bin`. This should be on your `PATH`
-    automatically, though you might need to activate a virtual environment
-    depending on how you installed Synapse.
-
-
-## Compatibility dropped for Mjolnir 1.3.1 and earlier
-
-Synapse v1.55.0 drops support for Mjolnir 1.3.1 and earlier.
-If you use the Mjolnir module to moderate your homeserver,
-please upgrade Mjolnir to version 1.3.2 or later before upgrading Synapse.
-
-
 # Upgrading to v1.54.0

 ## Legacy structured logging configuration removal
@@ -12,7 +12,7 @@ UPDATE users SET admin = 1 WHERE name = '@foo:bar.com';
 ```

 A new server admin user can also be created using the `register_new_matrix_user`
-command. This is a script that is distributed as part of synapse. It is possibly
+command. This is a script that is located in the `scripts/` directory, or possibly
 already on your `$PATH` depending on how Synapse was installed.

 Finding your user's `access_token` is client-dependent, but will usually be shown in the client's settings.
@@ -178,11 +178,8 @@ recommend the use of `systemd` where available: for information on setting up

 ### `synapse.app.generic_worker`

-This worker can handle API requests matching the following regular expressions.
-These endpoints can be routed to any worker. If a worker is set up to handle a
-stream then, for maximum efficiency, additional endpoints should be routed to that
-worker: refer to the [stream writers](#stream-writers) section below for further
-information.
+This worker can handle API requests matching the following regular
+expressions:

    # Sync requests
    ^/_matrix/client/(v2_alpha|r0|v3)/sync$
@@ -212,6 +209,7 @@ information.
    ^/_matrix/federation/v1/user/devices/
    ^/_matrix/federation/v1/get_groups_publicised$
    ^/_matrix/key/v2/query
+    ^/_matrix/federation/unstable/org.matrix.msc2946/spaces/
    ^/_matrix/federation/(v1|unstable/org.matrix.msc2946)/hierarchy/

    # Inbound federation transaction request
@@ -224,25 +222,22 @@ information.
    ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/context/.*$
    ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/members$
    ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/state$
+    ^/_matrix/client/unstable/org.matrix.msc2946/rooms/.*/spaces$
    ^/_matrix/client/(v1|unstable/org.matrix.msc2946)/rooms/.*/hierarchy$
    ^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$
-    ^/_matrix/client/(r0|v3|unstable)/account/3pid$
-    ^/_matrix/client/(r0|v3|unstable)/devices$
+    ^/_matrix/client/(api/v1|r0|v3|unstable)/account/3pid$
+    ^/_matrix/client/(api/v1|r0|v3|unstable)/devices$
+    ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/query$
+    ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/changes$
    ^/_matrix/client/versions$
    ^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$
-    ^/_matrix/client/(r0|v3|unstable)/joined_groups$
-    ^/_matrix/client/(r0|v3|unstable)/publicised_groups$
-    ^/_matrix/client/(r0|v3|unstable)/publicised_groups/
+    ^/_matrix/client/(api/v1|r0|v3|unstable)/joined_groups$
+    ^/_matrix/client/(api/v1|r0|v3|unstable)/publicised_groups$
+    ^/_matrix/client/(api/v1|r0|v3|unstable)/publicised_groups/
    ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/event/
    ^/_matrix/client/(api/v1|r0|v3|unstable)/joined_rooms$
    ^/_matrix/client/(api/v1|r0|v3|unstable)/search$

-    # Encryption requests
-    ^/_matrix/client/(r0|v3|unstable)/keys/query$
-    ^/_matrix/client/(r0|v3|unstable)/keys/changes$
-    ^/_matrix/client/(r0|v3|unstable)/keys/claim$
-    ^/_matrix/client/(r0|v3|unstable)/room_keys/
-
    # Registration/login requests
    ^/_matrix/client/(api/v1|r0|v3|unstable)/login$
    ^/_matrix/client/(r0|v3|unstable)/register$
@@ -256,20 +251,6 @@ information.
    ^/_matrix/client/(api/v1|r0|v3|unstable)/join/
    ^/_matrix/client/(api/v1|r0|v3|unstable)/profile/

-    # Device requests
-    ^/_matrix/client/(r0|v3|unstable)/sendToDevice/
-
-    # Account data requests
-    ^/_matrix/client/(r0|v3|unstable)/.*/tags
-    ^/_matrix/client/(r0|v3|unstable)/.*/account_data
-
-    # Receipts requests
-    ^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt
-    ^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers
-
-    # Presence requests
-    ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/
-

 Additionally, the following REST endpoints can be handled for GET requests:

@@ -349,13 +330,12 @@ Additionally, there is *experimental* support for moving writing of specific
 streams (such as events) off of the main process to a particular worker. (This
 is only supported with Redis-based replication.)

-To enable this, the worker must have a HTTP replication listener configured,
-have a `worker_name` and be listed in the `instance_map` config. The same worker
-can handle multiple streams, but unless otherwise documented, each stream can only
-have a single writer.
+Currently supported streams are `events` and `typing`.

-For example, to move event persistence off to a dedicated worker, the shared
-configuration would include:
+To enable this, the worker must have a HTTP replication listener configured,
+have a `worker_name` and be listed in the `instance_map` config. For example to
+move event persistence off to a dedicated worker, the shared configuration would
+include:

 ```yaml
 instance_map:
@@ -367,14 +347,8 @@ stream_writers:
    events: event_persister1
 ```

-Some of the streams have associated endpoints which, for maximum efficiency, should
-be routed to the workers handling that stream. See below for the currently supported
-streams and the endpoints associated with them:
-
-##### The `events` stream
-
-The `events` stream experimentally supports having multiple writers, where work
-is sharded between them by room ID. Note that you *must* restart all worker
+The `events` stream also experimentally supports having multiple writers, where
+work is sharded between them by room ID. Note that you *must* restart all worker
 instances when adding or removing event persisters. An example `stream_writers`
 configuration with multiple writers:

@@ -385,43 +359,6 @@ stream_writers:
        - event_persister2
 ```

-##### The `typing` stream
-
-The following endpoints should be routed directly to the worker configured as
-the stream writer for the `typing` stream:
-
-    ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/typing
-
-##### The `to_device` stream
-
-The following endpoints should be routed directly to the worker configured as
-the stream writer for the `to_device` stream:
-
-    ^/_matrix/client/(api/v1|r0|v3|unstable)/sendToDevice/
-
-##### The `account_data` stream
-
-The following endpoints should be routed directly to the worker configured as
-the stream writer for the `account_data` stream:
-
-    ^/_matrix/client/(api/v1|r0|v3|unstable)/.*/tags
-    ^/_matrix/client/(api/v1|r0|v3|unstable)/.*/account_data
-
-##### The `receipts` stream
-
-The following endpoints should be routed directly to the worker configured as
-the stream writer for the `receipts` stream:
-
-    ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/receipt
-    ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/read_markers
-
-##### The `presence` stream
-
-The following endpoints should be routed directly to the worker configured as
-the stream writer for the `presence` stream:
-
-    ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/
-
 #### Background tasks

 There is also *experimental* support for moving background tasks to a separate
@@ -11,7 +11,7 @@ local_partial_types = True
 no_implicit_optional = True

 files =
-  scripts-dev/,
+  scripts-dev/sign_json,
  setup.py,
  synapse/,
  tests/
@@ -23,20 +23,6 @@ files =
 # https://docs.python.org/3/library/re.html#re.X
 exclude = (?x)
  ^(
-   |scripts-dev/build_debian_packages.py
-   |scripts-dev/check_signature.py
-   |scripts-dev/definitions.py
-   |scripts-dev/federation_client.py
-   |scripts-dev/hash_history.py
-   |scripts-dev/list_url_patterns.py
-   |scripts-dev/release.py
-   |scripts-dev/tail-synapse.py
-
-   |synapse/_scripts/export_signing_key.py
-   |synapse/_scripts/move_remote_media_to_new_store.py
-   |synapse/_scripts/synapse_port_db.py
-   |synapse/_scripts/update_synapse_database.py
-
   |synapse/storage/databases/__init__.py
   |synapse/storage/databases/main/__init__.py
   |synapse/storage/databases/main/cache.py
@@ -88,8 +74,21 @@ exclude = (?x)
   |tests/push/test_http.py
   |tests/push/test_presentable_names.py
   |tests/push/test_push_rule_evaluator.py
+   |tests/rest/client/test_account.py
+   |tests/rest/client/test_events.py
+   |tests/rest/client/test_filter.py
+   |tests/rest/client/test_groups.py
+   |tests/rest/client/test_register.py
+   |tests/rest/client/test_report_event.py
+   |tests/rest/client/test_rooms.py
+   |tests/rest/client/test_third_party_rules.py
   |tests/rest/client/test_transactions.py
+   |tests/rest/client/test_typing.py
+   |tests/rest/client/utils.py
+   |tests/rest/key/v2/test_remote_key_resource.py
+   |tests/rest/media/v1/test_base.py
   |tests/rest/media/v1/test_media_storage.py
+   |tests/rest/media/v1/test_url_preview.py
   |tests/scripts/test_new_matrix_user.py
   |tests/server.py
   |tests/server_notices/test_resource_limits_server_notices.py
@@ -251,7 +250,10 @@ disallow_untyped_defs = True
 [mypy-tests.storage.test_user_directory]
 disallow_untyped_defs = True

-[mypy-tests.rest.*]
+[mypy-tests.rest.admin.*]
+disallow_untyped_defs = True
+
+[mypy-tests.rest.client.test_directory]
 disallow_untyped_defs = True

 [mypy-tests.federation.transport.test_client]
@@ -352,6 +354,3 @@ ignore_missing_imports = True

 [mypy-zope]
 ignore_missing_imports = True
-
-[mypy-incremental.*]
-ignore_missing_imports = True
@@ -35,7 +35,7 @@ CONTRIBUTING_GUIDE_TEXT="!! Please see the contributing guide for help writing y
 https://github.com/matrix-org/synapse/blob/develop/CONTRIBUTING.md#changelog"

 # If check-newsfragment returns a non-zero exit code, print the contributing guide and exit
-python -m towncrier.check --compare-with=origin/develop || (echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2 && exit 1)
+tox -qe check-newsfragment || (echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2 && exit 1)

 echo
 echo "--------------------------"
@@ -71,4 +71,4 @@ fi

 # Run the tests!
 echo "Images built; running complement"
-go test -v -tags synapse_blacklist,msc2403,msc2716,msc3030 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests/...
+go test -v -tags synapse_blacklist,msc2403 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests/...
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+#
+# Update/check the docs/sample_config.yaml
+
+set -e
+
+cd "$(dirname "$0")/.."
+
+SAMPLE_CONFIG="docs/sample_config.yaml"
+SAMPLE_LOG_CONFIG="docs/sample_log_config.yaml"
+
+check() {
+    diff -u "$SAMPLE_LOG_CONFIG" <(./scripts/generate_log_config) >/dev/null || return 1
+}
+
+if [ "$1" == "--check" ]; then
+    diff -u "$SAMPLE_CONFIG" <(./scripts/generate_config --header-file docs/.sample_config_header.yaml) >/dev/null || {
+        echo -e "\e[1m\e[31m$SAMPLE_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config\`.\e[0m" >&2
+        exit 1
+    }
+    diff -u "$SAMPLE_LOG_CONFIG" <(./scripts/generate_log_config) >/dev/null || {
+        echo -e "\e[1m\e[31m$SAMPLE_LOG_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config\`.\e[0m" >&2
+        exit 1
+    }
+else
+    ./scripts/generate_config --header-file docs/.sample_config_header.yaml -o "$SAMPLE_CONFIG"
+    ./scripts/generate_log_config -o "$SAMPLE_LOG_CONFIG"
+fi
@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-#
-# Update/check the docs/sample_config.yaml
-
-set -e
-
-cd "$(dirname "$0")/.."
-
-SAMPLE_CONFIG="docs/sample_config.yaml"
-SAMPLE_LOG_CONFIG="docs/sample_log_config.yaml"
-
-check() {
-    diff -u "$SAMPLE_LOG_CONFIG" <(synapse/_scripts/generate_log_config.py) >/dev/null || return 1
-}
-
-if [ "$1" == "--check" ]; then
-    diff -u "$SAMPLE_CONFIG" <(synapse/_scripts/generate_config.py --header-file docs/.sample_config_header.yaml) >/dev/null || {
-        echo -e "\e[1m\e[31m$SAMPLE_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config.sh\`.\e[0m" >&2
-        exit 1
-    }
-    diff -u "$SAMPLE_LOG_CONFIG" <(synapse/_scripts/generate_log_config.py) >/dev/null || {
-        echo -e "\e[1m\e[31m$SAMPLE_LOG_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config.sh\`.\e[0m" >&2
-        exit 1
-    }
-else
-    synapse/_scripts/generate_config.py --header-file docs/.sample_config_header.yaml -o "$SAMPLE_CONFIG"
-    synapse/_scripts/generate_log_config.py -o "$SAMPLE_LOG_CONFIG"
-fi
@@ -84,8 +84,17 @@ else
      files=(
          "synapse" "docker" "tests"
          # annoyingly, black doesn't find these so we have to list them
+          "scripts/export_signing_key"
+          "scripts/generate_config"
+          "scripts/generate_log_config"
+          "scripts/hash_password"
+          "scripts/register_new_matrix_user"
+          "scripts/synapse_port_db"
+          "scripts/update_synapse_database"
          "scripts-dev"
-          "contrib" "setup.py" "synmark" "stubs" ".ci"
+          "scripts-dev/build_debian_packages"
+          "scripts-dev/sign_json"
+          "contrib" "synctl" "setup.py" "synmark" "stubs" ".ci"
      )
  fi
 fi
@@ -147,7 +147,7 @@ python -m synapse.app.homeserver --generate-keys -c "$SQLITE_CONFIG"

 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
 echo "Running db background jobs..."
-synapse/_scripts/update_synapse_database.py --database-config --run-background-updates "$SQLITE_CONFIG"
+scripts/update_synapse_database --database-config --run-background-updates "$SQLITE_CONFIG"

 # Create the PostgreSQL database.
 echo "Creating postgres database..."
@@ -156,10 +156,10 @@ createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_DB_NAME"
 echo "Copying data from SQLite3 to Postgres with synapse_port_db..."
 if [ -z "$COVERAGE" ]; then
  # No coverage needed
-  synapse/_scripts/synapse_port_db.py --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG"
+  scripts/synapse_port_db --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG"
 else
  # Coverage desired
-  coverage run synapse/_scripts/synapse_port_db.py --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG"
+  coverage run scripts/synapse_port_db --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG"
 fi

 # Delete schema_version, applied_schema_deltas and applied_module_schemas tables
@@ -17,8 +17,6 @@
 """An interactive script for doing a release. See `cli()` below.
 """

-import glob
-import os
 import re
 import subprocess
 import sys
@@ -211,8 +209,8 @@ def prepare():
    with open("synapse/__init__.py", "w") as f:
        f.write(parsed_synapse_ast.dumps())

-    # Generate changelogs.
-    generate_and_write_changelog(current_version)
+    # Generate changelogs
+    run_until_successful("python3 -m towncrier", shell=True)

    # Generate debian changelogs
    if parsed_new_version.pre is not None:
@@ -525,29 +523,5 @@ def get_changes_for_version(wanted_version: version.Version) -> str:
    return "\n".join(version_changelog)


-def generate_and_write_changelog(current_version: version.Version):
-    # We do this by getting a draft so that we can edit it before writing to the
-    # changelog.
-    result = run_until_successful(
-        "python3 -m towncrier --draft", shell=True, capture_output=True
-    )
-    new_changes = result.stdout.decode("utf-8")
-    new_changes = new_changes.replace(
-        "No significant changes.", f"No significant changes since {current_version}."
-    )
-
-    # Prepend changes to changelog
-    with open("CHANGES.md", "r+") as f:
-        existing_content = f.read()
-        f.seek(0, 0)
-        f.write(new_changes)
-        f.write("\n")
-        f.write(existing_content)
-
-    # Remove all the news fragments
-    for f in glob.iglob("changelog.d/*.*"):
-        os.remove(f)
-
-
 if __name__ == "__main__":
    cli()
@@ -0,0 +1,61 @@
+# workers_setup
+
+This gives you a **development-grade** installation of workerised Synapse.
+
+DO NOT USE ME IN PRODUCTION.
+
+## Known defects
+
+* Non-generic workers aren't set up properly with their worker type.
+* I haven't checked the routes that well; they are probably wrong.
+
+
+## Requirements from you:
+
+* Redis on default port (unauthenticated)
+  ```
+  # You need Redis. On Ubuntu, this gets you what you need running on the right port:
+  apt install redis-server redis-tools
+  ```
+* Postgres on default port, using UNIX sockets for authentication.
+  This means you want your normal user account to have a corresponding Postgres account,
+  and let Postgres authenticate you automatically.
+  On Ubuntu, this just means you need to `createuser <your Linux account name>`.
+  You need a database with the same name as your server_name (I used `syn7`).
+  It should be owned by your user; see `createdb` to do that properly (and don't
+  forget to follow the Synapse instructions to use a C locale!)
+  Typing `psql syn7` should just work once your database is ready.
+  (If your UNIX socket is not numbered 5432, you might have to add `port: 5433`
+  to the config. Somehow I messed up my Postgres installation ages ago that it
+  chose port 5433 rather than the default 5432...)
+* Virtualenv with Synapse (don't forget: `[postgres,redis]`)
+* You'll need a bog standard Caddy binary (as the reverse proxy / router).
+  The website offers pre-built static binaries.
+* (Optional): If you want to federate, you can set up TLS yourself afterwards.
+  I haven't bothered so far.
+
+
+## Run the script
+
+```
+# python scripts-dev/workers_setup.py (path to server dir) (server name)
+python scripts-dev/workers_setup.py ../servers/syn7_auto syn7
+```
+
+
+## Launching the homeserver
+
+```
+cd syn7_auto
+/path/to/synapse/.venv/bin/synctl start homeserver.yaml -a workers
+/path/to/caddy run
+```
+
+
+## Stopping the homeserver
+
+```
+# ^C to stop Caddy
+/path/to/synapse/.venv/bin/synctl stop homeserver.yaml -a workers
+```
+
@@ -0,0 +1,305 @@
+#!/usr/bin/env python
+# Copyright 2022 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import dataclasses
+import sys
+from os.path import dirname
+from pathlib import Path
+from typing import Collection, Dict, Iterable, List, Sequence, Tuple
+
+from jinja2 import Environment, FileSystemLoader
+from signedjson.key import generate_signing_key, write_signing_keys
+
+from synapse.util.stringutils import random_string
+
+DESIRED_WORKERS = (
+    ("main", 1),
+    ("synchrotron", 2),
+    ("federation_inbound", 2),
+    ("federation_reader", 2),
+    ("federation_sender", 2),
+    ("typing", 1),
+    ("appservice", 1),
+    ("client_reader", 2),
+    ("event_creator", 2),
+    ("event_persister", 2),
+    ("media_repository", 1),
+    ("pusher", 2),
+    ("user_dir", 1),
+    ("background_worker", 1),
+    # TODO ("encryption", 1),  # ??
+    ("receipts_account_data", 1)
+    # TODO frontend_proxy?
+)
+
+# TODO These are probably all wrong
+
+# ^/_matrix/client/(api/v1|r0|v3|unstable)/sendToDevice/ ?
+# ^/_matrix/client/(api/v1|r0|v3|unstable)/.*/tags
+# ^/_matrix/client/(api/v1|r0|v3|unstable)/.*/account_data ?
+# ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/receipt
+# ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/read_markers ?
+# ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/ ?
+
+WORKER_ROUTES: Dict[str, Tuple[str, ...]] = {
+    "main": (),
+    "synchrotron": (
+        "^/_matrix/client/(v2_alpha|r0|v3)/sync$",
+        "^/_matrix/client/(api/v1|v2_alpha|r0|v3)/events$",
+        "^/_matrix/client/(api/v1|r0|v3)/initialSync$",
+        "^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$",
+    ),
+    "federation_inbound": ("^/_matrix/federation/v1/send/",),
+    "federation_reader": (
+        "^/_matrix/federation/v1/event/",
+        "^/_matrix/federation/v1/state/",
+        "^/_matrix/federation/v1/state_ids/",
+        "^/_matrix/federation/v1/backfill/",
+        "^/_matrix/federation/v1/get_missing_events/",
+        "^/_matrix/federation/v1/publicRooms",
+        "^/_matrix/federation/v1/query/",
+        "^/_matrix/federation/v1/make_join/",
+        "^/_matrix/federation/v1/make_leave/",
+        "^/_matrix/federation/v1/send_join/",
+        "^/_matrix/federation/v2/send_join/",
+        "^/_matrix/federation/v1/send_leave/",
+        "^/_matrix/federation/v2/send_leave/",
+        "^/_matrix/federation/v1/invite/",
+        "^/_matrix/federation/v2/invite/",
+        "^/_matrix/federation/v1/query_auth/",
+        "^/_matrix/federation/v1/event_auth/",
+        "^/_matrix/federation/v1/exchange_third_party_invite/",
+        "^/_matrix/federation/v1/user/devices/",
+        "^/_matrix/federation/v1/get_groups_publicised$",
+        "^/_matrix/key/v2/query",
+        "^/_matrix/federation/(v1|unstable/org.matrix.msc2946)/hierarchy/",
+    ),
+    "federation_sender": (),
+    "typing": ("^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/typing",),
+    "appservice": (),
+    "client_reader": (
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/createRoom$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/publicRooms$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/joined_members$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/context/.*$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/members$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/state$",
+        "^/_matrix/client/(v1|unstable/org.matrix.msc2946)/rooms/.*/hierarchy$",
+        "^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$",
+        "^/_matrix/client/(r0|v3|unstable)/account/3pid$",
+        "^/_matrix/client/(r0|v3|unstable)/devices$",
+        "^/_matrix/client/versions$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$",
+        "^/_matrix/client/(r0|v3|unstable)/joined_groups$",
+        "^/_matrix/client/(r0|v3|unstable)/publicised_groups$",
+        "^/_matrix/client/(r0|v3|unstable)/publicised_groups/",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/event/",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/joined_rooms$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/search$",
+        "^/_matrix/client/(r0|v3|unstable)/keys/query$",
+        "^/_matrix/client/(r0|v3|unstable)/keys/changes$",
+        "^/_matrix/client/(r0|v3|unstable)/keys/claim$",
+        "^/_matrix/client/(r0|v3|unstable)/room_keys/",
+        #
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/login$",
+        "^/_matrix/client/(r0|v3|unstable)/register$",
+        "^/_matrix/client/v1/register/m.login.registration_token/validity$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/redact",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/send",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/state/",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/join/",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/profile/",
+        "^/_matrix/client/(r0|v3|unstable)/sendToDevice/",
+        "^/_matrix/client/(r0|v3|unstable)/.*/tags",
+        "^/_matrix/client/(r0|v3|unstable)/.*/account_data",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/presence/",
+        "^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt",
+        "^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers",
+        "^/_matrix/client/(r0|v3|unstable)/.*/tags",
+        "^/_matrix/client/(r0|v3|unstable)/.*/account_data",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/messages$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/login/sso/redirect",
+        "^/_synapse/client/pick_idp$",
+        "^/_synapse/client/pick_username",
+        "^/_synapse/client/new_user_consent$",
+        "^/_synapse/client/sso_register$",
+        "^/_synapse/client/oidc/callback$",
+        "^/_synapse/client/saml2/authn_response$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/login/cas/ticket$",
+    ),
+    "event_creator": (),
+    "event_persister": (),
+    "media_repository": (
+        "^/_synapse/admin/v1/purge_media_cache$",
+        "^/_synapse/admin/v1/room/.*/media.*$",
+        "^/_synapse/admin/v1/user/.*/media.*$",
+        "^/_synapse/admin/v1/media/.*$",
+        "^/_synapse/admin/v1/quarantine_media/.*$",
+        "^/_synapse/admin/v1/users/.*/media$",
+    ),
+    "pusher": (),
+    "user_dir": ("^/_matrix/client/(api/v1|r0|v3|unstable)/user_directory/search$",),
+    "background_worker": (),
+    "receipts_account_data": (),
+}
+
+
+@dataclasses.dataclass
+class Worker:
+    name: str
+    kind: str
+    index: int
+    ip: str
+
+
+def worker_num_to_ip(num: int) -> str:
+    return f"127.0.57.{num}"
+
+
+def make_workers(workers: Iterable[Tuple[str, int]]) -> List[Worker]:
+    result = []
+    worker_overall_num = 0
+    for worker_type, worker_type_count in workers:
+        for worker_idx in range(1, worker_type_count + 1):
+            worker_overall_num += 1
+            if worker_type == "main":
+                worker_name = "main"
+            else:
+                worker_name = f"{worker_type}{worker_idx}"
+
+            result.append(
+                Worker(
+                    worker_name,
+                    worker_type,
+                    worker_idx,
+                    worker_num_to_ip(worker_overall_num),
+                )
+            )
+
+    return result
+
+
+def generate(
+    worker_counts: Tuple[Tuple[str, int], ...], target_path: Path, server_name: str
+) -> None:
+    if target_path.exists():
+        print("Target path already exists. Won't overwrite.")
+        return
+    target_path.mkdir()
+
+    # Generate a signing key
+    key_id = "a_" + random_string(4)
+    key = (generate_signing_key(key_id),)
+    with open(target_path.joinpath("signing.key"), "w") as fout:
+        write_signing_keys(fout, key)
+
+    macaroon_secret_key = random_string(32)
+
+    env = Environment(loader=FileSystemLoader(dirname(__file__) + "/workers_setup"))
+    hs_template = env.get_template("homeserver.yaml.j2")
+    worker_template = env.get_template("worker.yaml.j2")
+    logging_template = env.get_template("logging.yaml.j2")
+    rp_template = env.get_template("Caddyfile.j2")
+
+    worker_dir = target_path.joinpath("workers")
+    worker_dir.mkdir()
+    worker_logging_dir = target_path.joinpath("workers.logging")
+    worker_logging_dir.mkdir()
+    worker_dir = worker_dir.resolve()
+
+    logs_dir = target_path.joinpath("logs")
+    logs_dir.mkdir()
+    logs_dir = logs_dir.resolve()
+
+    all_workers = make_workers(worker_counts)
+    workers_by_name = {worker.name: worker for worker in all_workers}
+
+    for worker in all_workers:
+        log_config_path = worker_logging_dir.joinpath(f"{worker.name}.logging.yaml")
+        log_config = logging_template.render(
+            worker=worker,
+            worker_dir=worker_dir,
+            logs_dir=logs_dir,
+            all_workers=all_workers,
+            workers_by_name=workers_by_name,
+        )
+        with open(log_config_path, "w") as fout:
+            fout.write(log_config)
+
+        # if worker.name == "main":
+        # Main can't use a worker file.
+        # continue
+
+        worker_config_path = worker_dir.joinpath(f"{worker.name}.yaml")
+        worker_config = worker_template.render(
+            worker=worker,
+            worker_dir=worker_dir,
+            logs_dir=logs_dir,
+            all_workers=all_workers,
+            workers_by_name=workers_by_name,
+        )
+        with open(worker_config_path, "w") as fout:
+            fout.write(worker_config)
+
+    hs_config_path = target_path.joinpath("homeserver.yaml")
+    hs_config = hs_template.render(
+        all_workers=all_workers,
+        workers_by_name=workers_by_name,
+        worker_dir=worker_dir,
+        logs_dir=logs_dir,
+        server_name=server_name,
+        macaroon_secret_key=macaroon_secret_key,
+    )
+    with open(hs_config_path, "w") as fout:
+        fout.write(hs_config)
+
+    caddy_config_path = target_path.joinpath("Caddyfile")
+    caddy_config = rp_template.render(
+        server_name=server_name,
+        port=8447,
+        http_ip=worker_num_to_ip(1),
+        routing=build_routes_template_var(all_workers),
+        main_server=f"{worker_num_to_ip(1)}:8080",
+    )
+    with open(caddy_config_path, "w") as fout:
+        fout.write(caddy_config)
+
+
+def build_routes_template_var(
+    all_workers: List[Worker],
+) -> Sequence[Tuple[str, Collection[str], List[str]]]:
+    route_groups = {}
+    for worker in all_workers:
+        if worker.kind not in route_groups:
+            if WORKER_ROUTES[worker.kind]:
+                route_groups[worker.kind] = worker.kind, WORKER_ROUTES[worker.kind], []
+            else:
+                continue
+
+        _, _routes, server_endpoints = route_groups[worker.kind]
+
+        server_endpoints.append(f"{worker.ip}:8080")
+
+    return tuple(route_groups.values())
+
+
+def main(target_path: Path, server_name: str) -> None:
+    generate(DESIRED_WORKERS, target_path, server_name)
+
+
+if __name__ == "__main__":
+    target_path = Path(sys.argv[1])
+    server_name = sys.argv[2]
+    main(target_path, server_name)
@@ -0,0 +1,24 @@
+{
+  # Prevents Caddy from asking for sudo password to install a root cert that
+  # we don't even want to use here.
+  skip_install_trust
+}
+
+# If you want TLS, you can add https:// schemes and configure the TLS cert... somehow.
+http://{{ server_name }}:{{ port }}, http://{{ http_ip }}:{{ port }} {
+  {%- for route_group_name, routes, route_servers in routing %}
+    @{{ route_group_name }} {
+      {%- for route in routes %}
+      path_regexp {{ route }}
+      {%- endfor %}
+    }
+    route @{{ route_group_name }} {
+      reverse_proxy {% for server in route_servers %} {{ server }} {% endfor %}
+    }
+  {%- endfor %}
+
+  # fallback to main
+  route {
+    reverse_proxy {{ main_server }}
+  }
+}
@@ -0,0 +1,96 @@
+
+server_name: {{ server_name }}
+
+report_stats: false
+
+signing_key_path: "{{ worker_dir }}/../signing.key"
+
+macaroon_secret_key: "{{ macaroon_secret_key }}"
+
+enable_registration: true
+
+redis:
+  enabled: true
+  #host: localhost
+  #port: 6379
+
+trusted_key_servers: []
+
+listeners:
+  - port: 8080
+    bind_address: {{ workers_by_name.main.ip }}
+    type: http
+    resources:
+      - names: [client, federation]
+
+  # The HTTP replication port
+  - port: 9090
+    bind_address: {{ workers_by_name.main.ip }}
+    type: http
+    resources:
+      - names: [replication]
+
+database:
+  name: psycopg2
+  args:
+    # Comment out user, password and host to use UNIX socket auth.
+    # For testing, create a database owned by your Postgres user that is logged
+    # in with your UNIX user
+    #user: "synapse"
+    #password:
+    database: "{{ server_name }}"
+    #host: "localhost"
+    cp_min: 5
+    cp_max: 10
+
+instance_map:
+{%- for worker in all_workers %}
+  {{ worker.name }}:
+    host: {{ worker.ip }}
+    port: 9090
+{%- endfor %}
+
+stream_writers:
+  events:
+{%- for worker in all_workers %}
+{%- if worker.kind == "event_persister" %}
+    - {{ worker.name }}
+{%- endif %}
+{%- endfor %}
+  typing:
+{%- for worker in all_workers %}
+{%- if worker.kind == "typing" %}
+    - {{ worker.name }}
+{%- endif %}
+{%- endfor %}
+
+
+start_pushers: false
+
+pusher_instances:
+{% for worker in all_workers -%}
+{%- if worker.kind == "pusher" %}
+    - {{ worker.name }}
+{%- endif %}
+{%- endfor %}
+
+notify_appservices: False
+
+federation_sender_instances:
+{% for worker in all_workers -%}
+{%- if worker.kind == "federation_sender" %}
+    - {{ worker.name }}
+{% endif -%}
+{% endfor %}
+
+enable_media_repo: False
+
+media_instance_running_background_jobs: "media1"
+
+update_user_directory: False
+
+pid_file: "{{ logs_dir }}/main.pid"
+
+log_config: '{{ worker_dir }}.logging/main.logging.yaml'
+
+run_background_tasks_on: background_worker1
@@ -0,0 +1,32 @@
+
+version: 1
+
+formatters:
+    precise:
+        format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+
+filters:
+    context:
+        (): synapse.util.logcontext.LoggingContextFilter
+        request: ""
+
+handlers:
+    console:
+        class: logging.FileHandler
+        formatter: precise
+        filters: [context]
+        encoding: 'UTF-8'
+        filename: '{{ logs_dir }}/{{ worker.name }}.log'
+
+loggers:
+    synapse:
+        level: DEBUG
+
+    synapse.storage.SQL:
+        # beware: increasing this to DEBUG will make synapse log sensitive
+        # information such as access tokens.
+        level: INFO
+
+root:
+    level: WARNING
+    handlers: [console]
@@ -0,0 +1,36 @@
+{% set main_worker = workers_by_name.main %}
+
+{% if worker.kind == "main" %}
+
+worker_app: synapse.app.homeserver
+
+{% else %}
+
+worker_app: synapse.app.generic_worker
+worker_name: {{ worker.name }}
+# The replication listener on the main synapse process.
+worker_replication_host: {{ main_worker.ip }}
+worker_replication_http_port: 9090
+worker_listeners:
+  - type: http
+    bind_address: {{ worker.ip }}
+    port: 8080
+    resources:
+    - names:
+      - client
+      - federation
+{%- if worker.kind == "media" %}
+      - media
+{%- endif %}
+  - type: http
+    bind_address: {{ worker.ip }}
+    port: 9090
+    resources:
+    - names: [replication]
+
+worker_log_config: '{{ worker_dir }}.logging/{{ worker.name }}.logging.yaml'
+worker_pid_file: '{{ logs_dir }}/{{ worker.name }}.pid'
+
+worker_main_http_uri: http://{{ main_worker.ip }}:8080
+
+{% endif %}
@@ -50,7 +50,7 @@ def format_for_config(public_key: nacl.signing.VerifyKey, expiry_ts: int):
    )


-def main():
+if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    parser.add_argument(
@@ -85,6 +85,7 @@ def main():
        else format_plain
    )

+    keys = []
    for file in args.key_file:
        try:
            res = read_signing_keys(file)
@@ -97,7 +98,3 @@ def main():
            res = []
        for key in res:
            formatter(get_verify_key(key))
-
-
-if __name__ == "__main__":
-    main()
@@ -6,8 +6,7 @@ import sys

 from synapse.config.homeserver import HomeServerConfig

-
-def main():
+if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--config-dir",
@@ -77,7 +76,3 @@ def main():
        shutil.copyfileobj(args.header_file, args.output_file)

    args.output_file.write(conf)
-
-
-if __name__ == "__main__":
-    main()
@@ -19,8 +19,7 @@ import sys

 from synapse.config.logger import DEFAULT_LOG_CONFIG

-
-def main():
+if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    parser.add_argument(
@@ -43,7 +42,3 @@ def main():
    out = args.output_file
    out.write(DEFAULT_LOG_CONFIG.substitute(log_file=args.log_file))
    out.flush()
-
-
-if __name__ == "__main__":
-    main()
@@ -19,8 +19,7 @@ from signedjson.key import generate_signing_key, write_signing_keys

 from synapse.util.stringutils import random_string

-
-def main():
+if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    parser.add_argument(
@@ -35,7 +34,3 @@ def main():
    key_id = "a_" + random_string(4)
    key = (generate_signing_key(key_id),)
    write_signing_keys(args.output_file, key)
-
-
-if __name__ == "__main__":
-    main()
@@ -8,6 +8,9 @@ import unicodedata
 import bcrypt
 import yaml

+bcrypt_rounds = 12
+password_pepper = ""
+

 def prompt_for_pass():
    password = getpass.getpass("Password: ")
@@ -23,10 +26,7 @@ def prompt_for_pass():
    return password


-def main():
-    bcrypt_rounds = 12
-    password_pepper = ""
-
+if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description=(
            "Calculate the hash of a new password, so that passwords can be reset"
@@ -77,7 +77,3 @@ def main():
    ).decode("ascii")

    print(hashed)
-
-
-if __name__ == "__main__":
-    main()
@@ -28,7 +28,7 @@ This can be extracted from postgres with::

 To use, pipe the above into::

-    PYTHON_PATH=. synapse/_scripts/move_remote_media_to_new_store.py <source repo> <dest repo>
+    PYTHON_PATH=. ./scripts/move_remote_media_to_new_store.py <source repo> <dest repo>
 """

 import argparse
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+# Copyright 2015, 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse._scripts.register_new_matrix_user import main
+
+if __name__ == "__main__":
+    main()
@@ -1146,7 +1146,7 @@ class TerminalProgress(Progress):
 ##############################################


-def main():
+if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="A script to port an existing synapse SQLite database to"
        " a new PostgreSQL database."
@@ -1251,7 +1251,3 @@ def main():
        sys.stderr.write(end_error)

        sys.exit(5)
-
-
-if __name__ == "__main__":
-    main()
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse._scripts.review_recent_signups import main
+
+if __name__ == "__main__":
+    main()
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Olivier Wilkinson (reivilibre)	3933bb17de	Fix running background tasks on BG worker	2022-03-18 13:53:55 +00:00
Olivier Wilkinson (reivilibre)	70ee28f386	WORKING workers setup template	2022-03-17 10:42:28 +00:00
Olivier Wilkinson (reivilibre)	f5bb3887c8	STASH	2022-03-16 14:02:58 +00:00
Olivier Wilkinson (reivilibre)	85774e17be	STASH	2022-03-11 11:36:47 +00:00
Olivier Wilkinson (reivilibre)	a0e699b301	STASH	2022-02-24 17:10:23 +00:00
				`@@ -0,0 +1 @@`
				Deduplicate in-flight requests in `_get_state_for_groups`.
				`@@ -0,0 +1 @@`
				Make a `POST` to `/rooms/<room_id>/receipt/m.read/<event_id>` only trigger a push notification if the count of unread messages is different to the one in the last successfully sent push.
				`@@ -0,0 +1 @@`
				`Remove unnecessary condition on knock->leave auth rule check.`
				`@@ -0,0 +1 @@`
				`Add tests for device list changes between local users.`
				`@@ -0,0 +1 @@`
				Optimise calculating device_list changes in `/sync`.
				`@@ -0,0 +1 @@`
				`Fetch images when previewing Twitter URLs. Contributed by @AndrewRyanChama.`
				`@@ -0,0 +1 @@`
				`Refactor the search code for improved readability.`
				`@@ -0,0 +1 @@`
				`Fix a bug introduced in Synapse v1.48.0 where an edit of the latest event in a thread would not be properly applied to the thread summary.`
				`@@ -0,0 +1 @@`
				Move common deduplication code down into `_auth_and_persist_outliers`.