merge in develop

.ci/scripts/calculate_jobs.py

@@ -29,12 +29,11 @@ IS_PR = os.environ["GITHUB_REF"].startswith("refs/pull/")
 
 # First calculate the various trial jobs.
 #
-# For PRs, we only run each type of test with the oldest Python version supported (which
-# is Python 3.8 right now)
+# For each type of test we only run on Py3.7 on PRs
 
 trial_sqlite_tests = [
     {
-        "python-version": "3.8",
+        "python-version": "3.7",
         "database": "sqlite",
         "extras": "all",
     }
@@ -47,13 +46,13 @@ if not IS_PR:
             "database": "sqlite",
             "extras": "all",
         }
-        for version in ("3.9", "3.10", "3.11")
+        for version in ("3.8", "3.9", "3.10", "3.11")
     )
 
 
 trial_postgres_tests = [
     {
-        "python-version": "3.8",
+        "python-version": "3.7",
         "database": "postgres",
         "postgres-version": "11",
         "extras": "all",
@@ -72,7 +71,7 @@ if not IS_PR:
 
 trial_no_extra_tests = [
     {
-        "python-version": "3.8",
+        "python-version": "3.7",
         "database": "sqlite",
         "extras": "",
     }
@@ -134,6 +133,11 @@ if not IS_PR:
                 "sytest-tag": "testing",
                 "postgres": "postgres",
             },
+            {
+                "sytest-tag": "buster",
+                "postgres": "multi-postgres",
+                "workers": "workers",
+            },
         ]
     )
 

.ci/scripts/prepare_old_deps.sh

@@ -31,6 +31,34 @@ sed -i \
    -e '/systemd/d' \
    pyproject.toml
 
+# Use poetry to do the installation. This ensures that the versions are all mutually
+# compatible (as far the package metadata declares, anyway); pip's package resolver
+# is more lax.
+#
+# Rather than `poetry install --no-dev`, we drop all dev dependencies from the
+# toml file. This means we don't have to ensure compatibility between old deps and
+# dev tools.
+
+pip install toml wheel
+
+REMOVE_DEV_DEPENDENCIES="
+import toml
+with open('pyproject.toml', 'r') as f:
+    data = toml.loads(f.read())
+
+del data['tool']['poetry']['dev-dependencies']
+
+with open('pyproject.toml', 'w') as f:
+    toml.dump(data, f)
+"
+python3 -c "$REMOVE_DEV_DEPENDENCIES"
+
+pip install poetry==1.3.2
+poetry lock
+
 echo "::group::Patched pyproject.toml"
 cat pyproject.toml
 echo "::endgroup::"
+echo "::group::Lockfile after patch"
+cat poetry.lock
+echo "::endgroup::"

.ci/scripts/setup_complement_prerequisites.sh

@@ -9,6 +9,16 @@ set -eu
 alias block='{ set +x; } 2>/dev/null; func() { echo "::group::$*"; set -x; }; func'
 alias endblock='{ set +x; } 2>/dev/null; func() { echo "::endgroup::"; set -x; }; func'
 
+block Set Go Version
+  # The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
+  # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
+
+  # Add Go 1.17 to the PATH: see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-2
+  echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
+  # Add the Go path to the PATH: We need this so we can call gotestfmt
+  echo "~/go/bin" >> $GITHUB_PATH
+endblock
+
 block Install Complement Dependencies
   sudo apt-get -qq update && sudo apt-get install -qqy libolm3 libolm-dev
   go install -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest

.github/ISSUE_TEMPLATE/BUG_REPORT.yml

@@ -129,7 +129,7 @@ body:
     attributes:
       label: Relevant log output
       description: |
-        Please copy and paste any relevant log output as text (not images), ideally at INFO or DEBUG log level.
+        Please copy and paste any relevant log output, ideally at INFO or DEBUG log level.
         This will be automatically formatted into code, so there is no need for backticks (`\``).
 
         Please be careful to remove any personal or private data.

.github/workflows/dependabot_changelog.yml

@@ -0,0 +1,49 @@
+name: Write changelog for dependabot PR
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened  # For debugging!
+
+permissions:
+  # Needed to be able to push the commit. See
+  #     https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#enable-auto-merge-on-a-pull-request
+  # for a similar example
+  contents: write
+
+jobs:
+  add-changelog:
+    runs-on: 'ubuntu-latest'
+    if: ${{ github.actor == 'dependabot[bot]' }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+      - name: Write, commit and push changelog
+        env:
+          PR_TITLE: ${{ github.event.pull_request.title }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          echo "${PR_TITLE}." > "changelog.d/${PR_NUMBER}".misc
+          git add changelog.d
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git config user.name "GitHub Actions"
+          git commit -m "Changelog"
+          git push
+        shell: bash
+      # The `git push` above does not trigger CI on the dependabot PR.
+      #
+      # By default, workflows can't trigger other workflows when they're just using the
+      # default `GITHUB_TOKEN` access token. (This is intended to stop you from writing
+      # recursive workflow loops by accident, because that'll get very expensive very
+      # quickly.) Instead, you have to manually call out to another workflow, or else
+      # make your changes (i.e. the `git push` above) using a personal access token.
+      # See
+      # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
+      #
+      # I have tried and failed to find a way to trigger CI on the "merge ref" of the PR.
+      # See git commit history for previous attempts. If anyone desperately wants to try
+      # again in the future, make a matrix-bot account and use its access token to git push.
+
+  # THIS WORKFLOW HAS WRITE PERMISSIONS---do not add other jobs here unless they
+  # are sufficiently locked down to dependabot only as above.

.github/workflows/docker.yml

@@ -10,7 +10,6 @@ on:
 
 permissions:
   contents: read
-  packages: write
 
 jobs:
   build:
@@ -35,20 +34,11 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Log in to GHCR
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Calculate docker image tag
         id: set-tag
         uses: docker/metadata-action@master
         with:
-          images: |
-            docker.io/matrixdotorg/synapse
-            ghcr.io/matrix-org/synapse
+          images: matrixdotorg/synapse
           flavor: |
             latest=false
           tags: |

.github/workflows/docs-pr-netlify.yaml

@@ -14,7 +14,7 @@ jobs:
       # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
       # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
       - name: 📥 Download artifact
-        uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615 # v2.27.0
+        uses: dawidd6/action-download-artifact@5e780fc7bbd0cac69fc73271ed86edf5dcb72d67 # v2.26.0
         with:
           workflow: docs-pr.yaml
           run_id: ${{ github.event.workflow_run.id }}
@@ -22,7 +22,7 @@ jobs:
           path: book
 
       - name: 📤 Deploy to Netlify
-        uses: matrix-org/netlify-pr-preview@v2
+        uses: matrix-org/netlify-pr-preview@v1
         with:
           path: book
           owner: ${{ github.event.workflow_run.head_repository.owner.login }}

.github/workflows/docs.yaml

@@ -13,10 +13,25 @@ on:
   workflow_dispatch:
 
 jobs:
-  pre:
-    name: Calculate variables for GitHub Pages deployment
+  pages:
+    name: GitHub Pages
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup mdbook
+        uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
+        with:
+          mdbook-version: '0.4.17'
+
+      - name: Build the documentation
+        # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
+        # However, we're using docs/README.md for other purposes and need to pick a new page
+        # as the default. Let's opt for the welcome page instead.
+        run: |
+          mdbook build
+          cp book/welcome_and_overview.html book/index.html
+
       # Figure out the target directory.
       #
       # The target directory depends on the name of the branch
@@ -40,65 +55,11 @@ jobs:
 
           # finally, set the 'branch-version' var.
           echo "branch-version=$branch" >> "$GITHUB_OUTPUT"
-    outputs:
-      branch-version: ${{ steps.vars.outputs.branch-version }}
-
-################################################################################
-  pages-docs:
-    name: GitHub Pages
-    runs-on: ubuntu-latest
-    needs:
-      - pre
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Setup mdbook
-        uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
-        with:
-          mdbook-version: '0.4.17'
-
-      - name: Build the documentation
-        # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
-        # However, we're using docs/README.md for other purposes and need to pick a new page
-        # as the default. Let's opt for the welcome page instead.
-        run: |
-          mdbook build
-          cp book/welcome_and_overview.html book/index.html
-
+          
       # Deploy to the target directory.
       - name: Deploy to gh pages
-        uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3
+        uses: peaceiris/actions-gh-pages@bd8c6b06eba6b3d25d72b7a1767993c0aeee42e7 # v3.9.2
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./book
-          destination_dir: ./${{ needs.pre.outputs.branch-version }}
-
-################################################################################
-  pages-devdocs:
-    name: GitHub Pages (developer docs)
-    runs-on: ubuntu-latest
-    needs:
-      - pre
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: "Set up Sphinx"
-        uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: "3.x"
-          poetry-version: "1.3.2"
-          groups: "dev-docs"
-          extras: ""
-
-      - name: Build the documentation
-        run: |
-          cd dev-docs
-          poetry run make html
-
-      # Deploy to the target directory.
-      - name: Deploy to gh pages
-        uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./dev-docs/_build/html
-          destination_dir: ./dev-docs/${{ needs.pre.outputs.branch-version }}
+          destination_dir: ./${{ steps.vars.outputs.branch-version }}

.github/workflows/latest_deps.yml

@@ -22,26 +22,14 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  check_repo:
-    # Prevent this workflow from running on any fork of Synapse other than matrix-org/synapse, as it is
-    # only useful to the Synapse core team.
-    # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest
-    # of the workflow will be skipped as well.
-    runs-on: ubuntu-latest
-    outputs:
-      should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }}
-    steps:
-      - id: check_condition
-        run: echo "should_run_workflow=${{ github.repository == 'matrix-org/synapse' }}" >> "$GITHUB_OUTPUT"
-
   mypy:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
+        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        with:
+          toolchain: stable
       - uses: Swatinem/rust-cache@v2
 
       # The dev dependencies aren't exposed in the wheel metadata (at least with current
@@ -61,8 +49,6 @@ jobs:
         run: sed '/warn_unused_ignores = True/d' -i mypy.ini
       - run: poetry run mypy
   trial:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -75,7 +61,9 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
+        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        with:
+          toolchain: stable
       - uses: Swatinem/rust-cache@v2
 
       - run: sudo apt-get -qq install xmlsec1
@@ -121,8 +109,6 @@ jobs:
 
 
   sytest:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     container:
       image: matrixdotorg/sytest-synapse:testing
@@ -148,7 +134,9 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
+        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        with:
+          toolchain: stable
       - uses: Swatinem/rust-cache@v2
 
       - name: Ensure sytest runs `pip install`
@@ -174,8 +162,7 @@ jobs:
 
 
   complement:
-    needs: check_repo
-    if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "${{ !failure() && !cancelled() }}"
     runs-on: ubuntu-latest
 
     strategy:
@@ -197,8 +184,6 @@ jobs:
         with:
           path: synapse
 
-      - uses: actions/setup-go@v4
-
       - name: Prepare Complement's Prerequisites
         run: synapse/.ci/scripts/setup_complement_prerequisites.sh
 
@@ -211,7 +196,7 @@ jobs:
   # Open an issue if the build fails, so we know about it.
   # Only do this if we're not experimenting with this action in a PR.
   open-issue:
-    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request' && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request'"
     needs:
       # TODO: should mypy be included here? It feels more brittle than the others.
       - mypy

.github/workflows/release-artifacts.yml

@@ -34,7 +34,6 @@ jobs:
       - id: set-distros
         run: |
           # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
-          # NOTE: inside the actual Dockerfile-dhvirtualenv, the image name is expanded into its full image path
           dists='["debian:sid"]'
           if [[ $GITHUB_REF == refs/tags/* ]]; then
               dists=$(scripts-dev/build_debian_packages.py --show-dists-json)
@@ -144,7 +143,7 @@ jobs:
 
       - name: Only build a single wheel on PR
         if: startsWith(github.ref, 'refs/pull/')
-        run: echo "CIBW_BUILD="cp38-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
+        run: echo "CIBW_BUILD="cp37-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
 
       - name: Build wheels
         run: python -m cibuildwheel --output-dir wheelhouse

.github/workflows/tests.yml

@@ -34,9 +34,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
-      - uses: Swatinem/rust-cache@v2
       - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: "3.x"
@@ -65,60 +62,9 @@ jobs:
       - run: .ci/scripts/check_lockfile.py
 
   lint:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Setup Poetry
-        uses: matrix-org/setup-python-poetry@v1
-        with:
-          install-project: "false"
-
-      - name: Import order (isort)
-        run: poetry run isort --check --diff .
-
-      - name: Code style (black)
-        run: poetry run black --check --diff .
-
-      - name: Semantic checks (ruff)
-        # --quiet suppresses the update check.
-        run: poetry run ruff --quiet .
-
-  lint-mypy:
-    runs-on: ubuntu-latest
-    name: Typechecking
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
-      - uses: Swatinem/rust-cache@v2
-
-      - name: Setup Poetry
-        uses: matrix-org/setup-python-poetry@v1
-        with:
-          # We want to make use of type hints in optional dependencies too.
-          extras: all
-          # We have seen odd mypy failures that were resolved when we started
-          # installing the project again:
-          # https://github.com/matrix-org/synapse/pull/15376#issuecomment-1498983775
-          # To make CI green, err towards caution and install the project.
-          install-project: "true"
-
-      # Cribbed from
-      # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17
-      - name: Restore/persist mypy's cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            .mypy_cache
-          key: mypy-cache-${{ github.context.sha }}
-          restore-keys: mypy-cache-
-
-      - name: Run mypy
-        run: poetry run mypy
+    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v2"
+    with:
+      typechecking-extras: "all"
 
   lint-crlf:
     runs-on: ubuntu-latest
@@ -149,9 +95,6 @@ jobs:
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
-      - uses: Swatinem/rust-cache@v2
       - uses: matrix-org/setup-python-poetry@v1
         with:
           poetry-version: "1.3.2"
@@ -167,8 +110,12 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        # There don't seem to be versioned releases of this action per se: for each rust
+        # version there is a branch which gets constantly rebased on top of master.
+        # We pin to a specific commit for paranoia's sake.
+        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
         with:
+            toolchain: 1.58.1
             components: clippy
       - uses: Swatinem/rust-cache@v2
 
@@ -185,7 +132,10 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@master
+        # There don't seem to be versioned releases of this action per se: for each rust
+        # version there is a branch which gets constantly rebased on top of master.
+        # We pin to a specific commit for paranoia's sake.
+        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
         with:
             toolchain: nightly-2022-12-01
             components: clippy
@@ -202,7 +152,10 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@master
+        # There don't seem to be versioned releases of this action per se: for each rust
+        # version there is a branch which gets constantly rebased on top of master.
+        # We pin to a specific commit for paranoia's sake.
+        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
         with:
           # We use nightly so that it correctly groups together imports
           toolchain: nightly-2022-12-01
@@ -216,7 +169,6 @@ jobs:
     if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
     needs:
       - lint
-      - lint-mypy
       - lint-crlf
       - lint-newsfile
       - lint-pydantic
@@ -268,7 +220,12 @@ jobs:
             postgres:${{ matrix.job.postgres-version }}
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        # There don't seem to be versioned releases of this action per se: for each rust
+        # version there is a branch which gets constantly rebased on top of master.
+        # We pin to a specific commit for paranoia's sake.
+        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        with:
+            toolchain: 1.58.1
       - uses: Swatinem/rust-cache@v2
 
       - uses: matrix-org/setup-python-poetry@v1
@@ -308,39 +265,52 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        # There don't seem to be versioned releases of this action per se: for each rust
+        # version there is a branch which gets constantly rebased on top of master.
+        # We pin to a specific commit for paranoia's sake.
+        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        with:
+            toolchain: 1.58.1
       - uses: Swatinem/rust-cache@v2
 
       # There aren't wheels for some of the older deps, so we need to install
       # their build dependencies
       - run: |
-          sudo apt-get -qq update
           sudo apt-get -qq install build-essential libffi-dev python-dev \
-            libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
+          libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
 
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.7'
 
+      # Calculating the old-deps actually takes a bunch of time, so we cache the
+      # pyproject.toml / poetry.lock. We need to cache pyproject.toml as
+      # otherwise the `poetry install` step will error due to the poetry.lock
+      # file being outdated.
+      #
+      # This caches the output of `Prepare old deps`, which should generate the
+      # same `pyproject.toml` and `poetry.lock` for a given `pyproject.toml` input.
+      - uses: actions/cache@v3
+        id: cache-poetry-old-deps
+        name: Cache poetry.lock
+        with:
+          path: |
+            poetry.lock
+            pyproject.toml
+          key: poetry-old-deps2-${{ hashFiles('pyproject.toml') }}
       - name: Prepare old deps
         if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true'
         run: .ci/scripts/prepare_old_deps.sh
 
-      # Note: we install using `pip` here, not poetry. `poetry install` ignores the
-      # build-system section (https://github.com/python-poetry/poetry/issues/6154), but
-      # we explicitly want to test that you can `pip install` using the oldest version
-      # of poetry-core and setuptools-rust.
-      - run: pip install .[all,test]
+      # We only now install poetry so that `setup-python-poetry` caches the
+      # right poetry.lock's dependencies.
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          python-version: '3.7'
+          poetry-version: "1.3.2"
+          extras: "all test"
 
-      # We nuke the local copy, as we've installed synapse into the virtualenv
-      # (rather than use an editable install, which we no longer support). If we
-      # don't do this then python can't find the native lib.
-      - run: rm -rf synapse/
-
-      # Sanity check we can import/run Synapse
-      - run: python -m synapse.app.homeserver --help
-
-      - run: python -m twisted.trial -j6 tests
+      - run: poetry run trial -j6 tests
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
         if: ${{ always() }}
@@ -362,7 +332,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["pypy-3.8"]
+        python-version: ["pypy-3.7"]
         extras: ["all"]
 
     steps:
@@ -399,8 +369,8 @@ jobs:
       env:
         SYTEST_BRANCH: ${{ github.head_ref }}
         POSTGRES: ${{ matrix.job.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') || '' }}
-        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') || '' }}
+        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}}
+        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') && 1 }}
         WORKERS: ${{ matrix.job.workers && 1 }}
         BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }}
         TOP: ${{ github.workspace }}
@@ -416,7 +386,12 @@ jobs:
         run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        # There don't seem to be versioned releases of this action per se: for each rust
+        # version there is a branch which gets constantly rebased on top of master.
+        # We pin to a specific commit for paranoia's sake.
+        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        with:
+            toolchain: 1.58.1
       - uses: Swatinem/rust-cache@v2
 
       - name: Run SyTest
@@ -477,7 +452,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - python-version: "3.8"
+          - python-version: "3.7"
             postgres-version: "11"
 
           - python-version: "3.11"
@@ -556,11 +531,14 @@ jobs:
           path: synapse
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        # There don't seem to be versioned releases of this action per se: for each rust
+        # version there is a branch which gets constantly rebased on top of master.
+        # We pin to a specific commit for paranoia's sake.
+        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        with:
+            toolchain: 1.58.1
       - uses: Swatinem/rust-cache@v2
 
-      - uses: actions/setup-go@v4
-
       - name: Prepare Complement's Prerequisites
         run: synapse/.ci/scripts/setup_complement_prerequisites.sh
 
@@ -584,7 +562,12 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        # There don't seem to be versioned releases of this action per se: for each rust
+        # version there is a branch which gets constantly rebased on top of master.
+        # We pin to a specific commit for paranoia's sake.
+        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        with:
+            toolchain: 1.58.1
       - uses: Swatinem/rust-cache@v2
 
       - run: cargo test
@@ -602,7 +585,10 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@master
+        # There don't seem to be versioned releases of this action per se: for each rust
+        # version there is a branch which gets constantly rebased on top of master.
+        # We pin to a specific commit for paranoia's sake.
+        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
         with:
             toolchain: nightly-2022-12-01
       - uses: Swatinem/rust-cache@v2

.github/workflows/twisted_trunk.yml

@@ -5,42 +5,22 @@ on:
     - cron: 0 8 * * *
 
   workflow_dispatch:
-    inputs:
-      twisted_ref:
-        description: Commit, branch or tag to checkout from upstream Twisted.
-        required: false
-        default: 'trunk'
-        type: string
-
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  check_repo:
-    # Prevent this workflow from running on any fork of Synapse other than matrix-org/synapse, as it is
-    # only useful to the Synapse core team.
-    # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest
-    # of the workflow will be skipped as well.
-    if: github.repository == 'matrix-org/synapse'
-    runs-on: ubuntu-latest
-    outputs:
-      should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }}
-    steps:
-      - id: check_condition
-        run: echo "should_run_workflow=${{ github.repository == 'matrix-org/synapse' }}" >> "$GITHUB_OUTPUT"
-
   mypy:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
+        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        with:
+          toolchain: stable
       - uses: Swatinem/rust-cache@v2
 
       - uses: matrix-org/setup-python-poetry@v1
@@ -49,15 +29,13 @@ jobs:
           extras: "all"
       - run: |
           poetry remove twisted
-          poetry add --extras tls git+https://github.com/twisted/twisted.git#${{ inputs.twisted_ref }}
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
           poetry install --no-interaction --extras "all test"
       - name: Remove warn_unused_ignores from mypy config
         run: sed '/warn_unused_ignores = True/d' -i mypy.ini
       - run: poetry run mypy
 
   trial:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
 
     steps:
@@ -65,7 +43,9 @@ jobs:
       - run: sudo apt-get -qq install xmlsec1
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
+        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        with:
+          toolchain: stable
       - uses: Swatinem/rust-cache@v2
 
       - uses: matrix-org/setup-python-poetry@v1
@@ -92,15 +72,9 @@ jobs:
           || true
 
   sytest:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     container:
-      # We're using ubuntu:focal because it uses Python 3.8 which is our minimum supported Python version.
-      # This job is a canary to warn us about unreleased twisted changes that would cause problems for us if
-      # they were to be released immediately. For simplicity's sake (and to save CI runners) we use the oldest
-      # version, assuming that any incompatibilities on newer versions would also be present on the oldest.
-      image: matrixdotorg/sytest-synapse:focal
+      image: matrixdotorg/sytest-synapse:buster
       volumes:
         - ${{ github.workspace }}:/src
 
@@ -108,7 +82,9 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
+        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        with:
+          toolchain: stable
       - uses: Swatinem/rust-cache@v2
 
       - name: Patch dependencies
@@ -142,8 +118,7 @@ jobs:
             /logs/**/*.log*
 
   complement:
-    needs: check_repo
-    if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "${{ !failure() && !cancelled() }}"
     runs-on: ubuntu-latest
 
     strategy:
@@ -165,8 +140,6 @@ jobs:
         with:
           path: synapse
 
-      - uses: actions/setup-go@v4
-
       - name: Prepare Complement's Prerequisites
         run: synapse/.ci/scripts/setup_complement_prerequisites.sh
 
@@ -190,7 +163,7 @@ jobs:
 
   # open an issue if the build fails, so we know about it.
   open-issue:
-    if: failure() && needs.check_repo.outputs.should_run_workflow == 'true'
+    if: failure()
     needs:
       - mypy
       - trial

.gitignore

@@ -15,10 +15,9 @@ _trial_temp*/
 .DS_Store
 __pycache__/
 
-# We do want poetry, cargo and flake lockfiles.
+# We do want the poetry and cargo lockfile.
 !poetry.lock
 !Cargo.lock
-!flake.lock
 
 # stuff that is likely to exist when you run a server locally
 /*.db
@@ -34,15 +33,11 @@ __pycache__/
 /logs
 /media_store/
 /uploads
-/homeserver-config-overrides.d
 
 # For direnv users
 /.envrc
 .direnv/
 
-# For nix/devenv users
-.devenv/
-
 # IDEs
 /.idea/
 /.ropeproject/
@@ -58,7 +53,6 @@ __pycache__/
 /coverage.*
 /dist/
 /docs/build/
-/dev-docs/_build/
 /htmlcov
 /pip-wheel-metadata/
 
@@ -67,7 +61,7 @@ book/
 
 # complement
 /complement-*
-/main.tar.gz
+/master.tar.gz
 
 # rust
 /target/

Cargo.lock

@@ -4,18 +4,18 @@ version = 3
 
 [[package]]
 name = "aho-corasick"
-version = "1.0.2"
+version = "0.7.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
+checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e"
 dependencies = [
  "memchr",
 ]
 
 [[package]]
 name = "anyhow"
-version = "1.0.72"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854"
+checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800"
 
 [[package]]
 name = "arc-swap"
@@ -132,9 +132,12 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.19"
+version = "0.4.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
+checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
+dependencies = [
+ "cfg-if",
+]
 
 [[package]]
 name = "memchr"
@@ -182,9 +185,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.64"
+version = "1.0.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da"
+checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b"
 dependencies = [
  "unicode-ident",
 ]
@@ -229,9 +232,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-log"
-version = "0.8.3"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f47b0777feb17f61eea78667d61103758b243a871edc09a7786500a50467b605"
+checksum = "f9c8b57fe71fb5dcf38970ebedc2b1531cf1c14b1b9b4c560a182a57e115575c"
 dependencies = [
  "arc-swap",
  "log",
@@ -247,7 +250,7 @@ dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
  "quote",
- "syn 1.0.104",
+ "syn",
 ]
 
 [[package]]
@@ -258,7 +261,7 @@ checksum = "c8df9be978a2d2f0cdebabb03206ed73b11314701a5bfe71b0d753b81997777f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.104",
+ "syn",
 ]
 
 [[package]]
@@ -273,9 +276,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.29"
+version = "1.0.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
+checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
 dependencies = [
  "proc-macro2",
 ]
@@ -291,21 +294,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.9.1"
+version = "1.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-automata",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-automata"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83d3daa6976cffb758ec878f108ba0e062a45b2d6ca3a2cca965338855476caf"
+checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -314,9 +305,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.7.3"
+version = "0.6.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ab07dc67230e4a4718e70fd5c20055a4334b121f1f9db8fe63ef39ce9b8c846"
+checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
 
 [[package]]
 name = "ryu"
@@ -332,29 +323,29 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "serde"
-version = "1.0.171"
+version = "1.0.155"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9"
+checksum = "71f2b4817415c6d4210bfe1c7bfcf4801b2d904cb4d0e1a8fdb651013c9e86b8"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.171"
+version = "1.0.155"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682"
+checksum = "d071a94a3fac4aff69d023a7f411e33f40f3483f8c5190b1953822b6b76d7630"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.25",
+ "syn",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.103"
+version = "1.0.94"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b"
+checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea"
 dependencies = [
  "itoa",
  "ryu",
@@ -384,17 +375,6 @@ dependencies = [
  "unicode-ident",
 ]
 
-[[package]]
-name = "syn"
-version = "2.0.25"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
 [[package]]
 name = "synapse"
 version = "0.1.0"

Cargo.toml

@@ -3,4 +3,3 @@
 
 [workspace]
 members = ["rust"]
-resolver = "2"

changelog.d/14755.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug in which the user directory would assume any remote membership state events represent a profile change.

changelog.d/14756.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug in which the user directory would assume any remote membership state events represent a profile change.

changelog.d/14921.misc

@@ -0,0 +1 @@
+Add additional functionality to declaring worker types when starting Complement in worker mode.

changelog.d/15131.misc

@@ -0,0 +1 @@
+Add a new third party callback `check_event_allowed_v2` that is compatible with new batch persisting mechanisms.

changelog.d/15187.feature

@@ -0,0 +1 @@
+Stabilise support for [MSC3966](https://github.com/matrix-org/matrix-spec-proposals/pull/3966): `event_property_contains` push condition.

changelog.d/15190.bugfix

@@ -0,0 +1 @@
+Implement [MSC3873](https://github.com/matrix-org/matrix-spec-proposals/pull/3873) to fix a long-standing bug where properties with dots were handled ambiguously in push rules.

changelog.d/15195.misc

@@ -0,0 +1 @@
+Improve performance of creating and authenticating events.

changelog.d/15200.misc

@@ -0,0 +1 @@
+Make the `HttpTransactionCache` use the `Requester` in addition of the just the `Request` to build the transaction key.

changelog.d/15222.misc

@@ -0,0 +1 @@
+Improve log lines when purging rooms.

changelog.d/15223.doc

@@ -0,0 +1 @@
+Add a missing endpoint to the workers documentation.

changelog.d/15229.misc

@@ -0,0 +1 @@
+Add topic and name events to group of events that are batch persisted when creating a room.

changelog.d/15230.misc

@@ -0,0 +1 @@
+Improve type hints.

changelog.d/15231.misc

@@ -0,0 +1 @@
+Improve type hints.

changelog.d/15232.bugfix

@@ -0,0 +1 @@
+Faster joins: Fix a bug introduced in Synapse 1.66 where spurious "Failed to find memberships ..." errors would be logged.

changelog.d/15237.misc

@@ -0,0 +1 @@
+Move various module API callback registration methods to a dedicated class.

changelog.d/15238.misc

@@ -0,0 +1 @@
+Improve type hints.

changelog.d/15239.docker

@@ -0,0 +1 @@
+Ensure the Dockerfile builds on platforms that don't have a `cryptography` wheel.

changelog.d/15244.misc

@@ -0,0 +1 @@
+Configure GitHub Actions for merge queues.

changelog.d/15247.misc

@@ -0,0 +1 @@
+Add schema comments about the `destinations` and `destination_rooms` tables.

changelog.d/15249.feature

@@ -0,0 +1 @@
+Implement [MSC2659](https://github.com/matrix-org/matrix-spec-proposals/pull/2659): application service ping endpoint. Contributed by Tulir @ Beeper.

changelog.d/15252.misc

@@ -0,0 +1 @@
+Bump hiredis from 2.2.1 to 2.2.2.

changelog.d/15253.misc

@@ -0,0 +1 @@
+Bump serde from 1.0.152 to 1.0.155.

changelog.d/15254.misc

@@ -0,0 +1 @@
+Bump pysaml2 from 7.2.1 to 7.3.1.

changelog.d/15255.misc

@@ -0,0 +1 @@
+Bump msgpack from 1.0.4 to 1.0.5.

changelog.d/15256.misc

@@ -0,0 +1 @@
+Bump gitpython from 3.1.30 to 3.1.31.

changelog.d/15257.misc

@@ -0,0 +1 @@
+Bump cryptography from 39.0.1 to 39.0.2.

changelog.d/15262.misc

@@ -0,0 +1 @@
+Skip processing of auto-join room behaviour if there are not auto-join rooms configured.

changelog.d/15266.misc

@@ -0,0 +1 @@
+Remove unused store method `_set_destination_retry_timings_emulated`.

changelog.d/15272.misc

@@ -0,0 +1 @@
+Clean-up direct TCP replication code.

changelog.d/15274.misc

@@ -0,0 +1 @@
+Clean-up direct TCP replication code.

changelog.d/15708.feature

@@ -1 +0,0 @@
-Add Unix Socket support for HTTP Replication Listeners. Document and provide usage instructions for utilizing Unix sockets in Synapse. Contributed by Jason Little.

changelog.d/15820.bugfix

@@ -1 +0,0 @@
-Fix long-standing bug where remote invites weren't correctly pushed.

changelog.d/15884.misc

@@ -1 +0,0 @@
-Mark `get_user_in_directory` private since it is only used in tests. Also remove the cache from it.

changelog.d/15887.misc

@@ -1 +0,0 @@
-Fix background schema updates failing over a large upgrade gap.

changelog.d/15909.misc

@@ -1 +0,0 @@
-Document which Python version runs on a given Linux distribution so we can more easily clean up later.

changelog.d/15911.feature

@@ -1 +0,0 @@
-Allow `+` in Matrix IDs, per [MSC4009](https://github.com/matrix-org/matrix-spec-proposals/pull/4009).

changelog.d/15912.feature

@@ -1 +0,0 @@
-Support room version 11 from [MSC3820](https://github.com/matrix-org/matrix-spec-proposals/pull/3820).

changelog.d/15913.feature

@@ -1 +0,0 @@
-Allow configuring the set of workers to proxy outbound federation traffic through via `outbound_federation_restricted_to`.

changelog.d/15921.doc

@@ -1 +0,0 @@
-Better clarify how to run a worker instance (pass both configs).

changelog.d/15922.misc

@@ -1 +0,0 @@
-Add details to warning in log when we fail to fetch an alias.

changelog.d/15924.feature

@@ -1 +0,0 @@
-Add Unix Socket support for HTTP Replication Listeners. Document and provide usage instructions for utilizing Unix sockets in Synapse. Contributed by Jason Little.

changelog.d/15925.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in 1.86.0 where Synapse starting with an empty `experimental_features` configuration setting.

changelog.d/15926.misc

@@ -1 +0,0 @@
-Remove unneeded `__init__`.

changelog.d/15928.removal

@@ -1 +0,0 @@
-Remove support for calling the `/register` endpoint with an unspecced `user` property for application services.

changelog.d/15933.misc

@@ -1 +0,0 @@
-Fix bug with read/write lock implementation. This is currently unused so has no observable effects.

changelog.d/15938.doc

@@ -1 +0,0 @@
-Improve the documentation for the login as a user admin API.

changelog.d/15940.misc

@@ -1 +0,0 @@
-Unbreak the nix development environment by pinning the Rust version to 1.70.0.

changelog.d/15952.misc

@@ -1 +0,0 @@
-Update presence metrics to differentiate remote vs local users.

changelog.d/15958.misc

@@ -1 +0,0 @@
-Fix bug with read/write lock implementation. This is currently unused so has no observable effects.

changelog.d/15960.misc

@@ -1 +0,0 @@
-Ensure a long state res does not starve CPU by occasionally yielding to the reactor.

changelog.d/15968.misc

@@ -1 +0,0 @@
-Reduce the amount of state we pull out.

changelog.d/15969.feature

@@ -1 +0,0 @@
-Allow configuring the set of workers to proxy outbound federation traffic through via `outbound_federation_restricted_to`.

changelog.d/15970.misc

@@ -1 +0,0 @@
-Speed up calculation of remote hosts.

contrib/docker_compose_workers/README.md

@@ -70,10 +70,6 @@ redis:
   port: 6379
   # dbid:  <redis_logical_db_id>
   # password: <secret_password>  
-  # use_tls: True
-  # certificate_file: <path_to_certificate>
-  # private_key_file: <path_to_private_key>
-  # ca_file: <path_to_ca_certificate>
 ```
 
 This assumes that your Redis service is called `redis` in your Docker Compose file.

contrib/lnav/synapse-log-format.json

@@ -29,7 +29,7 @@
         "level": "error"
       },
       {
-        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix-federation://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
+        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
         "level": "warning"
       },
       {

debian/changelog

@@ -1,153 +1,3 @@
-matrix-synapse-py3 (1.88.0) stable; urgency=medium
-
-  * New Synapse release 1.88.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 18 Jul 2023 13:59:28 +0100
-
-matrix-synapse-py3 (1.88.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.88.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 11 Jul 2023 10:20:19 +0100
-
-matrix-synapse-py3 (1.87.0) stable; urgency=medium
-
-  * New Synapse release 1.87.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 04 Jul 2023 16:24:00 +0100
-
-matrix-synapse-py3 (1.87.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.87.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 27 Jun 2023 15:27:04 +0000
-
-matrix-synapse-py3 (1.86.0) stable; urgency=medium
-
-  * New Synapse release 1.86.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Jun 2023 17:22:46 +0200
-
-matrix-synapse-py3 (1.86.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.86.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 14 Jun 2023 12:16:27 +0200
-
-matrix-synapse-py3 (1.86.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.86.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Jun 2023 14:30:45 +0200
-
-matrix-synapse-py3 (1.85.2) stable; urgency=medium
-
-  * New Synapse release 1.85.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 08 Jun 2023 13:04:18 +0100
-
-matrix-synapse-py3 (1.85.1) stable; urgency=medium
-
-  * New Synapse release 1.85.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 07 Jun 2023 10:51:12 +0100
-
-matrix-synapse-py3 (1.85.0) stable; urgency=medium
-
-  * New Synapse release 1.85.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 06 Jun 2023 09:39:29 +0100
-
-matrix-synapse-py3 (1.85.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.85.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 01 Jun 2023 09:16:18 -0700
-
-matrix-synapse-py3 (1.85.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.85.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 30 May 2023 13:56:54 +0100
-
-matrix-synapse-py3 (1.84.1) stable; urgency=medium
-
-  * New Synapse release 1.84.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 26 May 2023 16:15:30 +0100
-
-matrix-synapse-py3 (1.84.0) stable; urgency=medium
-
-  * New Synapse release 1.84.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 23 May 2023 10:57:22 +0100
-
-matrix-synapse-py3 (1.84.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.84.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 16 May 2023 11:12:02 +0100
-
-matrix-synapse-py3 (1.83.0) stable; urgency=medium
-
-  * New Synapse release 1.83.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 09 May 2023 18:13:37 +0200
-
-matrix-synapse-py3 (1.83.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.83.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 02 May 2023 15:56:38 +0100
-
-matrix-synapse-py3 (1.82.0) stable; urgency=medium
-
-  * New Synapse release 1.82.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 25 Apr 2023 11:56:06 +0100
-
-matrix-synapse-py3 (1.82.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.82.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 18 Apr 2023 09:47:30 +0100
-
-matrix-synapse-py3 (1.81.0) stable; urgency=medium
-
-  * New Synapse release 1.81.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 11 Apr 2023 14:18:35 +0100
-
-matrix-synapse-py3 (1.81.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.81.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 06 Apr 2023 16:07:54 +0100
-
-matrix-synapse-py3 (1.81.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.81.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 04 Apr 2023 14:29:03 +0100
-
-matrix-synapse-py3 (1.80.0) stable; urgency=medium
-
-  * New Synapse release 1.80.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 28 Mar 2023 11:10:33 +0100
-
-matrix-synapse-py3 (1.80.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.80.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 22 Mar 2023 08:30:16 -0700
-
-matrix-synapse-py3 (1.80.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.80.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 21 Mar 2023 10:56:08 -0700
-
 matrix-synapse-py3 (1.79.0) stable; urgency=medium
 
   * New Synapse release 1.79.0.

demo/start.sh

@@ -46,7 +46,7 @@ for port in 8080 8081 8082; do
             echo ''
 
 			# Warning, this heredoc depends on the interaction of tabs and spaces.
-			# Please don't accidentally bork me with your fancy settings.
+			# Please don't accidentaly bork me with your fancy settings.
 			listeners=$(cat <<-PORTLISTENERS
 			# Configure server to listen on both $https_port and $port
 			# This overides some of the default settings above
@@ -80,8 +80,12 @@ for port in 8080 8081 8082; do
             echo "tls_certificate_path: \"$DIR/$port/localhost:$port.tls.crt\""
             echo "tls_private_key_path: \"$DIR/$port/localhost:$port.tls.key\""
 
-            # Request keys directly from servers contacted over federation
-            echo 'trusted_key_servers: []'
+            # Ignore keys from the trusted keys server
+            echo '# Ignore keys from the trusted keys server'
+            echo 'trusted_key_servers:'
+            echo '  - server_name: "matrix.org"'
+            echo '    accept_keys_insecurely: true'
+            echo ''
 
 			# Allow the servers to communicate over localhost.
 			allow_list=$(cat <<-ALLOW_LIST

dev-docs/Makefile

@@ -1,20 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = .
-BUILDDIR      = _build
-
-# Put it first so that "make" without argument is like "make help".
-help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

dev-docs/conf.py

@@ -1,50 +0,0 @@
-# Configuration file for the Sphinx documentation builder.
-#
-# For the full list of built-in configuration values, see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-# -- Project information -----------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
-
-project = "Synapse development"
-copyright = "2023, The Matrix.org Foundation C.I.C."
-author = "The Synapse Maintainers and Community"
-
-# -- General configuration ---------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
-
-extensions = [
-    "autodoc2",
-    "myst_parser",
-]
-
-templates_path = ["_templates"]
-exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
-
-
-# -- Options for Autodoc2 ----------------------------------------------------
-
-autodoc2_docstring_parser_regexes = [
-    # this will render all docstrings as 'MyST' Markdown
-    (r".*", "myst"),
-]
-
-autodoc2_packages = [
-    {
-        "path": "../synapse",
-        # Don't render documentation for everything as a matter of course
-        "auto_mode": False,
-    },
-]
-
-
-# -- Options for MyST (Markdown) ---------------------------------------------
-
-# myst_heading_anchors = 2
-
-
-# -- Options for HTML output -------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
-
-html_theme = "furo"
-html_static_path = ["_static"]

dev-docs/index.rst

@@ -1,22 +0,0 @@
-.. Synapse Developer Documentation documentation master file, created by
-   sphinx-quickstart on Mon Mar 13 08:59:51 2023.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
-
-Welcome to the Synapse Developer Documentation!
-===========================================================
-
-.. toctree::
-   :maxdepth: 2
-   :caption: Contents:
-
-   modules/federation_sender
-
-
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`

dev-docs/modules/federation_sender.md

@@ -1,5 +0,0 @@
-Federation Sender
-=================
-
-```{autodoc2-docstring} synapse.federation.sender
-```

docker/Dockerfile

@@ -27,7 +27,7 @@ ARG PYTHON_VERSION=3.11
 ###
 # We hardcode the use of Debian bullseye here because this could change upstream
 # and other Dockerfiles used for testing are expecting bullseye.
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye as requirements
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye as requirements
 
 # RUN --mount is specific to buildkit and is documented at
 # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
@@ -37,7 +37,7 @@ RUN \
   --mount=type=cache,target=/var/cache/apt,sharing=locked \
   --mount=type=cache,target=/var/lib/apt,sharing=locked \
   apt-get update -qq && apt-get install -yqq \
-  build-essential curl git libffi-dev libssl-dev pkg-config \
+  build-essential curl git libffi-dev libssl-dev \
   && rm -rf /var/lib/apt/lists/*
 
 # Install rust and ensure its in the PATH.
@@ -87,7 +87,7 @@ RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
 ###
 ### Stage 1: builder
 ###
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye as builder
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye as builder
 
 # install the OS build deps
 RUN \
@@ -158,7 +158,7 @@ RUN --mount=type=cache,target=/synapse/target,sharing=locked \
 ### Stage 2: runtime
 ###
 
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye
 
 LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
 LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'

docker/Dockerfile-dhvirtualenv

@@ -24,16 +24,16 @@ ARG distro=""
 # https://launchpad.net/~jyrki-pulliainen/+archive/ubuntu/dh-virtualenv, but
 # it's not obviously easier to use that than to build our own.)
 
-FROM docker.io/library/${distro} as builder
+FROM ${distro} as builder
 
 RUN apt-get update -qq -o Acquire::Languages=none
 RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
-    -yqq --no-install-recommends \
-    build-essential \
-    ca-certificates \
-    devscripts \
-    equivs \
-    wget
+        -yqq --no-install-recommends \
+        build-essential \
+        ca-certificates \
+        devscripts \
+        equivs \
+        wget
 
 # fetch and unpack the package
 # We are temporarily using a fork of dh-virtualenv due to an incompatibility with Python 3.11, which ships with
@@ -55,36 +55,40 @@ RUN cd /dh-virtualenv && DEB_BUILD_OPTIONS=nodoc dpkg-buildpackage -us -uc -b
 ###
 ### Stage 1
 ###
-FROM docker.io/library/${distro}
+FROM ${distro}
 
 # Get the distro we want to pull from as a dynamic build variable
 # (We need to define it in each build stage)
 ARG distro=""
 ENV distro ${distro}
 
+# Python < 3.7 assumes LANG="C" means ASCII-only and throws on printing unicode
+# http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
 # Install the build dependencies
 #
 # NB: keep this list in sync with the list of build-deps in debian/control
 # TODO: it would be nice to do that automatically.
 RUN apt-get update -qq -o Acquire::Languages=none \
     && env DEBIAN_FRONTEND=noninteractive apt-get install \
-    -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
-    build-essential \
-    curl \
-    debhelper \
-    devscripts \
-    libsystemd-dev \
-    lsb-release \
-    pkg-config \
-    python3-dev \
-    python3-pip \
-    python3-setuptools \
-    python3-venv \
-    sqlite3 \
-    libpq-dev \
-    libicu-dev \
-    pkg-config \
-    xmlsec1
+        -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
+        build-essential \
+        curl \
+        debhelper \
+        devscripts \
+        libsystemd-dev \
+        lsb-release \
+        pkg-config \
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        python3-venv \
+        sqlite3 \
+        libpq-dev \
+        libicu-dev \
+        pkg-config \
+        xmlsec1
 
 # Install rust and ensure it's in the PATH
 ENV RUSTUP_HOME=/rust

docker/Dockerfile-workers

@@ -7,7 +7,7 @@ ARG FROM=matrixdotorg/synapse:$SYNAPSE_VERSION
 # target image. For repeated rebuilds, this is much faster than apt installing
 # each time.
 
-FROM docker.io/library/debian:bullseye-slim AS deps_base
+FROM debian:bullseye-slim AS deps_base
     RUN \
        --mount=type=cache,target=/var/cache/apt,sharing=locked \
        --mount=type=cache,target=/var/lib/apt,sharing=locked \
@@ -21,7 +21,7 @@ FROM docker.io/library/debian:bullseye-slim AS deps_base
 # which makes it much easier to copy (but we need to make sure we use an image
 # based on the same debian version as the synapse image, to make sure we get
 # the expected version of libc.
-FROM docker.io/library/redis:7-bullseye AS redis_base
+FROM redis:6-bullseye AS redis_base
 
 # now build the final image, based on the the regular Synapse docker image
 FROM $FROM

docker/README.md

@@ -73,8 +73,7 @@ The following environment variables are supported in `generate` mode:
   will log sensitive information such as access tokens.
   This should not be needed unless you are a developer attempting to debug something
   particularly tricky.
-* `SYNAPSE_LOG_TESTING`: if set, Synapse will log additional information useful
-  for testing.
+
 
 ## Postgres
 

docker/complement/Dockerfile

@@ -7,7 +7,6 @@
 # https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse
 
 ARG SYNAPSE_VERSION=latest
-# This is an intermediate image, to be built locally (not pulled from a registry).
 ARG FROM=matrixdotorg/synapse-workers:$SYNAPSE_VERSION
 
 FROM $FROM
@@ -20,8 +19,8 @@ FROM $FROM
     # the same debian version as Synapse's docker image (so the versions of the
     # shared libraries match).
     RUN adduser --system --uid 999 postgres --home /var/lib/postgresql
-    COPY --from=docker.io/library/postgres:13-bullseye /usr/lib/postgresql /usr/lib/postgresql
-    COPY --from=docker.io/library/postgres:13-bullseye /usr/share/postgresql /usr/share/postgresql
+    COPY --from=postgres:13-bullseye /usr/lib/postgresql /usr/lib/postgresql
+    COPY --from=postgres:13-bullseye /usr/share/postgresql /usr/share/postgresql
     RUN mkdir /var/run/postgresql && chown postgres /var/run/postgresql
     ENV PATH="${PATH}:/usr/lib/postgresql/13/bin"
     ENV PGDATA=/var/lib/postgresql/data

docker/complement/conf/workers-shared-extra.yaml.j2

@@ -92,6 +92,8 @@ allow_device_name_lookup_over_federation: true
 ## Experimental Features ##
 
 experimental_features:
+  # Enable history backfilling support
+  msc2716_enabled: true
   # client-side support for partial state in /send_join responses
   faster_joins: true
   # Enable support for polls

docker/conf-workers/nginx.conf.j2

@@ -35,11 +35,7 @@ server {
 
     # Send all other traffic to the main process
     location ~* ^(\\/_matrix|\\/_synapse) {
-{% if using_unix_sockets %}
-        proxy_pass http://unix:/run/main_public.sock;
-{% else %}
         proxy_pass http://localhost:8080;
-{% endif %}
         proxy_set_header X-Forwarded-For $remote_addr;
         proxy_set_header X-Forwarded-Proto $scheme;
         proxy_set_header Host $host;

docker/conf-workers/shared.yaml.j2

@@ -6,9 +6,6 @@
 {% if enable_redis %}
 redis:
     enabled: true
-    {% if using_unix_sockets %}
-    path: /tmp/redis.sock
-    {% endif %}
 {% endif %}
 
 {% if appservice_registrations is not none %}

docker/conf-workers/supervisord.conf.j2

@@ -19,11 +19,7 @@ username=www-data
 autorestart=true
 
 [program:redis]
-{% if using_unix_sockets %}
-command=/usr/local/bin/prefix-log /usr/local/bin/redis-server --unixsocket /tmp/redis.sock
-{% else %}
 command=/usr/local/bin/prefix-log /usr/local/bin/redis-server
-{% endif %}
 priority=1
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0

docker/conf-workers/worker.yaml.j2

@@ -6,13 +6,13 @@
 worker_app: "{{ app }}"
 worker_name: "{{ name }}"
 
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_listeners:
   - type: http
-{% if using_unix_sockets %}
-    path: "/run/worker.{{ port }}"
-{% else %}
     port: {{ port }}
-{% endif %}
 {% if listener_resources %}
     resources:
       - names:

docker/conf/homeserver.yaml

@@ -36,17 +36,12 @@ listeners:
 
   # Allow configuring in case we want to reverse proxy 8008
   # using another process in the same container
-{% if SYNAPSE_USE_UNIX_SOCKET %}
-  # Unix sockets don't care about TLS or IP addresses or ports
-  - path: '/run/main_public.sock'
-    type: http
-{% else %}
   - port: {{ SYNAPSE_HTTP_PORT or 8008 }}
     tls: false
     bind_addresses: ['::']
     type: http
     x_forwarded: false
-{% endif %}
+
     resources:
       - names: [client]
         compress: true
@@ -62,11 +57,8 @@ database:
     user: "{{ POSTGRES_USER or "synapse" }}"
     password: "{{ POSTGRES_PASSWORD }}"
     database: "{{ POSTGRES_DB or "synapse" }}"
-{% if not SYNAPSE_USE_UNIX_SOCKET %}
-{# Synapse will use a default unix socket for Postgres when host/port is not specified (behavior from `psycopg2`). #}
     host: "{{ POSTGRES_HOST or "db" }}"
     port: "{{ POSTGRES_PORT or "5432" }}"
-{% endif %}
     cp_min: 5
     cp_max: 10
 {% else %}

docker/conf/log.config

@@ -49,35 +49,17 @@ handlers:
     class: logging.StreamHandler
     formatter: precise
 
+{% if not SYNAPSE_LOG_SENSITIVE %}
+{#
+  If SYNAPSE_LOG_SENSITIVE is unset, then override synapse.storage.SQL to INFO
+  so that DEBUG entries (containing sensitive information) are not emitted.
+#}
 loggers:
-    # This is just here so we can leave `loggers` in the config regardless of whether
-    # we configure other loggers below (avoid empty yaml dict error).
-    _placeholder:
-        level: "INFO"
-
-    {% if not SYNAPSE_LOG_SENSITIVE %}
-    {#
-      If SYNAPSE_LOG_SENSITIVE is unset, then override synapse.storage.SQL to INFO
-      so that DEBUG entries (containing sensitive information) are not emitted.
-    #}
     synapse.storage.SQL:
         # beware: increasing this to DEBUG will make synapse log sensitive
         # information such as access tokens.
         level: INFO
-    {% endif %}
-
-    {% if SYNAPSE_LOG_TESTING %}
-    {#
-      If Synapse is under test, log a few more useful things for a developer
-      attempting to debug something particularly tricky.
-
-      With `synapse.visibility.filtered_event_debug`, it logs when events are (maybe
-      unexpectedly) filtered out of responses in tests. It's just nice to be able to
-      look at the CI log and figure out why an event isn't being returned.
-    #}
-    synapse.visibility.filtered_event_debug:
-        level: DEBUG
-    {% endif %}
+{% endif %}
 
 root:
     level: {{ SYNAPSE_LOG_LEVEL or "INFO" }}

docker/configure_workers_and_start.py

@@ -40,8 +40,6 @@
 #         log level. INFO is the default.
 #   * SYNAPSE_LOG_SENSITIVE: If unset, SQL and SQL values won't be logged,
 #         regardless of the SYNAPSE_LOG_LEVEL setting.
-#   * SYNAPSE_LOG_TESTING: if set, Synapse will log additional information useful
-#     for testing.
 #
 # NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined
 # in the project's README), this script may be run multiple times, and functionality should
@@ -71,12 +69,6 @@ import yaml
 from jinja2 import Environment, FileSystemLoader
 
 MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
-MAIN_PROCESS_INSTANCE_NAME = "main"
-MAIN_PROCESS_LOCALHOST_ADDRESS = "127.0.0.1"
-MAIN_PROCESS_REPLICATION_PORT = 9093
-# Obviously, these would only be used with the UNIX socket option
-MAIN_PROCESS_UNIX_SOCKET_PUBLIC_PATH = "/run/main_public.sock"
-MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH = "/run/main_private.sock"
 
 # A simple name used as a placeholder in the WORKERS_CONFIG below. This will be replaced
 # during processing with the name of the worker.
@@ -171,7 +163,6 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
             "^/_matrix/client/versions$",
             "^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$",
             "^/_matrix/client/(r0|v3|unstable)/register$",
-            "^/_matrix/client/(r0|v3|unstable)/register/available$",
             "^/_matrix/client/(r0|v3|unstable)/auth/.*/fallback/web$",
             "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/messages$",
             "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/event",
@@ -180,9 +171,6 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
             "^/_matrix/client/v1/rooms/.*/timestamp_to_event$",
             "^/_matrix/client/(api/v1|r0|v3|unstable)/search",
             "^/_matrix/client/(r0|v3|unstable)/user/.*/filter(/|$)",
-            "^/_matrix/client/(r0|v3|unstable)/password_policy$",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/directory/room/.*$",
-            "^/_matrix/client/(r0|v3|unstable)/capabilities$",
         ],
         "shared_extra_conf": {},
         "worker_extra_conf": "",
@@ -247,6 +235,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
             "^/_matrix/client/(api/v1|r0|v3|unstable)/join/",
             "^/_matrix/client/(api/v1|r0|v3|unstable)/knock/",
             "^/_matrix/client/(api/v1|r0|v3|unstable)/profile/",
+            "^/_matrix/client/(v1|unstable/org.matrix.msc2716)/rooms/.*/batch_send",
         ],
         "shared_extra_conf": {},
         "worker_extra_conf": "",
@@ -410,15 +399,11 @@ def add_worker_roles_to_shared_config(
         )
 
         # Map of stream writer instance names to host/ports combos
-        if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False):
-            instance_map[worker_name] = {
-                "path": f"/run/worker.{worker_port}",
-            }
-        else:
-            instance_map[worker_name] = {
-                "host": "localhost",
-                "port": worker_port,
-            }
+        instance_map[worker_name] = {
+            "host": "localhost",
+            "port": worker_port,
+        }
+
     # Update the list of stream writers. It's convenient that the name of the worker
     # type is the same as the stream to write. Iterate over the whole list in case there
     # is more than one.
@@ -430,19 +415,14 @@ def add_worker_roles_to_shared_config(
 
             # Map of stream writer instance names to host/ports combos
             # For now, all stream writers need http replication ports
-            if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False):
-                instance_map[worker_name] = {
-                    "path": f"/run/worker.{worker_port}",
-                }
-            else:
-                instance_map[worker_name] = {
-                    "host": "localhost",
-                    "port": worker_port,
-                }
+            instance_map[worker_name] = {
+                "host": "localhost",
+                "port": worker_port,
+            }
 
 
 def merge_worker_template_configs(
-    existing_dict: Optional[Dict[str, Any]],
+    existing_dict: Dict[str, Any] | None,
     to_be_merged_dict: Dict[str, Any],
 ) -> Dict[str, Any]:
     """When given an existing dict of worker template configuration consisting with both
@@ -730,29 +710,17 @@ def generate_worker_files(
     # Note that yaml cares about indentation, so care should be taken to insert lines
     # into files at the correct indentation below.
 
-    # Convenience helper for if using unix sockets instead of host:port
-    using_unix_sockets = environ.get("SYNAPSE_USE_UNIX_SOCKET", False)
     # First read the original config file and extract the listeners block. Then we'll
     # add another listener for replication. Later we'll write out the result to the
     # shared config file.
-    listeners: List[Any]
-    if using_unix_sockets:
-        listeners = [
-            {
-                "path": MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH,
-                "type": "http",
-                "resources": [{"names": ["replication"]}],
-            }
-        ]
-    else:
-        listeners = [
-            {
-                "port": MAIN_PROCESS_REPLICATION_PORT,
-                "bind_address": MAIN_PROCESS_LOCALHOST_ADDRESS,
-                "type": "http",
-                "resources": [{"names": ["replication"]}],
-            }
-        ]
+    listeners = [
+        {
+            "port": 9093,
+            "bind_address": "127.0.0.1",
+            "type": "http",
+            "resources": [{"names": ["replication"]}],
+        }
+    ]
     with open(config_path) as file_stream:
         original_config = yaml.safe_load(file_stream)
         original_listeners = original_config.get("listeners")
@@ -793,17 +761,7 @@ def generate_worker_files(
 
     # A list of internal endpoints to healthcheck, starting with the main process
     # which exists even if no workers do.
-    # This list ends up being part of the command line to curl, (curl added support for
-    # Unix sockets in version 7.40).
-    if using_unix_sockets:
-        healthcheck_urls = [
-            f"--unix-socket {MAIN_PROCESS_UNIX_SOCKET_PUBLIC_PATH} "
-            # The scheme and hostname from the following URL are ignored.
-            # The only thing that matters is the path `/health`
-            "http://localhost/health"
-        ]
-    else:
-        healthcheck_urls = ["http://localhost:8080/health"]
+    healthcheck_urls = ["http://localhost:8080/health"]
 
     # Get the set of all worker types that we have configured
     all_worker_types_in_use = set(chain(*requested_worker_types.values()))
@@ -840,12 +798,8 @@ def generate_worker_files(
         # given worker_type needs to stay assigned and not be replaced.
         worker_config["shared_extra_conf"].update(shared_config)
         shared_config = worker_config["shared_extra_conf"]
-        if using_unix_sockets:
-            healthcheck_urls.append(
-                f"--unix-socket /run/worker.{worker_port} http://localhost/health"
-            )
-        else:
-            healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))
+
+        healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))
 
         # Update the shared config with sharding-related options if necessary
         add_worker_roles_to_shared_config(
@@ -864,7 +818,6 @@ def generate_worker_files(
             "/conf/workers/{name}.yaml".format(name=worker_name),
             **worker_config,
             worker_log_config_filepath=log_config_filepath,
-            using_unix_sockets=using_unix_sockets,
         )
 
         # Save this worker's port number to the correct nginx upstreams
@@ -885,13 +838,8 @@ def generate_worker_files(
     nginx_upstream_config = ""
     for upstream_worker_base_name, upstream_worker_ports in nginx_upstreams.items():
         body = ""
-        if using_unix_sockets:
-            for port in upstream_worker_ports:
-                body += f"    server unix:/run/worker.{port};\n"
-
-        else:
-            for port in upstream_worker_ports:
-                body += f"    server localhost:{port};\n"
+        for port in upstream_worker_ports:
+            body += f"    server localhost:{port};\n"
 
         # Add to the list of configured upstreams
         nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
@@ -918,19 +866,6 @@ def generate_worker_files(
 
     workers_in_use = len(requested_worker_types) > 0
 
-    # If there are workers, add the main process to the instance_map too.
-    if workers_in_use:
-        instance_map = shared_config.setdefault("instance_map", {})
-        if using_unix_sockets:
-            instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
-                "path": MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH,
-            }
-        else:
-            instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
-                "host": MAIN_PROCESS_LOCALHOST_ADDRESS,
-                "port": MAIN_PROCESS_REPLICATION_PORT,
-            }
-
     # Shared homeserver config
     convert(
         "/conf/shared.yaml.j2",
@@ -939,7 +874,6 @@ def generate_worker_files(
         appservice_registrations=appservice_registrations,
         enable_redis=workers_in_use,
         workers_in_use=workers_in_use,
-        using_unix_sockets=using_unix_sockets,
     )
 
     # Nginx config
@@ -950,7 +884,6 @@ def generate_worker_files(
         upstream_directives=nginx_upstream_config,
         tls_cert_path=os.environ.get("SYNAPSE_TLS_CERT"),
         tls_key_path=os.environ.get("SYNAPSE_TLS_KEY"),
-        using_unix_sockets=using_unix_sockets,
     )
 
     # Supervisord config
@@ -960,7 +893,6 @@ def generate_worker_files(
         "/etc/supervisor/supervisord.conf",
         main_config_path=config_path,
         enable_redis=workers_in_use,
-        using_unix_sockets=using_unix_sockets,
     )
 
     convert(
@@ -1000,7 +932,6 @@ def generate_worker_log_config(
     extra_log_template_args["SYNAPSE_LOG_SENSITIVE"] = environ.get(
         "SYNAPSE_LOG_SENSITIVE"
     )
-    extra_log_template_args["SYNAPSE_LOG_TESTING"] = environ.get("SYNAPSE_LOG_TESTING")
 
     # Render and write the file
     log_config_filepath = f"/conf/workers/{worker_name}.log.config"

docker/editable.Dockerfile

@@ -10,7 +10,7 @@ ARG PYTHON_VERSION=3.9
 ###
 # We hardcode the use of Debian bullseye here because this could change upstream
 # and other Dockerfiles used for testing are expecting bullseye.
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye
 
 # Install Rust and other dependencies (stolen from normal Dockerfile)
 # install the OS build deps

docs/SUMMARY.md

@@ -57,7 +57,6 @@
       - [Account Validity](admin_api/account_validity.md)
       - [Background Updates](usage/administration/admin_api/background_updates.md)
       - [Event Reports](admin_api/event_reports.md)
-      - [Experimental Features](admin_api/experimental_features.md)
       - [Media](admin_api/media_admin_api.md)
       - [Purge History](admin_api/purge_history_api.md)
       - [Register Users](admin_api/register_api.md)

docs/admin_api/experimental_features.md

@@ -1,55 +0,0 @@
-# Experimental Features API
-
-This API allows a server administrator to enable or disable some experimental features on a per-user
-basis. The currently supported features are: 
-- [MSC3026](https://github.com/matrix-org/matrix-spec-proposals/pull/3026): busy 
-presence state enabled
-- [MSC3881](https://github.com/matrix-org/matrix-spec-proposals/pull/3881): enable remotely toggling push notifications 
-for another client 
-- [MSC3967](https://github.com/matrix-org/matrix-spec-proposals/pull/3967): do not require
-UIA when first uploading cross-signing keys. 
-
-
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
-
-## Enabling/Disabling Features
-
-This API allows a server administrator to enable experimental features for a given user. The request must 
-provide a body containing the user id and listing the features to enable/disable in the following format:
-```json
-{
-   "features": {
-      "msc3026":true,
-      "msc3881":true
-   }
-}
-```
-where true is  used to enable the feature, and false is used to disable the feature.
-
-
-The API is:
-
-```
-PUT /_synapse/admin/v1/experimental_features/<user_id>
-```
-
-## Listing Enabled Features
- 
-To list which features are enabled/disabled for a given user send a request to the following API:
-
-```
-GET /_synapse/admin/v1/experimental_features/<user_id>
-```
-
-It will return a list of possible features and indicate whether they are enabled or disabled for the
-user like so:
-```json
-{
-   "features": {
-      "msc3026": true,
-      "msc3881": false,
-      "msc3967": false
-   }
-}
-```

docs/admin_api/rooms.md

@@ -419,7 +419,7 @@ The following query parameters are available:
 
 * `from` (required) - The token to start returning events from. This token can be obtained from a prev_batch
   or next_batch token returned by the /sync endpoint, or from an end token returned by a previous request to this endpoint.
-* `to` - The token to stop returning events at.
+* `to` - The token to spot returning events at.
 * `limit` - The maximum number of events to return. Defaults to `10`.
 * `filter` - A JSON RoomEventFilter to filter returned events with.
 * `dir` - The direction to return events from. Either `f` for forwards or `b` for backwards. Setting

docs/admin_api/statistics.md

@@ -81,52 +81,3 @@ The following fields are returned in the JSON response body:
   - `user_id` - string - Fully-qualified user ID (ex. `@user:server.com`).
 * `next_token` - integer - Opaque value used for pagination. See above.
 * `total` - integer - Total number of users after filtering.
-
-
-# Get largest rooms by size in database
-
-Returns the 10 largest rooms and an estimate of how much space in the database
-they are taking.
-
-This does not include the size of any associated media associated with the room.
-
-Returns an error on SQLite.
-
-*Note:* This uses the planner statistics from PostgreSQL to do the estimates,
-which means that the returned information can vary widely from reality. However,
-it should be enough to get a rough idea of where database disk space is going.
-
-
-The API is:
-
-```
-GET /_synapse/admin/v1/statistics/database/rooms
-```
-
-A response body like the following is returned:
-
-```json
-{
-  "rooms": [
-    {
-      "room_id": "!OGEhHVWSdvArJzumhm:matrix.org",
-      "estimated_size": 47325417353
-    }
-  ],
-}
-```
-
-
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-* `rooms` - An array of objects, sorted by largest room first. Objects contain
-  the following fields:
-  - `room_id` - string - The room ID.
-  - `estimated_size` - integer - Estimated disk space used in bytes by the room
-    in the database.
-
-
-*Added in Synapse 1.83.0*

docs/admin_api/user_admin_api.md

@@ -62,7 +62,7 @@ URL parameters:
 
 - `user_id`: fully-qualified user id: for example, `@user:server.com`.
 
-## Create or modify account
+## Create or modify Account
 
 This API allows an administrator to create or modify a user account with a
 specific `user_id`.
@@ -78,29 +78,28 @@ with a body of:
 ```json
 {
     "password": "user_password",
-    "logout_devices": false,
-    "displayname": "Alice Marigold",
-    "avatar_url": "mxc://example.com/abcde12345",
+    "displayname": "User",
     "threepids": [
         {
             "medium": "email",
-            "address": "alice@example.com"
+            "address": "<user_mail_1>"
         },
         {
             "medium": "email",
-            "address": "alice@domain.org"
+            "address": "<user_mail_2>"
         }
     ],
     "external_ids": [
         {
-            "auth_provider": "example",
-            "external_id": "12345"
+            "auth_provider": "<provider1>",
+            "external_id": "<user_id_provider_1>"
         },
         {
-            "auth_provider": "example2",
-            "external_id": "abc54321"
+            "auth_provider": "<provider2>",
+            "external_id": "<user_id_provider_2>"
         }
     ],
+    "avatar_url": "<avatar_url>",
     "admin": false,
     "deactivated": false,
     "user_type": null
@@ -113,51 +112,41 @@ Returns HTTP status code:
 
 URL parameters:
 
-- `user_id` - A fully-qualified user id. For example, `@user:server.com`.
+- `user_id`: fully-qualified user id: for example, `@user:server.com`.
 
 Body parameters:
 
-- `password` - **string**, optional. If provided, the user's password is updated and all
+- `password` - string, optional. If provided, the user's password is updated and all
   devices are logged out, unless `logout_devices` is set to `false`.
-- `logout_devices` - **bool**, optional, defaults to `true`. If set to `false`, devices aren't
+- `logout_devices` - bool, optional, defaults to `true`. If set to false, devices aren't
   logged out even when `password` is provided.
-- `displayname` - **string**, optional. If set to an empty string (`""`), the user's display name
-  will be removed.
-- `avatar_url` - **string**, optional. Must be a
-  [MXC URI](https://matrix.org/docs/spec/client_server/r0.6.0#matrix-content-mxc-uris).
-  If set to an empty string (`""`), the user's avatar is removed.
-- `threepids` - **array**, optional. If provided, the user's third-party IDs (email, msisdn) are
-  entirely replaced with the given list. Each item in the array is an object with the following
-  fields:
-  - `medium` - **string**, required. The type of third-party ID, either `email` or `msisdn` (phone number).
-  - `address` - **string**, required. The third-party ID itself, e.g. `alice@example.com` for `email` or
-    `447470274584` (for a phone number with country code "44") and `19254857364` (for a phone number
-    with country code "1") for `msisdn`.
-  Note: If a threepid is removed from a user via this option, Synapse will also attempt to remove
-  that threepid from any identity servers it is aware has a binding for it.
-- `external_ids` - **array**, optional. Allow setting the identifier of the external identity
-  provider for SSO (Single sign-on). More details are in the configuration manual under the
+- `displayname` - string, optional, defaults to the value of `user_id`.
+- `threepids` - array, optional, allows setting the third-party IDs (email, msisdn)
+  - `medium` - string. Kind of third-party ID, either `email` or `msisdn`.
+  - `address` - string. Value of third-party ID.
+  belonging to a user.
+- `external_ids` - array, optional. Allow setting the identifier of the external identity
+  provider for SSO (Single sign-on). Details in the configuration manual under the
   sections [sso](../usage/configuration/config_documentation.md#sso) and [oidc_providers](../usage/configuration/config_documentation.md#oidc_providers).
-  - `auth_provider` - **string**, required. The unique, internal ID of the external identity provider.
-    The same as `idp_id` from the homeserver configuration. Note that no error is raised if the
-    provided value is not in the homeserver configuration.
-  - `external_id` - **string**, required. An identifier for the user in the external identity provider.
-    When the user logs in to the identity provider, this must be the unique ID that they map to.
-- `admin` - **bool**, optional, defaults to `false`. Whether the user is a homeserver administrator,
-  granting them access to the Admin API, among other things.
-- `deactivated` - **bool**, optional. If unspecified, deactivation state will be left unchanged.
+  - `auth_provider` - string. ID of the external identity provider. Value of `idp_id`
+    in the homeserver configuration. Note that no error is raised if the provided
+    value is not in the homeserver configuration.
+  - `external_id` - string, user ID in the external identity provider.
+- `avatar_url` - string, optional, must be a
+  [MXC URI](https://matrix.org/docs/spec/client_server/r0.6.0#matrix-content-mxc-uris).
+- `admin` - bool, optional, defaults to `false`.
+- `deactivated` - bool, optional. If unspecified, deactivation state will be left
+  unchanged on existing accounts and set to `false` for new accounts.
+  A user cannot be erased by deactivating with this API. For details on
+  deactivating users see [Deactivate Account](#deactivate-account).
+- `user_type` - string or null, optional. If provided, the user type will be
+  adjusted. If `null` given, the user type will be cleared. Other 
+  allowed options are: `bot` and `support`.
 
-  Note: the `password` field must also be set if both of the following are true:
-  - `deactivated` is set to `false` and the user was previously deactivated (you are reactivating this user)
-  - Users are allowed to set their password on this homeserver (both `password_config.enabled` and
-    `password_config.localdb_enabled` config options are set to `true`).
-  Users' passwords are wiped upon account deactivation, hence the need to set a new one here.
+If the user already exists then optional parameters default to the current value.
 
-  Note: a user cannot be erased with this API. For more details on
-  deactivating and erasing users see [Deactivate Account](#deactivate-account).
-- `user_type` - **string** or null, optional. If not provided, the user type will be
-  not be changed. If `null` is given, the user type will be cleared.
-  Other allowed options are: `bot` and `support`.
+In order to re-activate an account `deactivated` must be set to `false`. If
+users do not login via single-sign-on, a new `password` must be provided.
 
 ## List Accounts
 
@@ -242,9 +231,6 @@ The following parameters should be set in the URL:
 
 - `dir` - Direction of media order. Either `f` for forwards or `b` for backwards.
   Setting this value to `b` will reverse the above sort order. Defaults to `f`.
-- `not_user_type` - Exclude certain user types, such as bot users, from the request.
-   Can be provided multiple times. Possible values are `bot`, `support` or "empty string".
-   "empty string" here means to exclude users without a type.
 
 Caution. The database only has indexes on the columns `name` and `creation_ts`.
 This means that if a different sort order is used (`is_guest`, `admin`,
@@ -732,8 +718,7 @@ POST /_synapse/admin/v1/users/<user_id>/login
 
 An optional `valid_until_ms` field can be specified in the request body as an
 integer timestamp that specifies when the token should expire. By default tokens
-do not expire. Note that this API does not allow a user to login as themselves
-(to create more tokens).
+do not expire.
 
 A response body like the following is returned:
 
@@ -817,33 +802,6 @@ The following fields are returned in the JSON response body:
 
 - `total` - Total number of user's devices.
 
-### Create a device
-
-Creates a new device for a specific `user_id` and `device_id`. Does nothing if the `device_id` 
-exists already.
-
-The API is:
-
-```
-POST /_synapse/admin/v2/users/<user_id>/devices
-
-{
-  "device_id": "QBUAZIFURK"
-}
-```
-
-An empty JSON dict is returned.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- `user_id` - fully qualified: for example, `@user:server.com`.
-
-The following fields are required in the JSON request body:
-
-- `device_id` - The device ID to create.
-
 ### Delete multiple devices
 Deletes the given devices for a specific `user_id`, and invalidates
 any access token associated with them.
@@ -1184,7 +1142,7 @@ The following parameters should be set in the URL:
 - `user_id` - The fully qualified MXID: for example, `@user:server.com`. The user must
   be local.
 
-## Check username availability
+### Check username availability
 
 Checks to see if a username is available, and valid, for the server. See [the client-server 
 API](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available)
@@ -1202,7 +1160,7 @@ GET /_synapse/admin/v1/username_available?username=$localpart
 The request and response format is the same as the
 [/_matrix/client/r0/register/available](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available) API.
 
-## Find a user based on their ID in an auth provider
+### Find a user based on their ID in an auth provider
 
 The API is:
 
@@ -1241,7 +1199,7 @@ Returns a `404` HTTP status code if no user was found, with a response body like
 _Added in Synapse 1.68.0._
 
 
-## Find a user based on their Third Party ID (ThreePID or 3PID)
+### Find a user based on their Third Party ID (ThreePID or 3PID)
 
 The API is:
 

docs/deprecation_policy.md

@@ -23,7 +23,7 @@ people building from source should ensure they can fetch recent versions of Rust
 (e.g. by using [rustup](https://rustup.rs/)).
 
 The oldest supported version of SQLite is the version
-[provided](https://packages.debian.org/bullseye/libsqlite3-0) by
+[provided](https://packages.debian.org/buster/libsqlite3-0) by
 [Debian oldstable](https://wiki.debian.org/DebianOldStable).
 
 Context

docs/development/contributing_guide.md

@@ -22,9 +22,6 @@ on Windows is not officially supported.
 
 The code of Synapse is written in Python 3. To do pretty much anything, you'll need [a recent version of Python 3](https://www.python.org/downloads/). Your Python also needs support for [virtual environments](https://docs.python.org/3/library/venv.html). This is usually built-in, but some Linux distributions like Debian and Ubuntu split it out into its own package. Running `sudo apt install python3-venv` should be enough.
 
-A recent version of the Rust compiler is needed to build the native modules. The
-easiest way of installing the latest version is to use [rustup](https://rustup.rs/).
-
 Synapse can connect to PostgreSQL via the [psycopg2](https://pypi.org/project/psycopg2/) Python library. Building this library from source requires access to PostgreSQL's C header files. On Debian or Ubuntu Linux, these can be installed with `sudo apt install libpq-dev`.
 
 Synapse has an optional, improved user search with better Unicode support. For that you need the development package of `libicu`. On Debian or Ubuntu Linux, this can be installed with `sudo apt install libicu-dev`.
@@ -33,6 +30,9 @@ The source code of Synapse is hosted on GitHub. You will also need [a recent ver
 
 For some tests, you will need [a recent version of Docker](https://docs.docker.com/get-docker/).
 
+A recent version of the Rust compiler is needed to build the native modules. The
+easiest way of installing the latest version is to use [rustup](https://rustup.rs/).
+
 
 # 3. Get the source.
 
@@ -53,11 +53,6 @@ can find many good git tutorials on the web.
 
 # 4. Install the dependencies
 
-
-Before installing the Python dependencies, make sure you have installed a recent version
-of Rust (see the "What do I need?" section above). The easiest way of installing the
-latest version is to use [rustup](https://rustup.rs/).
-
 Synapse uses the [poetry](https://python-poetry.org/) project to manage its dependencies
 and development environment. Once you have installed Python 3 and added the
 source, you should install `poetry`.
@@ -81,8 +76,7 @@ cd path/where/you/have/cloned/the/repository
 poetry install --extras all
 ```
 
-This will install the runtime and developer dependencies for the project.  Be sure to check
-that the `poetry install` step completed cleanly.
+This will install the runtime and developer dependencies for the project.
 
 ## Running Synapse via poetry
 
@@ -90,31 +84,14 @@ To start a local instance of Synapse in the locked poetry environment, create a
 
 ```sh
 cp docs/sample_config.yaml homeserver.yaml
-cp docs/sample_log_config.yaml log_config.yaml
 ```
 
-Now edit `homeserver.yaml`, things you might want to change include:
-
-- Set a `server_name`
-- Adjusting paths to be correct for your system like the `log_config` to point to the log config you just copied
-- Using a [PostgreSQL database instead of SQLite](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#database)
-- Adding a [`registration_shared_secret`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#registration_shared_secret) so you can use [`register_new_matrix_user` command](https://matrix-org.github.io/synapse/latest/setup/installation.html#registering-a-user).
-
-And then run Synapse with the following command:
+Now edit homeserver.yaml, and run Synapse with:
 
 ```sh
 poetry run python -m synapse.app.homeserver -c homeserver.yaml
 ```
 
-If you get an error like the following:
-
-```
-importlib.metadata.PackageNotFoundError: matrix-synapse
-```
-
-this probably indicates that the `poetry install` step did not complete cleanly - go back and
-resolve any issues and re-run until successful.
-
 # 5. Get in touch.
 
 Join our developer community on Matrix: [#synapse-dev:matrix.org](https://matrix.to/#/#synapse-dev:matrix.org)!
@@ -322,7 +299,7 @@ The following command will let you run the integration test with the most common
 configuration:
 
 ```sh
-$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:focal
+$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:buster
 ```
 (Note that the paths must be full paths! You could also write `$(realpath relative/path)` if needed.)
 
@@ -369,8 +346,6 @@ The above will run a monolithic (single-process) Synapse with SQLite as the data
     A safe example would be `WORKER_TYPES="federation_inbound, federation_sender, synchrotron"`.
     See the [worker documentation](../workers.md) for additional information on workers.
 - Passing `ASYNCIO_REACTOR=1` as an environment variable to use the Twisted asyncio reactor instead of the default one.
-- Passing `PODMAN=1` will use the [podman](https://podman.io/) container runtime, instead of docker.
-- Passing `UNIX_SOCKETS=1` will utilise Unix socket functionality for Synapse, Redis, and Postgres(when applicable).
 
 To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`, e.g:
 ```sh