Fix import order

Adress comments
Merge remote-tracking branch 'origin/develop' into mv/unbind-callback
2022-12-13 10:25:19 +01:00 · 2022-12-12 23:31:09 +01:00 · 2022-12-12 17:48:27 +01:00 · 2022-11-28 12:57:35 +01:00 · 2022-08-03 16:24:44 +02:00 · 2022-08-03 14:52:20 +02:00
419 changed files with 6904 additions and 13272 deletions
@@ -50,16 +50,7 @@ def cpython(wheel_file: str, name: str, version: Version, tag: Tag) -> str:

    check_is_abi3_compatible(wheel_file)

-    # HACK: it seems that some older versions of pip will consider a wheel marked
-    # as macosx_11_0 as incompatible with Big Sur. I haven't done the full archaeology
-    # here; there are some clues in
-    #     https://github.com/pantsbuild/pants/pull/12857
-    #     https://github.com/pypa/pip/issues/9138
-    #     https://github.com/pypa/packaging/pull/319
-    # Empirically this seems to work, note that macOS 11 and 10.16 are the same,
-    # both versions are valid for backwards compatibility.
-    platform = tag.platform.replace("macosx_11_0", "macosx_10_16")
-    abi3_tag = Tag(tag.interpreter, "abi3", platform)
+    abi3_tag = Tag(tag.interpreter, "abi3", tag.platform)

    dirname = os.path.dirname(wheel_file)
    new_wheel_file = os.path.join(
@@ -1,23 +0,0 @@
-#! /usr/bin/env python
-import sys
-
-if sys.version_info < (3, 11):
-    raise RuntimeError("Requires at least Python 3.11, to import tomllib")
-
-import tomllib
-
-with open("poetry.lock", "rb") as f:
-    lockfile = tomllib.load(f)
-
-try:
-    lock_version = lockfile["metadata"]["lock-version"]
-    assert lock_version == "2.0"
-except Exception:
-    print(
-        """\
-    Lockfile is not version 2.0. You probably need to upgrade poetry on your local box
-    and re-run `poetry lock --no-update`. See the Poetry cheat sheet at
-    https://matrix-org.github.io/synapse/develop/development/dependencies.html
-    """
-    )
-    raise
@@ -53,7 +53,7 @@ with open('pyproject.toml', 'w') as f:
 "
 python3 -c "$REMOVE_DEV_DEPENDENCIES"

-pip install poetry==1.3.2
+pip install poetry==1.2.0
 poetry lock

 echo "::group::Patched pyproject.toml"
@@ -23,9 +23,8 @@ poetry run python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-dat
 --output-directory /tmp/export_data

 # Test that the output directory exists and contains the rooms directory
-dir_r="/tmp/export_data/rooms"
-dir_u="/tmp/export_data/user_data"
-if [ -d "$dir_r" ] && [ -d "$dir_u" ]; then
+dir="/tmp/export_data/rooms"
+if [ -d "$dir" ]; then
  echo "Command successful, this test passes"
 else
  echo "No output directories found, the command fails against a sqlite database."
@@ -44,9 +43,8 @@ poetry run python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-d
 --output-directory /tmp/export_data2

 # Test that the output directory exists and contains the rooms directory
-dir_r2="/tmp/export_data2/rooms"
-dir_u2="/tmp/export_data2/user_data"
-if [ -d "$dir_r2" ] && [ -d "$dir_u2" ]; then
+dir2="/tmp/export_data2/rooms"
+if [ -d "$dir2" ]; then
  echo "Command successful, this test passes"
 else
  echo "No output directories found, the command fails against a postgres database."
@@ -0,0 +1,18 @@
+# TODO: incorporate this into pyproject.toml if flake8 supports it in the future.
+# See https://github.com/PyCQA/flake8/issues/234
+[flake8]
+# see https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes
+# for error codes. The ones we ignore are:
+#  W503: line break before binary operator
+#  W504: line break after binary operator
+#  E203: whitespace before ':' (which is contrary to pep8?)
+#  E731: do not assign a lambda expression, use a def
+#  E501: Line too long (black enforces this for us)
+#
+# flake8-bugbear runs extra checks. Its error codes are described at
+# https://github.com/PyCQA/flake8-bugbear#list-of-warnings
+#  B019: Use of functools.lru_cache or functools.cache on methods can lead to memory leaks
+#  B023: Functions defined inside a loop must not use variables redefined in the loop
+#  B024: Abstract base class with no abstract method.
+
+ignore=W503,W504,E203,E731,E501,B019,B023,B024
@@ -6,7 +6,7 @@ on:
      - reopened  # For debugging!

 permissions:
-  # Needed to be able to push the commit. See
+  # Needed to be able to push the commit. See 
  #     https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#enable-auto-merge-on-a-pull-request
  # for a similar example
  contents: write
@@ -20,11 +20,8 @@ jobs:
        with:
          ref: ${{ github.event.pull_request.head.ref }}
      - name: Write, commit and push changelog
-        env:
-          PR_TITLE: ${{ github.event.pull_request.title }}
-          PR_NUMBER: ${{ github.event.pull_request.number }}
        run: |
-          echo "${PR_TITLE}." > "changelog.d/${PR_NUMBER}".misc
+          echo "${{ github.event.pull_request.title }}." > "changelog.d/${{ github.event.pull_request.number }}".misc
          git add changelog.d
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git config user.name "GitHub Actions"
@@ -48,7 +48,7 @@ jobs:
            type=pep440,pattern={{raw}}

      - name: Build and push all platforms
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v3
        with:
          push: true
          labels: "gitsha1=${{ github.sha }}"
@@ -14,7 +14,7 @@ jobs:
      # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
      # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
      - name: 📥 Download artifact
-        uses: dawidd6/action-download-artifact@bd10f381a96414ce2b13a11bfa89902ba7cea07f # v2.24.3
+        uses: dawidd6/action-download-artifact@e6e25ac3a2b93187502a8be1ef9e9603afc34925 # v2.24.2
        with:
          workflow: docs-pr.yaml
          run_id: ${{ github.event.workflow_run.id }}
@@ -4,8 +4,6 @@ on:
  pull_request:
    paths:
      - docs/**
-      - book.toml
-      - .github/workflows/docs-pr.yaml

 jobs:
  pages:
@@ -34,27 +32,3 @@ jobs:
          path: book
          # We'll only use this in a workflow_run, then we're done with it
          retention-days: 1
-
-  link-check:
-    name: Check links in documentation
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Setup mdbook
-        uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
-        with:
-          mdbook-version: '0.4.17'
-
-      - name: Setup htmltest
-        run: |
-          wget https://github.com/wjdp/htmltest/releases/download/v0.17.0/htmltest_0.17.0_linux_amd64.tar.gz
-          echo '775c597ee74899d6002cd2d93076f897f4ba68686bceabe2e5d72e84c57bc0fb  htmltest_0.17.0_linux_amd64.tar.gz' | sha256sum -c
-          tar zxf htmltest_0.17.0_linux_amd64.tar.gz
-
-      - name: Test links with htmltest
-        # Build the book with `./` as the site URL (to make checks on 404.html possible)
-        # Then run htmltest (without checking external links since that involves the network and is slow).
-        run: |
-          MDBOOK_OUTPUT__HTML__SITE_URL="./" mdbook build
-          ./htmltest book --skip-external
@@ -58,7 +58,7 @@ jobs:
          
      # Deploy to the target directory.
      - name: Deploy to gh pages
-        uses: peaceiris/actions-gh-pages@bd8c6b06eba6b3d25d72b7a1767993c0aeee42e7 # v3.9.2
+        uses: peaceiris/actions-gh-pages@de7ea6f8efb354206b205ef54722213d99067935 # v3.9.0
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./book
@@ -27,7 +27,7 @@ jobs:
    steps:
      - uses: actions/checkout@v3
      - name: Install Rust
-        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
+        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
        with:
          toolchain: stable
      - uses: Swatinem/rust-cache@v2
@@ -37,7 +37,7 @@ jobs:
      - uses: matrix-org/setup-python-poetry@v1
        with:
          python-version: "3.x"
-          poetry-version: "1.3.2"
+          poetry-version: "1.2.0"
          extras: "all"
      # Dump installed versions for debugging.
      - run: poetry run pip list > before.txt
@@ -61,7 +61,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
+        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
        with:
          toolchain: stable
      - uses: Swatinem/rust-cache@v2
@@ -134,7 +134,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
+        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
        with:
          toolchain: stable
      - uses: Swatinem/rust-cache@v2
@@ -208,7 +208,7 @@ jobs:

    steps:
      - uses: actions/checkout@v3
-      - uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd # v2.9.1
+      - uses: JasonEtco/create-an-issue@77399b6110ef82b94c1c9f9f615acf9e604f7f56 # v2.5.0, 2020-12-06
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
@@ -127,7 +127,7 @@ jobs:
          python-version: "3.x"

      - name: Install cibuildwheel
-        run: python -m pip install cibuildwheel==2.9.0
+        run: python -m pip install cibuildwheel==2.9.0 poetry==1.2.0

      - name: Set up QEMU to emulate aarch64
        if: matrix.arch == 'aarch64'
@@ -148,7 +148,7 @@ jobs:
        env:
          # Skip testing for platforms which various libraries don't have wheels
          # for, and so need extra build deps.
-          CIBW_TEST_SKIP: pp3*-* *i686* *musl*
+          CIBW_TEST_SKIP: pp39-* *i686* *musl* pp37-macosx*
          # Fix Rust OOM errors on emulated aarch64: https://github.com/rust-lang/cargo/issues/10583
          CARGO_NET_GIT_FETCH_WITH_CLI: true
          CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI
@@ -33,10 +33,11 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
-      - uses: matrix-org/setup-python-poetry@v1
+      - uses: actions/setup-python@v4
        with:
          python-version: "3.x"
-          poetry-version: "1.3.2"
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
          extras: "all"
      - run: poetry run scripts-dev/generate_sample_config.sh --check
      - run: poetry run scripts-dev/config-lint.sh
@@ -51,17 +52,8 @@ jobs:
      - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'"
      - run: scripts-dev/check_schema_delta.py --force-colors

-  check-lockfile:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.x"
-      - run: .ci/scripts/check_lockfile.py
-
  lint:
-    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v2"
+    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
    with:
      typechecking-extras: "all"

@@ -96,7 +88,6 @@ jobs:
          ref: ${{ github.event.pull_request.head.sha }}
      - uses: matrix-org/setup-python-poetry@v1
        with:
-          poetry-version: "1.3.2"
          extras: "all"
      - run: poetry run scripts-dev/check_pydantic_models.py

@@ -112,7 +103,7 @@ jobs:
        # There don't seem to be versioned releases of this action per se: for each rust
        # version there is a branch which gets constantly rebased on top of master.
        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
+        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
        with:
            toolchain: 1.58.1
            components: clippy
@@ -134,7 +125,7 @@ jobs:
        # There don't seem to be versioned releases of this action per se: for each rust
        # version there is a branch which gets constantly rebased on top of master.
        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
+        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
        with:
            toolchain: nightly-2022-12-01
            components: clippy
@@ -154,7 +145,7 @@ jobs:
        # There don't seem to be versioned releases of this action per se: for each rust
        # version there is a branch which gets constantly rebased on top of master.
        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
+        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
        with:
          toolchain: 1.58.1
          components: rustfmt
@@ -172,7 +163,6 @@ jobs:
      - lint-pydantic
      - check-sampleconfig
      - check-schema-delta
-      - check-lockfile
      - lint-clippy
      - lint-rustfmt
    runs-on: ubuntu-latest
@@ -221,7 +211,7 @@ jobs:
        # There don't seem to be versioned releases of this action per se: for each rust
        # version there is a branch which gets constantly rebased on top of master.
        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
+        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
        with:
            toolchain: 1.58.1
      - uses: Swatinem/rust-cache@v2
@@ -229,7 +219,6 @@ jobs:
      - uses: matrix-org/setup-python-poetry@v1
        with:
          python-version: ${{ matrix.job.python-version }}
-          poetry-version: "1.3.2"
          extras: ${{ matrix.job.extras }}
      - name: Await PostgreSQL
        if: ${{ matrix.job.postgres-version }}
@@ -266,7 +255,7 @@ jobs:
        # There don't seem to be versioned releases of this action per se: for each rust
        # version there is a branch which gets constantly rebased on top of master.
        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
+        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
        with:
            toolchain: 1.58.1
      - uses: Swatinem/rust-cache@v2
@@ -305,7 +294,6 @@ jobs:
      - uses: matrix-org/setup-python-poetry@v1
        with:
          python-version: '3.7'
-          poetry-version: "1.3.2"
          extras: "all test"

      - run: poetry run trial -j6 tests
@@ -340,7 +328,6 @@ jobs:
      - uses: matrix-org/setup-python-poetry@v1
        with:
          python-version: ${{ matrix.python-version }}
-          poetry-version: "1.3.2"
          extras: ${{ matrix.extras }}
      - run: poetry run trial --jobs=2 tests
      - name: Dump logs
@@ -386,7 +373,7 @@ jobs:
        # There don't seem to be versioned releases of this action per se: for each rust
        # version there is a branch which gets constantly rebased on top of master.
        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
+        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
        with:
            toolchain: 1.58.1
      - uses: Swatinem/rust-cache@v2
@@ -432,7 +419,6 @@ jobs:
      - run: sudo apt-get -qq install xmlsec1 postgresql-client
      - uses: matrix-org/setup-python-poetry@v1
        with:
-          poetry-version: "1.3.2"
          extras: "postgres"
      - run: .ci/scripts/test_export_data_command.sh
        env:
@@ -484,7 +470,6 @@ jobs:
      - uses: matrix-org/setup-python-poetry@v1
        with:
          python-version: ${{ matrix.python-version }}
-          poetry-version: "1.3.2"
          extras: "postgres"
      - run: .ci/scripts/test_synapse_port_db.sh
        id: run_tester_script
@@ -531,7 +516,7 @@ jobs:
        # There don't seem to be versioned releases of this action per se: for each rust
        # version there is a branch which gets constantly rebased on top of master.
        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
+        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
        with:
            toolchain: 1.58.1
      - uses: Swatinem/rust-cache@v2
@@ -541,11 +526,8 @@ jobs:

      - run: |
          set -o pipefail
-          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
+          POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
        shell: bash
-        env:
-          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
-          WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
        name: Run Complement Tests

  cargo-test:
@@ -562,36 +544,13 @@ jobs:
        # There don't seem to be versioned releases of this action per se: for each rust
        # version there is a branch which gets constantly rebased on top of master.
        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
+        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
        with:
            toolchain: 1.58.1
      - uses: Swatinem/rust-cache@v2

      - run: cargo test

-  # We want to ensure that the cargo benchmarks still compile, which requires a
-  # nightly compiler.
-  cargo-bench:
-    if: ${{ needs.changes.outputs.rust == 'true' }}
-    runs-on: ubuntu-latest
-    needs:
-      - linting-done
-      - changes
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
-        with:
-            toolchain: nightly-2022-12-01
-      - uses: Swatinem/rust-cache@v2
-
-      - run: cargo bench --no-run
-
  # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
  tests-done:
    if: ${{ always() }}
@@ -603,7 +562,6 @@ jobs:
      - portdb
      - complement
      - cargo-test
-      - cargo-bench
    runs-on: ubuntu-latest
    steps:
      - uses: matrix-org/done-action@v2
@@ -615,4 +573,3 @@ jobs:
          skippable: |
            lint-newsfile
            cargo-test
-            cargo-bench
@@ -18,7 +18,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
+        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
        with:
          toolchain: stable
      - uses: Swatinem/rust-cache@v2
@@ -43,7 +43,7 @@ jobs:
      - run: sudo apt-get -qq install xmlsec1

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
+        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
        with:
          toolchain: stable
      - uses: Swatinem/rust-cache@v2
@@ -82,7 +82,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
+        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
        with:
          toolchain: stable
      - uses: Swatinem/rust-cache@v2
@@ -148,7 +148,7 @@ jobs:
        run: |
          set -x
          DEBIAN_FRONTEND=noninteractive sudo apt-get install -yqq python3 pipx
-          pipx install poetry==1.3.2
+          pipx install poetry==1.2.0

          poetry remove -n twisted
          poetry add -n --extras tls git+https://github.com/twisted/twisted.git#trunk
@@ -174,7 +174,7 @@ jobs:

    steps:
      - uses: actions/checkout@v3
-      - uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd # v2.9.1
+      - uses: JasonEtco/create-an-issue@77399b6110ef82b94c1c9f9f615acf9e604f7f56 # v2.5.0, 2020-12-06
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
@@ -36,7 +36,6 @@ __pycache__/

 # For direnv users
 /.envrc
-.direnv/

 # IDEs
 /.idea/
@@ -69,6 +68,3 @@ book/

 # Poetry will create a setup.py, which we don't want to include.
 /setup.py
-
-# Don't include users' poetry configs
-/poetry.toml
@@ -1,412 +1,3 @@
-Synapse 1.77.0 (2023-02-14)
-===========================
-
-No significant changes since 1.77.0rc2.
-
-
-Synapse 1.77.0rc2 (2023-02-10)
-==============================
-
-Bugfixes
--------
-
- Fix bug where retried replication requests would return a failure. Introduced in v1.76.0. ([\#15024](https://github.com/matrix-org/synapse/issues/15024))
-
-
-Internal Changes
----------------
-
- Prepare for future database schema changes. ([\#15036](https://github.com/matrix-org/synapse/issues/15036))
-
-
-Synapse 1.77.0rc1 (2023-02-07)
-==============================
-
-Features
--------
-
- Experimental support for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952): intentional mentions. ([\#14823](https://github.com/matrix-org/synapse/issues/14823), [\#14943](https://github.com/matrix-org/synapse/issues/14943), [\#14957](https://github.com/matrix-org/synapse/issues/14957), [\#14958](https://github.com/matrix-org/synapse/issues/14958))
- Experimental support to suppress notifications from message edits ([MSC3958](https://github.com/matrix-org/matrix-spec-proposals/pull/3958)). ([\#14960](https://github.com/matrix-org/synapse/issues/14960), [\#15016](https://github.com/matrix-org/synapse/issues/15016))
- Add profile information, devices and connections to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.77/usage/administration/admin_faq.html#how-can-i-export-user-data). ([\#14894](https://github.com/matrix-org/synapse/issues/14894))
- Improve performance when joining or sending an event in large rooms. ([\#14962](https://github.com/matrix-org/synapse/issues/14962))
- Improve performance of joining and leaving large rooms with many local users. ([\#14971](https://github.com/matrix-org/synapse/issues/14971))
-
-
-Bugfixes
--------
-
- Fix a bug introduced in Synapse 1.53.0 where `next_batch` tokens from `/sync` could not be used with the `/relations` endpoint. ([\#14866](https://github.com/matrix-org/synapse/issues/14866))
- Fix a bug introduced in Synapse 1.35.0 where the module API's `send_local_online_presence_to` would fail to send presence updates over federation. ([\#14880](https://github.com/matrix-org/synapse/issues/14880))
- Fix a bug introduced in Synapse 1.70.0 where the background updates to add non-thread unique indexes on receipts could fail when upgrading from 1.67.0 or earlier. ([\#14915](https://github.com/matrix-org/synapse/issues/14915))
- Fix a regression introduced in Synapse 1.69.0 which can result in database corruption when database migrations are interrupted on sqlite. ([\#14926](https://github.com/matrix-org/synapse/issues/14926))
- Fix a bug introduced in Synapse 1.68.0 where we were unable to service remote joins in rooms with `@room` notification levels set to `null` in their (malformed) power levels. ([\#14942](https://github.com/matrix-org/synapse/issues/14942))
- Fix a bug introduced in Synapse 1.64.0 where boolean power levels were erroneously permitted in [v10 rooms](https://spec.matrix.org/v1.5/rooms/v10/). ([\#14944](https://github.com/matrix-org/synapse/issues/14944))
- Fix a long-standing bug where sending messages on servers with presence enabled would spam "Re-starting finished log context" log lines. ([\#14947](https://github.com/matrix-org/synapse/issues/14947))
- Fix a bug introduced in Synapse 1.68.0 where logging from the Rust module was not properly logged. ([\#14976](https://github.com/matrix-org/synapse/issues/14976))
- Fix various long-standing bugs in Synapse's config, event and request handling where booleans were unintentionally accepted where an integer was expected. ([\#14945](https://github.com/matrix-org/synapse/issues/14945))
-
-
-Internal Changes
----------------
-
- Add missing type hints. ([\#14879](https://github.com/matrix-org/synapse/issues/14879), [\#14886](https://github.com/matrix-org/synapse/issues/14886), [\#14887](https://github.com/matrix-org/synapse/issues/14887), [\#14904](https://github.com/matrix-org/synapse/issues/14904), [\#14927](https://github.com/matrix-org/synapse/issues/14927), [\#14956](https://github.com/matrix-org/synapse/issues/14956), [\#14983](https://github.com/matrix-org/synapse/issues/14983), [\#14984](https://github.com/matrix-org/synapse/issues/14984), [\#14985](https://github.com/matrix-org/synapse/issues/14985), [\#14987](https://github.com/matrix-org/synapse/issues/14987), [\#14988](https://github.com/matrix-org/synapse/issues/14988), [\#14990](https://github.com/matrix-org/synapse/issues/14990), [\#14991](https://github.com/matrix-org/synapse/issues/14991), [\#14992](https://github.com/matrix-org/synapse/issues/14992), [\#15007](https://github.com/matrix-org/synapse/issues/15007))
- Use `StrCollection` to avoid potential bugs with `Collection[str]`. ([\#14922](https://github.com/matrix-org/synapse/issues/14922))
- Allow running the complement tests suites with the asyncio reactor enabled. ([\#14858](https://github.com/matrix-org/synapse/issues/14858))
- Improve performance of `/sync` in a few situations. ([\#14908](https://github.com/matrix-org/synapse/issues/14908), [\#14970](https://github.com/matrix-org/synapse/issues/14970))
- Document how to handle Dependabot pull requests. ([\#14916](https://github.com/matrix-org/synapse/issues/14916))
- Fix typo in release script. ([\#14920](https://github.com/matrix-org/synapse/issues/14920))
- Update build system requirements to allow building with poetry-core 1.5.0. ([\#14949](https://github.com/matrix-org/synapse/issues/14949), [\#15019](https://github.com/matrix-org/synapse/issues/15019))
- Add an [lnav](https://lnav.org) config file for Synapse logs to `/contrib/lnav`. ([\#14953](https://github.com/matrix-org/synapse/issues/14953))
- Faster joins: Refactor internal handling of servers in room to never store an empty list. ([\#14954](https://github.com/matrix-org/synapse/issues/14954))
- Faster joins: tag `v2/send_join/` requests to indicate if they served a partial join response. ([\#14950](https://github.com/matrix-org/synapse/issues/14950))
- Allow running `cargo` without the `extension-module` option. ([\#14965](https://github.com/matrix-org/synapse/issues/14965))
- Preparatory work for adding a denormalised event stream ordering column in the future. Contributed by Nick @ Beeper (@fizzadar). ([\#14979](https://github.com/matrix-org/synapse/issues/14979), [9cd7610](https://github.com/matrix-org/synapse/commit/9cd7610f86ab5051c9365dd38d1eec405a5f8ca6), [f10caa7](https://github.com/matrix-org/synapse/commit/f10caa73eee0caa91cf373966104d1ededae2aee); see [\#15014](https://github.com/matrix-org/synapse/issues/15014))
- Add tests for `_flatten_dict`. ([\#14981](https://github.com/matrix-org/synapse/issues/14981), [\#15002](https://github.com/matrix-org/synapse/issues/15002))
-
-<details><summary>Dependabot updates</summary>
-
- Bump dtolnay/rust-toolchain from e645b0cf01249a964ec099494d38d2da0f0b349f to 9cd00a88a73addc8617065438eff914dd08d0955. ([\#14968](https://github.com/matrix-org/synapse/issues/14968))
- Bump docker/build-push-action from 3 to 4. ([\#14952](https://github.com/matrix-org/synapse/issues/14952))
- Bump ijson from 3.1.4 to 3.2.0.post0. ([\#14935](https://github.com/matrix-org/synapse/issues/14935))
- Bump types-pyyaml from 6.0.12.2 to 6.0.12.3. ([\#14936](https://github.com/matrix-org/synapse/issues/14936))
- Bump types-jsonschema from 4.17.0.2 to 4.17.0.3. ([\#14937](https://github.com/matrix-org/synapse/issues/14937))
- Bump types-pillow from 9.4.0.3 to 9.4.0.5. ([\#14938](https://github.com/matrix-org/synapse/issues/14938))
- Bump hiredis from 2.0.0 to 2.1.1. ([\#14939](https://github.com/matrix-org/synapse/issues/14939))
- Bump hiredis from 2.1.1 to 2.2.1. ([\#14993](https://github.com/matrix-org/synapse/issues/14993))
- Bump types-setuptools from 65.6.0.3 to 67.1.0.0. ([\#14994](https://github.com/matrix-org/synapse/issues/14994))
- Bump prometheus-client from 0.15.0 to 0.16.0. ([\#14995](https://github.com/matrix-org/synapse/issues/14995))
- Bump anyhow from 1.0.68 to 1.0.69. ([\#14996](https://github.com/matrix-org/synapse/issues/14996))
- Bump serde_json from 1.0.91 to 1.0.92. ([\#14997](https://github.com/matrix-org/synapse/issues/14997))
- Bump isort from 5.11.4 to 5.11.5. ([\#14998](https://github.com/matrix-org/synapse/issues/14998))
- Bump phonenumbers from 8.13.4 to 8.13.5. ([\#14999](https://github.com/matrix-org/synapse/issues/14999))
-</details>
-
-Synapse 1.76.0 (2023-01-31)
-===========================
-
-The 1.76 release is the first to enable faster joins ([MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706) and [MSC3902](https://github.com/matrix-org/matrix-spec-proposals/pull/3902)) by default. Admins can opt-out: see [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.76/docs/upgrade.md#faster-joins-are-enabled-by-default) for more details.
-
-The upgrade from 1.75 to 1.76 changes the account data replication streams in a backwards-incompatible manner. Server operators running a multi-worker deployment should consult [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.76/docs/upgrade.md#changes-to-the-account-data-replication-streams).
-
-Those who are `poetry install`ing from source using our lockfile should ensure their poetry version is 1.3.2 or higher; [see upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.76/docs/upgrade.md#minimum-version-of-poetry-is-now-132).
-
-
-Notes on faster joins
---------------------
-
-The faster joins project sees the most benefit when joining a room with a large number of members (joined or historical). We expect it to be particularly useful for joining large public rooms like the [Matrix HQ](https://matrix.to/#/#matrix:matrix.org) or [Synapse Admins](https://matrix.to/#/#synapse:matrix.org) rooms. 
-
-After a faster join, Synapse considers that room "partially joined". In this state, you should be able to
-
- read incoming messages;
- see incoming state changes, e.g. room topic changes; and
- send messages, if the room is unencrypted.
-
-Synapse has to spend more effort to complete the join in the background. Once this finishes, you will be able to
-
- send messages, if the room is in encrypted;
- retrieve room history from before your join, if permitted by the room settings; and
- access the full list of room members.
-
-
-Improved Documentation
----------------------
-
- Describe the ideas and the internal machinery behind faster joins. ([\#14677](https://github.com/matrix-org/synapse/issues/14677))
-
-
-Synapse 1.76.0rc2 (2023-01-27)
-==============================
-
-Bugfixes
--------
-
- Faster joins: Fix a bug introduced in Synapse 1.69 where device list EDUs could fail to be handled after a restart when a faster join sync is in progress. ([\#14914](https://github.com/matrix-org/synapse/issues/14914))
-
-
-Internal Changes
----------------
-
- Faster joins: Improve performance of looking up partial-state status of rooms. ([\#14917](https://github.com/matrix-org/synapse/issues/14917))
-
-
-Synapse 1.76.0rc1 (2023-01-25)
-==============================
-
-Features
--------
-
- Update the default room version to [v10](https://spec.matrix.org/v1.5/rooms/v10/) ([MSC 3904](https://github.com/matrix-org/matrix-spec-proposals/pull/3904)). Contributed by @FSG-Cat. ([\#14111](https://github.com/matrix-org/synapse/issues/14111))
- Add a `set_displayname()` method to the module API for setting a user's display name. ([\#14629](https://github.com/matrix-org/synapse/issues/14629))
- Add a dedicated listener configuration for `health` endpoint. ([\#14747](https://github.com/matrix-org/synapse/issues/14747))
- Implement support for [MSC3890](https://github.com/matrix-org/matrix-spec-proposals/pull/3890): Remotely silence local notifications. ([\#14775](https://github.com/matrix-org/synapse/issues/14775))
- Implement experimental support for [MSC3930](https://github.com/matrix-org/matrix-spec-proposals/pull/3930): Push rules for ([MSC3381](https://github.com/matrix-org/matrix-spec-proposals/pull/3381)) Polls. ([\#14787](https://github.com/matrix-org/synapse/issues/14787))
- Per [MSC3925](https://github.com/matrix-org/matrix-spec-proposals/pull/3925), bundle the whole of the replacement with any edited events, and optionally inhibit server-side replacement. ([\#14811](https://github.com/matrix-org/synapse/issues/14811))
- Faster joins: always serve a partial join response to servers that request it with the stable query param. ([\#14839](https://github.com/matrix-org/synapse/issues/14839))
- Faster joins: allow non-lazy-loading ("eager") syncs to complete after a partial join by omitting partial state rooms until they become fully stated. ([\#14870](https://github.com/matrix-org/synapse/issues/14870))
- Faster joins: request partial joins by default. Admins can opt-out of this for the time being---see the upgrade notes. ([\#14905](https://github.com/matrix-org/synapse/issues/14905))
-
-
-Bugfixes
--------
-
- Add index to improve performance of the `/timestamp_to_event` endpoint used for jumping to a specific date in the timeline of a room. ([\#14799](https://github.com/matrix-org/synapse/issues/14799))
- Fix a long-standing bug where Synapse would exhaust the stack when processing many federation requests where the remote homeserver has disconencted early. ([\#14812](https://github.com/matrix-org/synapse/issues/14812), [\#14842](https://github.com/matrix-org/synapse/issues/14842))
- Fix rare races when using workers. ([\#14820](https://github.com/matrix-org/synapse/issues/14820))
- Fix a bug introduced in Synapse 1.64.0 when using room version 10 with frozen events enabled. ([\#14864](https://github.com/matrix-org/synapse/issues/14864))
- Fix a long-standing bug where the `populate_room_stats` background job could fail on broken rooms. ([\#14873](https://github.com/matrix-org/synapse/issues/14873))
- Faster joins: Fix a bug in worker deployments where the room stats and user directory would not get updated when finishing a fast join until another event is sent or received. ([\#14874](https://github.com/matrix-org/synapse/issues/14874))
- Faster joins: Fix incompatibility with joins into restricted rooms where no local users have the ability to invite. ([\#14882](https://github.com/matrix-org/synapse/issues/14882))
- Fix a regression introduced in Synapse 1.69.0 which can result in database corruption when database migrations are interrupted on sqlite. ([\#14910](https://github.com/matrix-org/synapse/issues/14910))
-
-
-Updates to the Docker image
---------------------------
-
- Bump default Python version in the Dockerfile from 3.9 to 3.11. ([\#14875](https://github.com/matrix-org/synapse/issues/14875))
-
-
-Improved Documentation
----------------------
-
- Include `x_forwarded` entry in the HTTP listener example configs and remove the remaining `worker_main_http_uri` entries. ([\#14667](https://github.com/matrix-org/synapse/issues/14667))
- Remove duplicate commands from the Code Style documentation page; point to the Contributing Guide instead. ([\#14773](https://github.com/matrix-org/synapse/issues/14773))
- Add missing documentation for `tag` to `listeners` section. ([\#14803](https://github.com/matrix-org/synapse/issues/14803))
- Updated documentation in configuration manual for `user_directory.search_all_users`. ([\#14818](https://github.com/matrix-org/synapse/issues/14818))
- Add `worker_manhole` to configuration manual. ([\#14824](https://github.com/matrix-org/synapse/issues/14824))
- Fix the example config missing the `id` field in [application service documentation](https://matrix-org.github.io/synapse/latest/application_services.html). ([\#14845](https://github.com/matrix-org/synapse/issues/14845))
- Minor corrections to the logging configuration documentation. ([\#14868](https://github.com/matrix-org/synapse/issues/14868))
- Document the export user data command. Contributed by @thezaidbintariq. ([\#14883](https://github.com/matrix-org/synapse/issues/14883))
-
-
-Deprecations and Removals
-------------------------
-
- Poetry 1.3.2 or higher is now required when `poetry install`ing from source. ([\#14860](https://github.com/matrix-org/synapse/issues/14860))
-
-
-Internal Changes
----------------
-
- Faster remote room joins (worker mode): do not populate external hosts-in-room cache when sending events as this requires blocking for full state. ([\#14749](https://github.com/matrix-org/synapse/issues/14749))
- Enable Complement tests for Faster Remote Room Joins against worker-mode Synapse. ([\#14752](https://github.com/matrix-org/synapse/issues/14752))
- Add some clarifying comments and refactor a portion of the `Keyring` class for readability. ([\#14804](https://github.com/matrix-org/synapse/issues/14804))
- Add local poetry config files (`poetry.toml`) to `.gitignore`. ([\#14807](https://github.com/matrix-org/synapse/issues/14807))
- Add missing type hints. ([\#14816](https://github.com/matrix-org/synapse/issues/14816), [\#14885](https://github.com/matrix-org/synapse/issues/14885), [\#14889](https://github.com/matrix-org/synapse/issues/14889))
- Refactor push tests. ([\#14819](https://github.com/matrix-org/synapse/issues/14819))
- Re-enable some linting that was disabled when we switched to ruff. ([\#14821](https://github.com/matrix-org/synapse/issues/14821))
- Add `cargo fmt` and `cargo clippy` to the lint script. ([\#14822](https://github.com/matrix-org/synapse/issues/14822))
- Drop unused table `presence`. ([\#14825](https://github.com/matrix-org/synapse/issues/14825))
- Merge the two account data and the two device list replication streams. ([\#14826](https://github.com/matrix-org/synapse/issues/14826), [\#14833](https://github.com/matrix-org/synapse/issues/14833))
- Faster joins: use stable identifiers from [MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706). ([\#14832](https://github.com/matrix-org/synapse/issues/14832), [\#14841](https://github.com/matrix-org/synapse/issues/14841))
- Add a parameter to control whether the federation client performs a partial state join. ([\#14843](https://github.com/matrix-org/synapse/issues/14843))
- Add check to avoid starting duplicate partial state syncs. ([\#14844](https://github.com/matrix-org/synapse/issues/14844))
- Add an early return when handling no-op presence updates. ([\#14855](https://github.com/matrix-org/synapse/issues/14855))
- Fix `wait_for_stream_position` to correctly wait for the right instance to advance its token. ([\#14856](https://github.com/matrix-org/synapse/issues/14856), [\#14872](https://github.com/matrix-org/synapse/issues/14872))
- Always notify replication when a stream advances automatically. ([\#14877](https://github.com/matrix-org/synapse/issues/14877))
- Reduce max time we wait for stream positions. ([\#14881](https://github.com/matrix-org/synapse/issues/14881))
- Faster joins: allow the resync process more time to fetch `/state` ids. ([\#14912](https://github.com/matrix-org/synapse/issues/14912))
- Bump regex from 1.7.0 to 1.7.1. ([\#14848](https://github.com/matrix-org/synapse/issues/14848))
- Bump peaceiris/actions-gh-pages from 3.9.1 to 3.9.2. ([\#14861](https://github.com/matrix-org/synapse/issues/14861))
- Bump ruff from 0.0.215 to 0.0.224. ([\#14862](https://github.com/matrix-org/synapse/issues/14862))
- Bump types-pillow from 9.4.0.0 to 9.4.0.3. ([\#14863](https://github.com/matrix-org/synapse/issues/14863))
- Bump types-opentracing from 2.4.10 to 2.4.10.1. ([\#14896](https://github.com/matrix-org/synapse/issues/14896))
- Bump ruff from 0.0.224 to 0.0.230. ([\#14897](https://github.com/matrix-org/synapse/issues/14897))
- Bump types-requests from 2.28.11.7 to 2.28.11.8. ([\#14899](https://github.com/matrix-org/synapse/issues/14899))
- Bump types-psycopg2 from 2.9.21.2 to 2.9.21.4. ([\#14900](https://github.com/matrix-org/synapse/issues/14900))
- Bump types-commonmark from 0.9.2 to 0.9.2.1. ([\#14901](https://github.com/matrix-org/synapse/issues/14901))
-
-
-Synapse 1.75.0 (2023-01-17)
-===========================
-
-No significant changes since 1.75.0rc2.
-
-
-Synapse 1.75.0rc2 (2023-01-12)
-==============================
-
-Bugfixes
--------
-
- Fix a bug introduced in Synapse 1.75.0rc1 where device lists could be miscalculated with some sync filters. ([\#14810](https://github.com/matrix-org/synapse/issues/14810))
- Fix race where calling `/members` or `/state` with an `at` parameter could fail for newly created rooms, when using multiple workers. ([\#14817](https://github.com/matrix-org/synapse/issues/14817))
-
-
-Synapse 1.75.0rc1 (2023-01-10)
-==============================
-
-Features
--------
-
- Add a `cached` function to `synapse.module_api` that returns a decorator to cache return values of functions. ([\#14663](https://github.com/matrix-org/synapse/issues/14663))
- Add experimental support for [MSC3391](https://github.com/matrix-org/matrix-spec-proposals/pull/3391) (removing account data). ([\#14714](https://github.com/matrix-org/synapse/issues/14714))
- Support [RFC7636](https://datatracker.ietf.org/doc/html/rfc7636) Proof Key for Code Exchange for OAuth single sign-on. ([\#14750](https://github.com/matrix-org/synapse/issues/14750))
- Support non-OpenID compliant userinfo claims for subject and picture. ([\#14753](https://github.com/matrix-org/synapse/issues/14753))
- Improve performance of `/sync` when filtering all rooms, message types, or senders. ([\#14786](https://github.com/matrix-org/synapse/issues/14786))
- Improve performance of the `/hierarchy` endpoint. ([\#14263](https://github.com/matrix-org/synapse/issues/14263))
-
-
-Bugfixes
--------
-
- Fix the *MAU Limits* section of the Grafana dashboard relying on a specific `job` name for the workers of a Synapse deployment. ([\#14644](https://github.com/matrix-org/synapse/issues/14644))
- Fix a bug introduced in Synapse 1.70.0 which could cause spurious `UNIQUE constraint failed` errors in the `rotate_notifs` background job. ([\#14669](https://github.com/matrix-org/synapse/issues/14669))
- Ensure stream IDs are always updated after caches get invalidated with workers. Contributed by Nick @ Beeper (@fizzadar). ([\#14723](https://github.com/matrix-org/synapse/issues/14723))
- Remove the unspecced `device` field from `/pushrules` responses. ([\#14727](https://github.com/matrix-org/synapse/issues/14727))
- Fix a bug introduced in Synapse 1.73.0 where the `picture_claim` configured under `oidc_providers` was unused (the default value of `"picture"` was used instead). ([\#14751](https://github.com/matrix-org/synapse/issues/14751))
- Unescape HTML entities in URL preview titles making use of oEmbed responses. ([\#14781](https://github.com/matrix-org/synapse/issues/14781))
- Disable sending confirmation email when 3pid is disabled. ([\#14725](https://github.com/matrix-org/synapse/issues/14725))
-
-
-Improved Documentation
----------------------
-
- Declare support for Python 3.11. ([\#14673](https://github.com/matrix-org/synapse/issues/14673))
- Fix `target_memory_usage` being used in the description for the actual `cache_autotune` sub-option `target_cache_memory_usage`. ([\#14674](https://github.com/matrix-org/synapse/issues/14674))
- Move `email` to Server section in config file documentation. ([\#14730](https://github.com/matrix-org/synapse/issues/14730))
- Fix broken links in the Synapse documentation. ([\#14744](https://github.com/matrix-org/synapse/issues/14744))
- Add missing worker settings to shared configuration documentation. ([\#14748](https://github.com/matrix-org/synapse/issues/14748))
- Document using Twitter as a OAuth 2.0 authentication provider. ([\#14778](https://github.com/matrix-org/synapse/issues/14778))
- Fix Synapse 1.74 upgrade notes to correctly explain how to install pyICU when installing Synapse from PyPI. ([\#14797](https://github.com/matrix-org/synapse/issues/14797))
- Update link to towncrier in contribution guide. ([\#14801](https://github.com/matrix-org/synapse/issues/14801))
- Use `htmltest` to check links in the Synapse documentation. ([\#14743](https://github.com/matrix-org/synapse/issues/14743))
-
-
-Internal Changes
----------------
-
- Faster remote room joins: stream the un-partial-stating of events over replication. ([\#14545](https://github.com/matrix-org/synapse/issues/14545), [\#14546](https://github.com/matrix-org/synapse/issues/14546))
- Use [ruff](https://github.com/charliermarsh/ruff/) instead of flake8. ([\#14633](https://github.com/matrix-org/synapse/issues/14633), [\#14741](https://github.com/matrix-org/synapse/issues/14741))
- Change `handle_new_client_event` signature so that a 429 does not reach clients on `PartialStateConflictError`, and internally retry when needed instead. ([\#14665](https://github.com/matrix-org/synapse/issues/14665))
- Remove dependency on jQuery on reCAPTCHA page. ([\#14672](https://github.com/matrix-org/synapse/issues/14672))
- Faster joins: make `compute_state_after_events` consistent with other state-fetching functions that take a `StateFilter`. ([\#14676](https://github.com/matrix-org/synapse/issues/14676))
- Add missing type hints. ([\#14680](https://github.com/matrix-org/synapse/issues/14680), [\#14681](https://github.com/matrix-org/synapse/issues/14681), [\#14687](https://github.com/matrix-org/synapse/issues/14687))
- Improve type annotations for the helper methods on a `CachedFunction`. ([\#14685](https://github.com/matrix-org/synapse/issues/14685))
- Check that the SQLite database file exists before porting to PostgreSQL. ([\#14692](https://github.com/matrix-org/synapse/issues/14692))
- Add `.direnv/` directory to .gitignore to prevent local state generated by the [direnv](https://direnv.net/) development tool from being committed. ([\#14707](https://github.com/matrix-org/synapse/issues/14707))
- Batch up replication requests to request the resyncing of remote users's devices. ([\#14716](https://github.com/matrix-org/synapse/issues/14716))
- If debug logging is enabled, log the `msgid`s of any to-device messages that are returned over `/sync`. ([\#14724](https://github.com/matrix-org/synapse/issues/14724))
- Change GHA CI job to follow best practices. ([\#14772](https://github.com/matrix-org/synapse/issues/14772))
- Switch to our fork of `dh-virtualenv` to work around an upstream Python 3.11 incompatibility. ([\#14774](https://github.com/matrix-org/synapse/issues/14774))
- Skip testing built wheels for PyPy 3.7 on Linux x86_64 as we lack new required dependencies in the build environment. ([\#14802](https://github.com/matrix-org/synapse/issues/14802))
-
-### Dependabot updates
-
-<details>
-
- Bump JasonEtco/create-an-issue from 2.8.1 to 2.8.2. ([\#14693](https://github.com/matrix-org/synapse/issues/14693))
- Bump anyhow from 1.0.66 to 1.0.68. ([\#14694](https://github.com/matrix-org/synapse/issues/14694))
- Bump blake2 from 0.10.5 to 0.10.6. ([\#14695](https://github.com/matrix-org/synapse/issues/14695))
- Bump serde_json from 1.0.89 to 1.0.91. ([\#14696](https://github.com/matrix-org/synapse/issues/14696))
- Bump serde from 1.0.150 to 1.0.151. ([\#14697](https://github.com/matrix-org/synapse/issues/14697))
- Bump lxml from 4.9.1 to 4.9.2. ([\#14698](https://github.com/matrix-org/synapse/issues/14698))
- Bump types-jsonschema from 4.17.0.1 to 4.17.0.2. ([\#14700](https://github.com/matrix-org/synapse/issues/14700))
- Bump sentry-sdk from 1.11.1 to 1.12.0. ([\#14701](https://github.com/matrix-org/synapse/issues/14701))
- Bump types-setuptools from 65.6.0.1 to 65.6.0.2. ([\#14702](https://github.com/matrix-org/synapse/issues/14702))
- Bump minimum PyYAML to 3.13. ([\#14720](https://github.com/matrix-org/synapse/issues/14720))
- Bump JasonEtco/create-an-issue from 2.8.2 to 2.9.1. ([\#14731](https://github.com/matrix-org/synapse/issues/14731))
- Bump towncrier from 22.8.0 to 22.12.0. ([\#14732](https://github.com/matrix-org/synapse/issues/14732))
- Bump isort from 5.10.1 to 5.11.4. ([\#14733](https://github.com/matrix-org/synapse/issues/14733))
- Bump attrs from 22.1.0 to 22.2.0. ([\#14734](https://github.com/matrix-org/synapse/issues/14734))
- Bump black from 22.10.0 to 22.12.0. ([\#14735](https://github.com/matrix-org/synapse/issues/14735))
- Bump sentry-sdk from 1.12.0 to 1.12.1. ([\#14736](https://github.com/matrix-org/synapse/issues/14736))
- Bump setuptools from 65.3.0 to 65.5.1. ([\#14738](https://github.com/matrix-org/synapse/issues/14738))
- Bump serde from 1.0.151 to 1.0.152. ([\#14758](https://github.com/matrix-org/synapse/issues/14758))
- Bump ruff from 0.0.189 to 0.0.206. ([\#14759](https://github.com/matrix-org/synapse/issues/14759))
- Bump pydantic from 1.10.2 to 1.10.4. ([\#14760](https://github.com/matrix-org/synapse/issues/14760))
- Bump gitpython from 3.1.29 to 3.1.30. ([\#14761](https://github.com/matrix-org/synapse/issues/14761))
- Bump pillow from 9.3.0 to 9.4.0. ([\#14762](https://github.com/matrix-org/synapse/issues/14762))
- Bump types-requests from 2.28.11.5 to 2.28.11.7. ([\#14763](https://github.com/matrix-org/synapse/issues/14763))
- Bump dawidd6/action-download-artifact from 2.24.2 to 2.24.3. ([\#14779](https://github.com/matrix-org/synapse/issues/14779))
- Bump peaceiris/actions-gh-pages from 3.9.0 to 3.9.1. ([\#14791](https://github.com/matrix-org/synapse/issues/14791))
- Bump types-pillow from 9.3.0.4 to 9.4.0.0. ([\#14792](https://github.com/matrix-org/synapse/issues/14792))
- Bump pyopenssl from 22.1.0 to 23.0.0. ([\#14793](https://github.com/matrix-org/synapse/issues/14793))
- Bump types-setuptools from 65.6.0.2 to 65.6.0.3. ([\#14794](https://github.com/matrix-org/synapse/issues/14794))
- Bump importlib-metadata from 4.2.0 to 6.0.0. ([\#14795](https://github.com/matrix-org/synapse/issues/14795))
- Bump ruff from 0.0.206 to 0.0.215. ([\#14796](https://github.com/matrix-org/synapse/issues/14796))
-</details>
-
-Synapse 1.74.0 (2022-12-20)
-===========================
-
-Improved Documentation
----------------------
-
- Add release note and update documentation regarding optional ICU support in user search. ([\#14712](https://github.com/matrix-org/synapse/issues/14712))
-
-
-Synapse 1.74.0rc1 (2022-12-13)
-==============================
-
-Features
--------
-
- Improve user search for international display names. ([\#14464](https://github.com/matrix-org/synapse/issues/14464))
- Stop using deprecated `keyIds` parameter when calling `/_matrix/key/v2/server`. ([\#14490](https://github.com/matrix-org/synapse/issues/14490), [\#14525](https://github.com/matrix-org/synapse/issues/14525))
- Add new `push.enabled` config option to allow opting out of push notification calculation. ([\#14551](https://github.com/matrix-org/synapse/issues/14551), [\#14619](https://github.com/matrix-org/synapse/issues/14619))
- Advertise support for Matrix 1.5 on `/_matrix/client/versions`. ([\#14576](https://github.com/matrix-org/synapse/issues/14576))
- Improve opentracing and logging for to-device message handling. ([\#14598](https://github.com/matrix-org/synapse/issues/14598))
- Allow selecting "prejoin" events by state keys in addition to event types. ([\#14642](https://github.com/matrix-org/synapse/issues/14642))
-
-
-Bugfixes
--------
-
- Fix a long-standing bug where a device list update might not be sent to clients in certain circumstances. ([\#14435](https://github.com/matrix-org/synapse/issues/14435), [\#14592](https://github.com/matrix-org/synapse/issues/14592), [\#14604](https://github.com/matrix-org/synapse/issues/14604))
- Suppress a spurious warning when `POST /rooms/<room_id>/<membership>/`, `POST /join/<room_id_or_alias`, or the unspecced `PUT /join/<room_id_or_alias>/<txn_id>` receive an empty HTTP request body. ([\#14600](https://github.com/matrix-org/synapse/issues/14600))
- Return spec-compliant JSON errors when unknown endpoints are requested. ([\#14620](https://github.com/matrix-org/synapse/issues/14620), [\#14621](https://github.com/matrix-org/synapse/issues/14621))
- Update html templates to load images over HTTPS. Contributed by @ashfame. ([\#14625](https://github.com/matrix-org/synapse/issues/14625))
- Fix a long-standing bug where the user directory would return 1 more row than requested. ([\#14631](https://github.com/matrix-org/synapse/issues/14631))
- Reject invalid read receipt requests with empty room or event IDs. Contributed by Nick @ Beeper (@fizzadar). ([\#14632](https://github.com/matrix-org/synapse/issues/14632))
- Fix a bug introduced in Synapse 1.67.0 where not specifying a config file or a server URL would lead to the `register_new_matrix_user` script failing. ([\#14637](https://github.com/matrix-org/synapse/issues/14637))
- Fix a long-standing bug where the user directory and room/user stats might be out of sync. ([\#14639](https://github.com/matrix-org/synapse/issues/14639), [\#14643](https://github.com/matrix-org/synapse/issues/14643))
- Fix a bug introduced in Synapse 1.72.0 where the background updates to add non-thread unique indexes on receipts would fail if they were previously interrupted. ([\#14650](https://github.com/matrix-org/synapse/issues/14650))
- Improve validation of field size limits in events. ([\#14664](https://github.com/matrix-org/synapse/issues/14664))
- Fix bugs introduced in Synapse 1.55.0 and 1.69.0 where application services would not be notified of events in the correct rooms, due to stale caches. ([\#14670](https://github.com/matrix-org/synapse/issues/14670))
-
-
-Improved Documentation
----------------------
-
- Update worker settings for `pusher` and `federation_sender` functionality. ([\#14493](https://github.com/matrix-org/synapse/issues/14493))
- Add links to third party package repositories, and point to the bug which highlights Ubuntu's out-of-date packages. ([\#14517](https://github.com/matrix-org/synapse/issues/14517))
- Remove old, incorrect minimum postgres version note and replace with a link to the [Dependency Deprecation Policy](https://matrix-org.github.io/synapse/v1.73/deprecation_policy.html). ([\#14590](https://github.com/matrix-org/synapse/issues/14590))
- Add Single-Sign On setup instructions for Mastodon-based instances. ([\#14594](https://github.com/matrix-org/synapse/issues/14594))
- Change `turn_allow_guests` example value to lowercase `true`. ([\#14634](https://github.com/matrix-org/synapse/issues/14634))
-
-
-Internal Changes
----------------
-
- Optimise push badge count calculations. Contributed by Nick @ Beeper (@fizzadar). ([\#14255](https://github.com/matrix-org/synapse/issues/14255))
- Faster remote room joins: stream the un-partial-stating of rooms over replication. ([\#14473](https://github.com/matrix-org/synapse/issues/14473), [\#14474](https://github.com/matrix-org/synapse/issues/14474))
- Share the `ClientRestResource` for both workers and the main process. ([\#14528](https://github.com/matrix-org/synapse/issues/14528))
- Add `--editable` flag to `complement.sh` which uses an editable install of Synapse for faster turn-around times whilst developing iteratively. ([\#14548](https://github.com/matrix-org/synapse/issues/14548))
- Faster joins: use servers list approximation to send read receipts when in partial state instead of waiting for the full state of the room. ([\#14549](https://github.com/matrix-org/synapse/issues/14549))
- Modernize unit tests configuration related to workers. ([\#14568](https://github.com/matrix-org/synapse/issues/14568))
- Bump jsonschema from 4.17.0 to 4.17.3. ([\#14591](https://github.com/matrix-org/synapse/issues/14591))
- Fix Rust lint CI. ([\#14602](https://github.com/matrix-org/synapse/issues/14602))
- Bump JasonEtco/create-an-issue from 2.5.0 to 2.8.1. ([\#14607](https://github.com/matrix-org/synapse/issues/14607))
- Alter some unit test environment parameters to decrease time spent running tests. ([\#14610](https://github.com/matrix-org/synapse/issues/14610))
- Switch to Go recommended installation method for `gotestfmt` template in CI. ([\#14611](https://github.com/matrix-org/synapse/issues/14611))
- Bump phonenumbers from 8.13.0 to 8.13.1. ([\#14612](https://github.com/matrix-org/synapse/issues/14612))
- Bump types-setuptools from 65.5.0.3 to 65.6.0.1. ([\#14613](https://github.com/matrix-org/synapse/issues/14613))
- Bump twine from 4.0.1 to 4.0.2. ([\#14614](https://github.com/matrix-org/synapse/issues/14614))
- Bump types-requests from 2.28.11.2 to 2.28.11.5. ([\#14615](https://github.com/matrix-org/synapse/issues/14615))
- Bump cryptography from 38.0.3 to 38.0.4. ([\#14616](https://github.com/matrix-org/synapse/issues/14616))
- Remove useless cargo install with apt from Dockerfile. ([\#14636](https://github.com/matrix-org/synapse/issues/14636))
- Bump certifi from 2021.10.8 to 2022.12.7. ([\#14645](https://github.com/matrix-org/synapse/issues/14645))
- Bump flake8-bugbear from 22.10.27 to 22.12.6. ([\#14656](https://github.com/matrix-org/synapse/issues/14656))
- Bump packaging from 21.3 to 22.0. ([\#14657](https://github.com/matrix-org/synapse/issues/14657))
- Bump types-pillow from 9.3.0.1 to 9.3.0.4. ([\#14658](https://github.com/matrix-org/synapse/issues/14658))
- Bump serde from 1.0.148 to 1.0.150. ([\#14659](https://github.com/matrix-org/synapse/issues/14659))
- Bump phonenumbers from 8.13.1 to 8.13.2. ([\#14660](https://github.com/matrix-org/synapse/issues/14660))
- Bump authlib from 1.1.0 to 1.2.0. ([\#14661](https://github.com/matrix-org/synapse/issues/14661))
- Move `StateFilter` to `synapse.types`. ([\#14668](https://github.com/matrix-org/synapse/issues/14668))
- Improve type hints. ([\#14597](https://github.com/matrix-org/synapse/issues/14597), [\#14646](https://github.com/matrix-org/synapse/issues/14646), [\#14671](https://github.com/matrix-org/synapse/issues/14671))
-
-
 Synapse 1.73.0 (2022-12-06)
 ===========================

@@ -13,9 +13,9 @@ dependencies = [

 [[package]]
 name = "anyhow"
-version = "1.0.69"
+version = "1.0.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800"
+checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6"

 [[package]]
 name = "arc-swap"
@@ -37,9 +37,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"

 [[package]]
 name = "blake2"
-version = "0.10.6"
+version = "0.10.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
+checksum = "b12e5fd123190ce1c2e559308a94c9bacad77907d4c6005d9e58fe1a0689e55e"
 dependencies = [
 "digest",
 ]
@@ -294,9 +294,9 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.7.1"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
+checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -323,18 +323,18 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"

 [[package]]
 name = "serde"
-version = "1.0.152"
+version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
+checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.152"
+version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
+checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -343,9 +343,9 @@ dependencies = [

 [[package]]
 name = "serde_json"
-version = "1.0.92"
+version = "1.0.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7434af0dc1cbd59268aa98b4c22c131c0584d2232f6fb166efb993e2832e896a"
+checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db"
 dependencies = [
 "itoa",
 "ryu",
@@ -0,0 +1 @@
+Add a module callback for unbinding a 3PID.
@@ -0,0 +1 @@
+Optimise push badge count calculations. Contributed by Nick @ Beeper (@fizzadar).
@@ -0,0 +1 @@
+Fix a long-standing bug where a device list update might not be sent to clients in certain circumstances.
@@ -0,0 +1 @@
+Improve user search for international display names.
@@ -0,0 +1 @@
+Faster remote room joins: stream the un-partial-stating of rooms over replication.
@@ -0,0 +1 @@
+Faster remote room joins: stream the un-partial-stating of rooms over replication.
@@ -0,0 +1 @@
+Stop using deprecated `keyIds` parameter when calling `/_matrix/key/v2/server`.
@@ -0,0 +1 @@
+Update worker settings for `pusher` and `federation_sender` functionality.
@@ -0,0 +1 @@
+Add links to third party package repositories, and point to the bug which highlights Ubuntu's out-of-date packages.
@@ -0,0 +1 @@
+Stop using deprecated `keyIds` parameter when calling `/_matrix/key/v2/server`.
@@ -0,0 +1 @@
+Share the `ClientRestResource` for both workers and the main process.
@@ -0,0 +1 @@
+Add `--editable` flag to `complement.sh` which uses an editable install of Synapse for faster turn-around times whilst developing iteratively.
@@ -0,0 +1 @@
+Faster joins: use servers list approximation to send read receipts when in partial state instead of waiting for the full state of the room.
@@ -0,0 +1 @@
+Add new `push.enabled` config option to allow opting out of push notification calculation.
@@ -0,0 +1 @@
+Modernize unit tests configuration related to workers.
@@ -0,0 +1 @@
+Advertise support for Matrix 1.5 on `/_matrix/client/versions`.
@@ -0,0 +1 @@
+Remove old, incorrect minimum postgres version note and replace with a link to the [Dependency Deprecation Policy](https://matrix-org.github.io/synapse/v1.73/deprecation_policy.html).
@@ -0,0 +1 @@
+Bump jsonschema from 4.17.0 to 4.17.3.
@@ -0,0 +1 @@
+Fix a long-standing bug where a device list update might not be sent to clients in certain circumstances.
@@ -0,0 +1 @@
+Add Single-Sign On setup instructions for Mastodon-based instances.
@@ -0,0 +1 @@
+Add missing type hints.
@@ -0,0 +1 @@
+Improve opentracing and logging for to-device message handling.
@@ -0,0 +1 @@
+Suppress a spurious warning when `POST /rooms/<room_id>/<membership>/`, `POST /join/<room_id_or_alias`, or the unspecced `PUT /join/<room_id_or_alias>/<txn_id>` receive an empty HTTP request body.
@@ -0,0 +1 @@
+Fix Rust lint CI.
@@ -0,0 +1 @@
+Fix a long-standing bug where a device list update might not be sent to clients in certain circumstances.
@@ -0,0 +1 @@
+Bump JasonEtco/create-an-issue from 2.5.0 to 2.8.1.
@@ -0,0 +1 @@
+Alter some unit test environment parameters to decrease time spent running tests.
@@ -0,0 +1 @@
+Switch to Go recommended installation method for `gotestfmt` template in CI.
@@ -0,0 +1 @@
+Bump phonenumbers from 8.13.0 to 8.13.1.
@@ -0,0 +1 @@
+Bump types-setuptools from 65.5.0.3 to 65.6.0.1.
@@ -0,0 +1 @@
+Bump twine from 4.0.1 to 4.0.2.
@@ -0,0 +1 @@
+Bump types-requests from 2.28.11.2 to 2.28.11.5.
@@ -0,0 +1 @@
+Bump cryptography from 38.0.3 to 38.0.4.
@@ -0,0 +1 @@
+Add new `push.enabled` config option to allow opting out of push notification calculation.
@@ -0,0 +1 @@
+Return spec-compliant JSON errors when unknown endpoints are requested.
@@ -0,0 +1 @@
+Return spec-compliant JSON errors when unknown endpoints are requested.
@@ -0,0 +1 @@
+Fix html templates to load images only on HTTPS. Contributed by @ashfame.
@@ -0,0 +1 @@
+Fix a long-standing bug where the user directory would return 1 more row than requested.
@@ -0,0 +1 @@
+Reject invalid read receipt requests with empty room or event IDs. Contributed by Nick @ Beeper (@fizzadar).
@@ -0,0 +1 @@
+Change `turn_allow_guests` example value to lowercase `true`.
@@ -0,0 +1 @@
+Remove useless cargo install with apt from Dockerfile.
@@ -0,0 +1 @@
+Fix a bug introduced in v1.67.0 where not specifying a config file or a server URL would lead to the `register_new_matrix_user` script failing.
@@ -0,0 +1 @@
+Fix a long-standing bug where the user directory and room/user stats might be out of sync.
@@ -0,0 +1 @@
+Fix a long-standing bug where the user directory and room/user stats might be out of sync.
@@ -0,0 +1 @@
+Bump certifi from 2021.10.8 to 2022.12.7.
@@ -0,0 +1 @@
+Add missing type hints.
@@ -0,0 +1,2 @@
+Fix a bug introduced in Synapse 1.72.0 where the background updates to add non-thread unique indexes on receipts would fail if they were previously interrupted.
+
@@ -0,0 +1 @@
+Bump flake8-bugbear from 22.10.27 to 22.12.6.
@@ -0,0 +1 @@
+Bump packaging from 21.3 to 22.0.
@@ -0,0 +1 @@
+Bump types-pillow from 9.3.0.1 to 9.3.0.4.
@@ -0,0 +1 @@
+Bump serde from 1.0.148 to 1.0.150.
@@ -0,0 +1 @@
+Bump phonenumbers from 8.13.1 to 8.13.2.
@@ -0,0 +1 @@
+Bump authlib from 1.1.0 to 1.2.0.
@@ -0,0 +1 @@
+(remove from changelog: unreleased) Revert the deletion of stale devices due to performance issues.
@@ -0,0 +1 @@
+Move `StateFilter` to `synapse.types`.
@@ -1008,7 +1008,8 @@
                "mode": "absolute",
                "steps": [
                  {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                  },
                  {
                    "color": "red",
@@ -1680,7 +1681,8 @@
                "mode": "absolute",
                "steps": [
                  {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                  },
                  {
                    "color": "red",
@@ -2531,7 +2533,8 @@
                "mode": "absolute",
                "steps": [
                  {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                  },
                  {
                    "color": "red",
@@ -11293,7 +11296,7 @@
                "uid": "$datasource"
              },
              "editorMode": "code",
-              "expr": "max(synapse_admin_mau_max{instance=\"$instance\"})",
+              "expr": "synapse_admin_mau_max{instance=\"$instance\", job=~\"(hhs_)?synapse\"}",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
@@ -11307,7 +11310,7 @@
                "uid": "$datasource"
              },
              "editorMode": "code",
-              "expr": "max(synapse_admin_mau_current{instance=\"$instance\"})",
+              "expr": "synapse_admin_mau_current{instance=\"$instance\", job=~\"(hhs_)?synapse\"}",
              "hide": false,
              "legendFormat": "Current",
              "range": true,
@@ -12757,6 +12760,6 @@
  "timezone": "",
  "title": "Synapse",
  "uid": "000000012",
-  "version": 150,
+  "version": 149,
  "weekStart": ""
 }
@@ -1,47 +0,0 @@
-# `lnav` config for Synapse logs
-
-[lnav](https://lnav.org/) is a log-viewing tool. It is particularly useful when 
-you need to interleave multiple log files, or for exploring a large log file
-with regex filters. The downside is that it is not as ubiquitous as tools like
-`less`, `grep`, etc.
-
-This directory contains an `lnav` [log format definition](
-    https://docs.lnav.org/en/v0.10.1/formats.html#defining-a-new-format
-) for Synapse logs as
-emitted by Synapse with the default [logging configuration](
-    https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#log_config
-). It supports lnav 0.10.1 because that's what's packaged by my distribution.
-
-This should allow lnav:
-
- to interpret timestamps, allowing log interleaving;
- to interpret log severity levels, allowing colouring by log level(!!!);
- to interpret request IDs, allowing you to skip through a specific request; and
- to highlight room, event and user IDs in logs.
-
-See also https://gist.github.com/benje/e2ab750b0a81d11920d83af637d289f7 for a
- similar example.
-
-## Example
-
-[![asciicast](https://asciinema.org/a/556133.svg)](https://asciinema.org/a/556133)
-
-## Tips
-
- `lnav -i /path/to/synapse/checkout/contrib/lnav/synapse-log-format.json`
- `lnav my_synapse_log_file` or `lnav synapse_log_files.*`, etc.
- `lnav --help` for CLI help.
-
-Within lnav itself:
-
- `?` for help within lnav itself.
- `q` to quit.
- `/` to search a-la `less` and `vim`, then `n` and `N` to continue searching 
-  down and up.
- Use `o` and `O` to skip through logs based on the request ID (`POST-1234`, or
-  else the value of the [`request_id_header`](
-    https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html?highlight=request_id_header#listeners
-  ) header). This may get confused if the same request ID is repeated among 
-  multiple files or process restarts.
- ???
- Profit
@@ -1,67 +0,0 @@
-{
-  "$schema": "https://lnav.org/schemas/format-v1.schema.json",
-  "synapse": {
-    "title": "Synapse logs",
-    "description": "Logs output by Synapse, a Matrix homesever, under its default logging config.",
-    "regex": {
-      "log": {
-        "pattern": ".*(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) - (?<logger>.+) - (?<lineno>\\d+) - (?<level>\\w+) - (?<context>.+) - (?<body>.*)"
-      }
-    },
-    "json": false,
-    "timestamp-field": "timestamp",
-    "timestamp-format": [
-      "%Y-%m-%d %H:%M:%S,%L"
-    ],
-    "level-field": "level",
-    "body-field": "body",
-    "opid-field": "context",
-    "level": {
-      "critical": "CRITICAL",
-      "error": "ERROR",
-      "warning": "WARNING",
-      "info": "INFO",
-      "debug": "DEBUG"
-    },
-    "sample": [
-      {
-        "line": "my-matrix-server-generic-worker-4 | 2023-01-27 09:47:09,818 - synapse.replication.tcp.client - 381 - ERROR - PUT-32992 - Timed out waiting for stream receipts",
-        "level": "error"
-      },
-      {
-        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
-        "level": "warning"
-      },
-      {
-        "line": "my-matrix-server  | 2023-01-25 20:55:54,433 - synapse.storage.databases - 66 - INFO - main - [database config 'master']: Checking database server",
-        "level": "info"
-      },
-      {
-        "line": "my-matrix-server  | 2023-01-26 15:08:40,447 - synapse.access.http.8008 - 460 - INFO - PUT-74929 - 0.0.0.0 - 8008 - {@alice:example.com} Processed request: 0.011sec/0.000sec (0.000sec, 0.000sec) (0.001sec/0.008sec/3) 2B 200 \"PUT /_matrix/client/r0/user/%40alice%3Atexample.com/account_data/im.vector.setting.breadcrumbs HTTP/1.0\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Element/1.11.20 Chrome/108.0.5359.179 Electron/22.0.3 Safari/537.36\" [0 dbevts]",
-        "level": "info"
-      }
-    ],
-    "highlights": {
-      "user_id": {
-        "pattern": "(@|%40)[^:% ]+(:|%3A)[\\[\\]0-9a-zA-Z.\\-:]+(:\\d{1,5})?(?<!:)",
-        "underline": true
-      },
-      "room_id": {
-        "pattern": "(!|%21)[^:% ]+(:|%3A)[\\[\\]0-9a-zA-Z.\\-:]+(:\\d{1,5})?(?<!:)",
-        "underline": true
-      },
-      "room_alias": {
-        "pattern": "(#|%23)[^:% ]+(:|%3A)[\\[\\]0-9a-zA-Z.\\-:]+(:\\d{1,5})?(?<!:)",
-        "underline": true
-      },
-      "event_id_v1_v2": {
-        "pattern": "(\\$|%25)[^:% ]+(:|%3A)[\\[\\]0-9a-zA-Z.\\-:]+(:\\d{1,5})?(?<!:)",
-        "underline": true
-      },
-      "event_id_v3_plus": {
-        "pattern": "(\\$|%25)([A-Za-z0-9+/_]|-){43}",
-        "underline": true
-      }
-    }
-  }
-}
@@ -15,19 +15,19 @@ worker_name: generic_worker$i
 worker_replication_host: 127.0.0.1
 worker_replication_http_port: 9093

+worker_main_http_uri: http://localhost:8008/
+
 worker_listeners:
  - type: http
    port: 808$i
-    x_forwarded: true
    resources:
      - names: [client, federation]

 worker_log_config: /etc/matrix-synapse/generic-worker-log.yaml
-#worker_pid_file: DATADIR/generic_worker$i.pid
 EOF
 done
 ```

 This would create five generic workers with a unique `worker_name` field in each file and listening on ports 8081-8085.

-Customise the script to your needs. Note that `worker_pid_file` is required if `worker_daemonize` is `true`. Uncomment and/or modify the line if needed.
+Customise the script to your needs.
@@ -8,9 +8,7 @@ It also prints out the example lines for Synapse main configuration file.

 Remember to route necessary endpoints directly to a worker associated with it.

-If you run the script as-is, it will create workers with the replication listener starting from port 8034 and another, regular http listener starting from 8044. If you don't need all of the stream writers listed in the script, just remove them from the ```STREAM_WRITERS``` array. 
-
-Hint: Note that `worker_pid_file` is required if `worker_daemonize` is `true`. Uncomment and/or modify the line if needed.
+If you run the script as-is, it will create workers with the replication listener starting from port 8034 and another, regular http listener starting from 8044. If you don't need all of the stream writers listed in the script, just remove them from the ```STREAM_WRITERS``` array.

 ```sh
 #!/bin/bash
@@ -48,11 +46,9 @@ worker_listeners:

  - type: http
    port: $(expr $HTTP_START_PORT + $i)
-    x_forwarded: true
    resources:
      - names: [client]

-#worker_pid_file: DATADIR/${STREAM_WRITERS[$i]}.pid
 worker_log_config: /etc/matrix-synapse/stream-writer-log.yaml
 EOF
 HOMESERVER_YAML_INSTANCE_MAP+=$"  ${STREAM_WRITERS[$i]}_stream_writer:
@@ -95,9 +91,7 @@ Simply run the script to create YAML files in the current folder and print out t

 ```console
 $ ./create_stream_writers.sh
-```
-You should receive an output similar to the following:
-```console
+
 # Add these lines to your homeserver.yaml.
 # Don't forget to configure your reverse proxy and
 # necessary endpoints to their respective worker.
@@ -31,11 +31,12 @@ case $(dpkg-architecture -q DEB_HOST_ARCH) in
 esac

 # Manually install Poetry and export a pip-compatible `requirements.txt`
+# We need a Poetry pre-release as the export command is buggy in < 1.2
 TEMP_VENV="$(mktemp -d)"
 python3 -m venv "$TEMP_VENV"
 source "$TEMP_VENV/bin/activate"
 pip install -U pip
-pip install poetry==1.3.2
+pip install poetry==1.2.0
 poetry export \
    --extras all \
    --extras test \
@@ -1,71 +1,9 @@
-matrix-synapse-py3 (1.77.0) stable; urgency=medium
-
-  * New Synapse release 1.77.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 14 Feb 2023 12:59:02 +0100
-
-matrix-synapse-py3 (1.77.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.77.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 10 Feb 2023 12:44:21 +0000
-
-matrix-synapse-py3 (1.77.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.77.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 07 Feb 2023 13:45:14 +0000
-
-matrix-synapse-py3 (1.76.0) stable; urgency=medium
-
-  * New Synapse release 1.76.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 31 Jan 2023 08:21:47 -0800
-
-matrix-synapse-py3 (1.76.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.76.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 27 Jan 2023 11:17:57 +0000
-
-matrix-synapse-py3 (1.76.0~rc1) stable; urgency=medium
-
-  * Use Poetry 1.3.2 to manage the bundled virtualenv included with this package.
-  * New Synapse release 1.76.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 25 Jan 2023 16:21:16 +0000
-
-matrix-synapse-py3 (1.75.0) stable; urgency=medium
-
-  * New Synapse release 1.75.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 17 Jan 2023 11:36:02 +0000
-
-matrix-synapse-py3 (1.75.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.75.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 12 Jan 2023 10:30:15 -0800
-
-matrix-synapse-py3 (1.75.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.75.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 10 Jan 2023 12:18:27 +0000
-
-matrix-synapse-py3 (1.74.0) stable; urgency=medium
-
-  * New Synapse release 1.74.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Dec 2022 16:07:38 +0000
-
-matrix-synapse-py3 (1.74.0~rc1) stable; urgency=medium
+matrix-synapse-py3 (1.74.0~rc1) UNRELEASED; urgency=medium

  * New dependency on libicu-dev to provide improved results for user
    search.
-  * New Synapse release 1.74.0rc1.

- -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Dec 2022 13:30:01 +0000
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 06 Dec 2022 15:28:10 +0000

 matrix-synapse-py3 (1.73.0) stable; urgency=medium

@@ -17,10 +17,16 @@

 # Irritatingly, there is no blessed guide on how to distribute an application with its
 # poetry-managed environment in a docker image. We have opted for
-# `poetry export | pip install -r /dev/stdin`, but beware: we have experienced bugs in
-# in `poetry export` in the past.
+# `poetry export | pip install -r /dev/stdin`, but there are known bugs in
+# in `poetry export` whose fixes (scheduled for poetry 1.2) have yet to be released.
+# In case we get bitten by those bugs in the future, the recommendations here might
+# be useful:
+#     https://github.com/python-poetry/poetry/discussions/1879#discussioncomment-216865
+#     https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker?answertab=scoredesc

-ARG PYTHON_VERSION=3.11
+
+
+ARG PYTHON_VERSION=3.9

 ###
 ### Stage 0: generate requirements.txt
@@ -34,16 +40,16 @@ FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye as requirements
 # Here we use it to set up a cache for apt (and below for pip), to improve
 # rebuild speeds on slow connections.
 RUN \
-  --mount=type=cache,target=/var/cache/apt,sharing=locked \
-  --mount=type=cache,target=/var/lib/apt,sharing=locked \
-  apt-get update -qq && apt-get install -yqq \
-  build-essential git libffi-dev libssl-dev \
-  && rm -rf /var/lib/apt/lists/*
+   --mount=type=cache,target=/var/cache/apt,sharing=locked \
+   --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update -qq && apt-get install -yqq \
+      build-essential git libffi-dev libssl-dev \
+    && rm -rf /var/lib/apt/lists/*

 # We install poetry in its own build stage to avoid its dependencies conflicting with
 # synapse's dependencies.
 RUN --mount=type=cache,target=/root/.cache/pip \
-  pip install --user "poetry==1.3.2"
+  pip install --user "poetry==1.2.0"

 WORKDIR /synapse

@@ -64,9 +70,9 @@ ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE
 # Otherwise, just create an empty requirements file so that the Dockerfile can
 # proceed.
 RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
-  /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}; \
+    /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}; \
  else \
-  touch /synapse/requirements.txt; \
+    touch /synapse/requirements.txt; \
  fi

 ###
@@ -76,24 +82,24 @@ FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye as builder

 # install the OS build deps
 RUN \
-  --mount=type=cache,target=/var/cache/apt,sharing=locked \
-  --mount=type=cache,target=/var/lib/apt,sharing=locked \
-  apt-get update -qq && apt-get install -yqq \
-  build-essential \
-  libffi-dev \
-  libjpeg-dev \
-  libpq-dev \
-  libssl-dev \
-  libwebp-dev \
-  libxml++2.6-dev \
-  libxslt1-dev \
-  openssl \
-  zlib1g-dev \
-  git \
-  curl \
-  libicu-dev \
-  pkg-config \
-  && rm -rf /var/lib/apt/lists/*
+   --mount=type=cache,target=/var/cache/apt,sharing=locked \
+   --mount=type=cache,target=/var/lib/apt,sharing=locked \
+ apt-get update -qq && apt-get install -yqq \
+    build-essential \
+    libffi-dev \
+    libjpeg-dev \
+    libpq-dev \
+    libssl-dev \
+    libwebp-dev \
+    libxml++2.6-dev \
+    libxslt1-dev \
+    openssl \
+    zlib1g-dev \
+    git \
+    curl \
+    libicu-dev \
+    pkg-config \
+    && rm -rf /var/lib/apt/lists/*


 # Install rust and ensure its in the PATH
@@ -134,9 +140,9 @@ ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE
 RUN --mount=type=cache,target=/synapse/target,sharing=locked \
  --mount=type=cache,target=${CARGO_HOME}/registry,sharing=locked \
  if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
-  pip install --prefix="/install" --no-deps --no-warn-script-location /synapse[all]; \
+    pip install --prefix="/install" --no-deps --no-warn-script-location /synapse[all]; \
  else \
-  pip install --prefix="/install" --no-warn-script-location /synapse[all]; \
+    pip install --prefix="/install" --no-warn-script-location /synapse[all]; \
  fi

 ###
@@ -151,20 +157,19 @@ LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git
 LABEL org.opencontainers.image.licenses='Apache-2.0'

 RUN \
-  --mount=type=cache,target=/var/cache/apt,sharing=locked \
-  --mount=type=cache,target=/var/lib/apt,sharing=locked \
+   --mount=type=cache,target=/var/cache/apt,sharing=locked \
+   --mount=type=cache,target=/var/lib/apt,sharing=locked \
  apt-get update -qq && apt-get install -yqq \
-  curl \
-  gosu \
-  libjpeg62-turbo \
-  libpq5 \
-  libwebp6 \
-  xmlsec1 \
-  libjemalloc2 \
-  libicu67 \
-  libssl-dev \
-  openssl \
-  && rm -rf /var/lib/apt/lists/*
+    curl \
+    gosu \
+    libjpeg62-turbo \
+    libpq5 \
+    libwebp6 \
+    xmlsec1 \
+    libjemalloc2 \
+    libssl-dev \
+    openssl \
+    && rm -rf /var/lib/apt/lists/*

 COPY --from=builder /install /usr/local
 COPY ./docker/start.py /start.py
@@ -175,4 +180,4 @@ EXPOSE 8008/tcp 8009/tcp 8448/tcp
 ENTRYPOINT ["/start.py"]

 HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
-  CMD curl -fSs http://localhost:8008/health || exit 1
+    CMD curl -fSs http://localhost:8008/health || exit 1
@@ -36,10 +36,8 @@ RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
        wget

 # fetch and unpack the package
-# We are temporarily using a fork of dh-virtualenv due to an incompatibility with Python 3.11, which ships with
-# Debian sid. TODO: Switch back to upstream once https://github.com/spotify/dh-virtualenv/pull/354 has merged.
 RUN mkdir /dh-virtualenv
-RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/matrix-org/dh-virtualenv/archive/refs/tags/matrixorg-2023010302.tar.gz
+RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/spotify/dh-virtualenv/archive/refs/tags/1.2.2.tar.gz
 RUN tar -xv --strip-components=1 -C /dh-virtualenv -f /dh-virtualenv.tar.gz

 # install its build deps. We do another apt-cache-update here, because we might
@@ -6,7 +6,7 @@ set -e

 echo "Complement Synapse launcher"
 echo "  Args: $@"
-echo "  Env: SYNAPSE_COMPLEMENT_DATABASE=$SYNAPSE_COMPLEMENT_DATABASE SYNAPSE_COMPLEMENT_USE_WORKERS=$SYNAPSE_COMPLEMENT_USE_WORKERS SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR=$SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR"
+echo "  Env: SYNAPSE_COMPLEMENT_DATABASE=$SYNAPSE_COMPLEMENT_DATABASE SYNAPSE_COMPLEMENT_USE_WORKERS=$SYNAPSE_COMPLEMENT_USE_WORKERS"

 function log {
    d=$(date +"%Y-%m-%d %H:%M:%S,%3N")
@@ -76,17 +76,6 @@ else
 fi


-if [[ -n "$SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR" ]]; then
-  if [[ -n "$SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER" ]]; then
-    export SYNAPSE_COMPLEMENT_FORKING_LAUNCHER_ASYNC_IO_REACTOR="1"
-  else
-    export SYNAPSE_ASYNC_IO_REACTOR="1"
-  fi
-else
-  export SYNAPSE_ASYNC_IO_REACTOR="0"
-fi
-
-
 # Add Complement's appservice registration directory, if there is one
 # (It can be absent when there are no application services in this test!)
 if [ -d /complement/appservice ]; then
@@ -94,14 +94,12 @@ allow_device_name_lookup_over_federation: true
 experimental_features:
  # Enable history backfilling support
  msc2716_enabled: true
+  # server-side support for partial state in /send_join responses
+  msc3706_enabled: true
+  {% if not workers_in_use %}
  # client-side support for partial state in /send_join responses
  faster_joins: true
-  # Enable support for polls
-  msc3381_polls_enabled: true
-  # Enable deleting device-specific notification settings stored in account data
-  msc3890_enabled: true
-  # Enable removing account data support
-  msc3391_enabled: true
+  {% endif %}
  # Filtering /messages by relation type.
  msc3874_enabled: true

@@ -97,7 +97,6 @@
    - [Log Contexts](log_contexts.md)
    - [Replication](replication.md)
    - [TCP Replication](tcp_replication.md)
-    - [Faster remote joins](development/synapse_architecture/faster_joins.md)
  - [Internal Documentation](development/internal_documentation/README.md)
    - [Single Sign-On]()
      - [SAML](development/saml.md)
@@ -5,7 +5,7 @@ use it, you must enable the account validity feature (under
 `account_validity`) in Synapse's configuration.

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 ## Renew account

@@ -3,7 +3,7 @@
 This API returns information about reported events.

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 The api is:
 ```
@@ -6,7 +6,7 @@ Details about the format of the `media_id` and storage of the media in the file
 are documented under [media repository](../media_repository.md).

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 ## List all media in a room

@@ -11,7 +11,7 @@ Note that Synapse requires at least one message in each room, so it will never
 delete the last message in a room.

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 The API is:

@@ -6,7 +6,7 @@ local users. The server administrator must be in the room and have permission to
 invite users.

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 ## Parameters

@@ -5,7 +5,7 @@ server. There are various parameters available that allow for filtering and
 sorting the returned list. This API supports pagination.

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 **Parameters**

@@ -400,7 +400,7 @@ sent to a room in a given timeframe. There are various parameters available
 that allow for filtering and ordering the returned list. This API supports pagination.

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 This endpoint mirrors the [Matrix Spec defined Messages API](https://spec.matrix.org/v1.1/client-server-api/#get_matrixclientv3roomsroomidmessages).

@@ -4,7 +4,7 @@ Returns information about all local media usage of users. Gives the
 possibility to filter them by time and user.

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 The API is:

@@ -1,7 +1,7 @@
 # User Admin API

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 ## Query User Account

@@ -15,7 +15,6 @@ app_service_config_files:
 The format of the AS configuration file is as follows:

 ```yaml
-id: <your-AS-id>
 url: <base url of AS>
 as_token: <token AS will add to requests to HS>
 hs_token: <token HS will add to requests to AS>
@@ -10,17 +10,26 @@ The necessary tools are:

 - [black](https://black.readthedocs.io/en/stable/), a source code formatter;
 - [isort](https://pycqa.github.io/isort/), which organises each file's imports;
- [ruff](https://github.com/charliermarsh/ruff), which can spot common errors; and
+- [flake8](https://flake8.pycqa.org/en/latest/), which can spot common errors; and
 - [mypy](https://mypy.readthedocs.io/en/stable/), a type checker.

-See [the contributing guide](development/contributing_guide.md#run-the-linters) for instructions
-on how to install the above tools and run the linters.
+Install them with:
+
+```sh
+pip install -e ".[lint,mypy]"
+```
+
+The easiest way to run the lints is to invoke the linter script as follows.
+
+```sh
+scripts-dev/lint.sh
+```

 It's worth noting that modern IDEs and text editors can run these tools
 automatically on save. It may be worth looking into whether this
 functionality is supported in your editor for a more convenient
-development workflow. It is not, however, recommended to run `mypy`
-on save as it takes a while and can be very resource intensive.
+development workflow. It is not, however, recommended to run `flake8` or `mypy`
+on save as they take a while and can be very resource intensive.

 ## General rules

@@ -24,8 +24,6 @@ The code of Synapse is written in Python 3. To do pretty much anything, you'll n

 Synapse can connect to PostgreSQL via the [psycopg2](https://pypi.org/project/psycopg2/) Python library. Building this library from source requires access to PostgreSQL's C header files. On Debian or Ubuntu Linux, these can be installed with `sudo apt install libpq-dev`.

-Synapse has an optional, improved user search with better Unicode support. For that you need the development package of `libicu`. On Debian or Ubuntu Linux, this can be installed with `sudo apt install libicu-dev`.
-
 The source code of Synapse is hosted on GitHub. You will also need [a recent version of git](https://github.com/git-guides/install-git).

 For some tests, you will need [a recent version of Docker](https://docs.docker.com/get-docker/).
@@ -67,7 +65,7 @@ pipx install poetry
 but see poetry's [installation instructions](https://python-poetry.org/docs/#installation)
 for other installation methods.

-Developing Synapse requires Poetry version 1.3.2 or later.
+Synapse requires Poetry version 1.2.0 or later.

 Next, open a terminal and install dependencies as follows:

@@ -106,8 +104,8 @@ regarding Synapse's Admin API, which is used mostly by sysadmins and external
 service developers.

 Synapse's code style is documented [here](../code_style.md). Please follow
-it, including the conventions for [configuration
-options and documentation](../code_style.md#configuration-code-and-documentation-format).
+it, including the conventions for the [sample configuration
+file](../code_style.md#configuration-file-format).

 We welcome improvements and additions to our documentation itself! When
 writing new pages, please
@@ -126,7 +124,7 @@ changes to the Rust code.


 # 8. Test, test, test!
-<a name="test-test-test" id="test-test-test"></a>
+<a name="test-test-test"></a>

 While you're developing and before submitting a patch, you'll
 want to test your code.
@@ -332,7 +330,6 @@ The above will run a monolithic (single-process) Synapse with SQLite as the data
    [here](https://github.com/matrix-org/synapse/blob/develop/docker/configure_workers_and_start.py#L54).
    A safe example would be `WORKER_TYPES="federation_inbound, federation_sender, synchrotron"`.
    See the [worker documentation](../workers.md) for additional information on workers.
- Passing `ASYNCIO_REACTOR=1` as an environment variable to use the Twisted asyncio reactor instead of the default one.

 To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`, e.g:
 ```sh
@@ -383,7 +380,7 @@ To prepare a Pull Request, please:
 ## Changelog

 All changes, even minor ones, need a corresponding changelog / newsfragment
-entry. These are managed by [Towncrier](https://github.com/twisted/towncrier).
+entry. These are managed by [Towncrier](https://github.com/hawkowl/towncrier).

 To create a changelog entry, make a new file in the `changelog.d` directory named
 in the format of `PRnumber.type`. The type can be one of the following:
@@ -425,7 +422,8 @@ chicken-and-egg problem.
 There are two options for solving this:

 1. Open the PR without a changelog file, see what number you got, and *then*
-   add the changelog file to your branch, or:
+   add the changelog file to your branch (see [Updating your pull
+   request](#updating-your-pull-request)), or:

 1. Look at the [list of all
   issues/PRs](https://github.com/matrix-org/synapse/issues?q=), add one to the
@@ -2,13 +2,6 @@

 This is a quick cheat sheet for developers on how to use [`poetry`](https://python-poetry.org/).

-# Installing
-
-See the [contributing guide](contributing_guide.md#4-install-the-dependencies).
-
-Developers should use Poetry 1.3.2 or higher. If you encounter problems related
-to poetry, please [double-check your poetry version](#check-the-version-of-poetry-with-poetry---version).
-
 # Background

 Synapse uses a variety of third-party Python packages to function as a homeserver.
@@ -130,7 +123,7 @@ context of poetry's venv, without having to run `poetry shell` beforehand.
 ## ...reset my venv to the locked environment?

 ```shell
-poetry install --all-extras --sync
+poetry install --extras all --remove-untracked
 ```

 ## ...delete everything and start over from scratch?
@@ -190,6 +183,7 @@ Either:
 - manually update `pyproject.toml`; then `poetry lock --no-update`; or else
 - `poetry add packagename`. See `poetry add --help`; note the `--dev`,
  `--extras` and `--optional` flags in particular.
+  - **NB**: this specifies the new package with a version given by a "caret bound". This won't get forced to its lowest version in the old deps CI job: see [this TODO](https://github.com/matrix-org/synapse/blob/4e1374373857f2f7a911a31c50476342d9070681/.ci/scripts/test_old_deps.sh#L35-L39).

 Include the updated `pyproject.toml` and `poetry.lock` files in your commit.

@@ -202,7 +196,7 @@ poetry remove packagename
 ```

 ought to do the trick. Alternatively, manually update `pyproject.toml` and
-`poetry lock --no-update`. Include the updated `pyproject.toml` and `poetry.lock`
+`poetry lock --no-update`. Include the updated `pyproject.toml` and poetry.lock`
 files in your commit.

 ## ...update the version range for an existing dependency?
@@ -246,6 +240,9 @@ poetry export --extras all

 Be wary of bugs in `poetry export` and `pip install -r requirements.txt`.

+Note: `poetry export` will be made a plugin in Poetry 1.2. Additional config may
+be required.
+
 ## ...build a test wheel?

 I usually use
@@ -258,26 +255,12 @@ because [`build`](https://github.com/pypa/build) is a standardish tool which
 doesn't require poetry. (It's what we use in CI too). However, you could try
 `poetry build` too.

-## ...handle a Dependabot pull request?
-
-Synapse uses Dependabot to keep the `poetry.lock` file up-to-date. When it
-creates a pull request a GitHub Action will run to automatically create a changelog
-file. Ensure that:
-
-* the lockfile changes look reasonable;
-* the upstream changelog file (linked in the description) doesn't include any
-  breaking changes;
-* continuous integration passes (due to permissions, the GitHub Actions run on
-  the changelog commit will fail, look at the initial commit of the pull request);
-
-In particular, any updates to the type hints (usually packages which start with `types-`)
-should be safe to merge if linting passes.

 # Troubleshooting

 ## Check the version of poetry with `poetry --version`.

-The minimum version of poetry supported by Synapse is 1.3.2.
+The minimum version of poetry supported by Synapse is 1.2.

 It can also be useful to check the version of `poetry-core` in use. If you've
 installed `poetry` with `pipx`, try `pipx runpip poetry list | grep
@@ -1,375 +0,0 @@
-# How do faster joins work?
-
-This is a work-in-progress set of notes with two goals:
- act as a reference, explaining how Synapse implements faster joins; and
- record the rationale behind our choices.
-
-See also [MSC3902](https://github.com/matrix-org/matrix-spec-proposals/pull/3902).
-
-The key idea is described by [MSC706](https://github.com/matrix-org/matrix-spec-proposals/pull/3902). This allows servers to
-request a lightweight response to the federation `/send_join` endpoint.
-This is called a **faster join**, also known as a **partial join**. In these
-notes we'll usually use the word "partial" as it matches the database schema.
-
-## Overview: processing events in a partially-joined room
-
-The response to a partial join consists of
- the requested join event `J`,
- a list of the servers in the room (according to the state before `J`),
- a subset of the state of the room before `J`,
- the full auth chain of that state subset.
-
-Synapse marks the room as partially joined by adding a row to the database table
-`partial_state_rooms`. It also marks the join event `J` as "partially stated",
-meaning that we have neither received nor computed the full state before/after
-`J`. This is done by adding a row to `partial_state_events`.
-
-<details><summary>DB schema</summary>
-
-```
-matrix=> \d partial_state_events
-Table "matrix.partial_state_events"
-  Column  │ Type │ Collation │ Nullable │ Default
-══════════╪══════╪═══════════╪══════════╪═════════
- room_id  │ text │           │ not null │
- event_id │ text │           │ not null │
- 
-matrix=> \d partial_state_rooms
-                Table "matrix.partial_state_rooms"
-         Column         │  Type  │ Collation │ Nullable │ Default 
-════════════════════════╪════════╪═══════════╪══════════╪═════════
- room_id                │ text   │           │ not null │ 
- device_lists_stream_id │ bigint │           │ not null │ 0
- join_event_id          │ text   │           │          │ 
- joined_via             │ text   │           │          │ 
-
-matrix=> \d partial_state_rooms_servers
-     Table "matrix.partial_state_rooms_servers"
-   Column    │ Type │ Collation │ Nullable │ Default 
-═════════════╪══════╪═══════════╪══════════╪═════════
- room_id     │ text │           │ not null │ 
- server_name │ text │           │ not null │ 
-```
-
-Indices, foreign-keys and check constraints are omitted for brevity.
-</details>
-
-While partially joined to a room, Synapse receives events `E` from remote
-homeservers as normal, and can create events at the request of its local users.
-However, we run into trouble when we enforce the [checks on an event].
-
-> 1. Is a valid event, otherwise it is dropped. For an event to be valid, it
-     must contain a room_id, and it must comply with the event format of that
->    room version.
-> 2. Passes signature checks, otherwise it is dropped.
-> 3. Passes hash checks, otherwise it is redacted before being processed further.
-> 4. Passes authorization rules based on the event’s auth events, otherwise it
->    is rejected.
-> 5. **Passes authorization rules based on the state before the event, otherwise
->    it is rejected.**
-> 6. **Passes authorization rules based on the current state of the room,
->    otherwise it is “soft failed”.**
-
-[checks on an event]: https://spec.matrix.org/v1.5/server-server-api/#checks-performed-on-receipt-of-a-pdu
-
-We can enforce checks 1--4 without any problems.
-But we cannot enforce checks 5 or 6 with complete certainty, since Synapse does
-not know the full state before `E`, nor that of the room.
-
-### Partial state
-
-Instead, we make a best-effort approximation.
-While the room is considered partially joined, Synapse tracks the "partial
-state" before events.
-This works in a similar way as regular state:
-
- The partial state before `J` is that given to us by the partial join response.
- The partial state before an event `E` is the resolution of the partial states
-  after each of `E`'s `prev_event`s.
- If `E` is rejected or a message event, the partial state after `E` is the
-  partial state before `E`.
- Otherwise, the partial state after `E` is the partial state before `E`, plus
-  `E` itself.
-
-More concisely, partial state propagates just like full state; the only
-difference is that we "seed" it with an incomplete initial state.
-Synapse records that we have only calculated partial state for this event with
-a row in `partial_state_events`.
-
-While the room remains partially stated, check 5 on incoming events to that
-room becomes:
-
-> 5. Passes authorization rules based on **the resolution between the partial
->    state before `E` and `E`'s auth events.** If the event fails to pass
->    authorization rules, it is rejected.
-
-Additionally, check 6 is deleted: no soft-failures are enforced.
-
-While partially joined, the current partial state of the room is defined as the
-resolution across the partial states after all forward extremities in the room.
-
-_Remark._ Events with partial state are _not_ considered
-[outliers](../room-dag-concepts.md#outliers).
-
-### Approximation error
-
-Using partial state means the auth checks can fail in a few different ways[^2].
-
-[^2]: Is this exhaustive?
-
- We may erroneously accept an incoming event in check 5 based on partial state
-  when it would have been rejected based on full state, or vice versa.
- This means that an event could erroneously be added to the current partial
-  state of the room when it would not be present in the full state of the room,
-  or vice versa.
- Additionally, we may have skipped soft-failing an event that would have been
-  soft-failed based on full state.
-
-(Note that the discrepancies described in the last two bullets are user-visible.)
-
-This means that we have to be very careful when we want to lookup pieces of room
-state in a partially-joined room. Our approximation of the state may be
-incorrect or missing. But we can make some educated guesses. If
-
- our partial state is likely to be correct, or
- the consequences of our partial state being incorrect are minor,
-
-then we proceed as normal, and let the resync process fix up any mistakes (see
-below).
-
-When is our partial state likely to be correct?
-
- It's more accurate the closer we are to the partial join event. (So we should
-  ideally complete the resync as soon as possible.)
- Non-member events: we will have received them as part of the partial join
-  response, if they were part of the room state at that point. We may
-  incorrectly accept or reject updates to that state (at first because we lack
-  remote membership information; later because of compounding errors), so these
-  can become incorrect over time.
- Local members' memberships: we are the only ones who can create join and
-  knock events for our users. We can't be completely confident in the
-  correctness of bans, invites and kicks from other homeservers, but the resync
-  process should correct any mistakes.
- Remote members' memberships: we did not receive these in the /send_join
-  response, so we have essentially no idea if these are correct or not.
-
-In short, we deem it acceptable to trust the partial state for non-membership
-and local membership events. For remote membership events, we wait for the
-resync to complete, at which point we have the full state of the room and can
-proceed as normal.
-
-### Fixing the approximation with a resync
-
-The partial-state approximation is only a temporary affair. In the background,
-synapse beings a "resync" process. This is a continuous loop, starting at the
-partial join event and proceeding downwards through the event graph. For each 
-`E` seen in the room since partial join, Synapse will fetch 
-
- the event ids in the state of the room before `E`, via 
-  [`/state_ids`](https://spec.matrix.org/v1.5/server-server-api/#get_matrixfederationv1state_idsroomid);
- the event ids in the full auth chain of `E`, included in the `/state_ids` 
-  response; and
- any events from the previous two bullets that Synapse hasn't persisted, via
-  [`/state](https://spec.matrix.org/v1.5/server-server-api/#get_matrixfederationv1stateroomid).
-
-This means Synapse has (or can compute) the full state before `E`, which allows
-Synapse to properly authorise or reject `E`. At this point ,the event
-is considered to have "full state" rather than "partial state". We record this
-by removing `E` from the `partial_state_events` table.
-
-\[**TODO:** Does Synapse persist a new state group for the full state
-before `E`, or do we alter the (partial-)state group in-place? Are state groups
-ever marked as partially-stated? \]
-
-This scheme means it is possible for us to have accepted and sent an event to 
-clients, only to reject it during the resync. From a client's perspective, the 
-effect is similar to a retroactive 
-state change due to state resolution---i.e. a "state reset".[^3]
-
-[^3]: Clients should refresh caches to detect such a change. Rumour has it that 
-sliding sync will fix this.
-
-When all events since the join `J` have been fully-stated, the room resync
-process is complete. We record this by removing the room from
-`partial_state_rooms`.
-
-## Faster joins on workers
-
-For the time being, the resync process happens on the master worker.
-A new replication stream `un_partial_stated_room` is added. Whenever a resync
-completes and a partial-state room becomes fully stated, a new message is sent
-into that stream containing the room ID.
-
-## Notes on specific cases
-
-> **NB.** The notes below are rough. Some of them are hidden under `<details>`
-disclosures because they have yet to be implemented in mainline Synapse.
-
-### Creating events during a partial join
-
-When sending out messages during a partial join, we assume our partial state is 
-accurate and proceed as normal. For this to have any hope of succeeding at all,
-our partial state must contain an entry for each of the (type, state key) pairs
-[specified by the auth rules](https://spec.matrix.org/v1.3/rooms/v10/#authorization-rules):
-
- `m.room.create`
- `m.room.join_rules`
- `m.room.power_levels`
- `m.room.third_party_invite`
- `m.room.member`
-
-The first four of these should be present in the state before `J` that is given
-to us in the partial join response; only membership events are omitted. In order
-for us to consider the user joined, we must have their membership event. That
-means the only possible omission is the target's membership in an invite, kick
-or ban.
-
-The worst possibility is that we locally invite someone who is banned according to
-the full state, because we lack their ban in our current partial state. The rest 
-of the federation---at least, those who are fully joined---should correctly 
-enforce the [membership transition constraints](
-    https://spec.matrix.org/v1.3/client-server-api/#room-membership
-). So any the erroneous invite should be ignored by fully-joined
-homeservers and resolved by the resync for partially-joined homeservers.
-
-
-
-In more generality, there are two problems we're worrying about here:
-
- We might create an event that is valid under our partial state, only to later
-  find out that is actually invalid according to the full state.
- Or: we might refuse to create an event that is invalid under our partial
-  state, even though it would be perfectly valid under the full state.
-
-However we expect such problems to be unlikely in practise, because
-
- We trust that the room has sensible power levels, e.g. that bad actors with
-  high power levels are demoted before their ban.
- We trust that the resident server provides us up-to-date power levels, join
-  rules, etc.
- State changes in rooms are relatively infrequent, and the resync period is
-  relatively quick.
-
-#### Sending out the event over federation
-
-**TODO:** needs prose fleshing out.
-
-Normally: send out in a fed txn to all HSes in the room.
-We only know that some HSes were in the room at some point. Wat do.
-Send it out to the list of servers from the first join.
-**TODO** what do we do here if we have full state?
-If the prev event was created by us, we can risk sending it to the wrong HS. (Motivation: privacy concern of the content. Not such a big deal for a public room or an encrypted room. But non-encrypted invite-only...)
-But don't want to send out sensitive data in other HS's events in this way.
-
-Suppose we discover after resync that we shouldn't have sent out one our events (not a prev_event) to a target HS. Not much we can do.
-What about if we didn't send them an event but shouldn't've?
-E.g. what if someone joined from a new HS shortly after you did? We wouldn't talk to them.
-Could imagine sending out the "Missed" events after the resync but... painful to work out what they shuld have seen if they joined/left.
-Instead, just send them the latest event (if they're still in the room after resync) and let them backfill.(?)
- Don't do this currently.
- If anyone who has received our messages sends a message to a HS we missed, they can backfill our messages
- Gap: rooms which are infrequently used and take a long time to resync.
-
-### Joining after a partial join
-
-**NB.** Not yet implemented.
-
-<details>
-
-**TODO:** needs prose fleshing out. Liase with Matthieu. Explain why /send_join
-(Rich was surprised we didn't just create it locally. Answer: to try and avoid
-a join which then gets rejected after resync.)
-
-We don't know for sure that any join we create would be accepted.
-E.g. the joined user might have been banned; the join rules might have changed in a way that we didn't realise... some way in which the partial state was mistaken.
-Instead, do another partial make-join/send-join handshake to confirm that the join works.
- Probably going to get a bunch of duplicate state events and auth events.... but the point of partial joins is that these should be small. Many are already persisted = good.
- What if the second send_join response includes a different list of reisdent HSes? Could ignore it.
-  - Could even have a special flag that says "just make me a join", i.e. don't bother giving me state or servers in room. Deffo want the auth chain tho.
- SQ: wrt device lists it's a lot safer to ignore it!!!!!
- What if the state at the second join is inconsistent with what we have? Ignore it?
-
-</details>
-
-### Leaving (and kicks and bans) after a partial join
-
-**NB.** Not yet implemented.
-
-<details>
-
-When you're fully joined to a room, to have `U` leave a room their homeserver
-needs to
-
- create a new leave event for `U` which will be accepted by other homeservers,
-  and
- send that event `U` out to the homeservers in the federation.
-
-When is a leave event accepted? See
-[v10 auth rules](https://spec.matrix.org/v1.5/rooms/v10/#authorization-rules):
-
-> 4. If type is m.room.member: [...]
-     >
-     >    5. If membership is leave:
-             >
-             >       1. If the sender matches state_key, allow if and only if that user’s current membership state is invite, join, or knock.
->       2. [...]
-
-I think this means that (well-formed!) self-leaves are governed entirely by
-4.5.1. This means that if we correctly calculate state which says that `U` is
-invited, joined or knocked and include it in the leave's auth events, our event
-is accepted by checks 4 and 5 on incoming events.
-
-> 4. Passes authorization rules based on the event’s auth events, otherwise
-     >    it is rejected.
-> 5. Passes authorization rules based on the state before the event, otherwise
-     >    it is rejected.
-
-The only way to fail check 6 is if the receiving server's current state of the
-room says that `U` is banned, has left, or has no membership event. But this is
-fine: the receiving server already thinks that `U` isn't in the room.
-
-> 6. Passes authorization rules based on the current state of the room,
-     >    otherwise it is “soft failed”.
-
-For the second point (publishing the leave event), the best thing we can do is
-to is publish to all HSes we know to be currently in the room. If they miss that
-event, they might send us traffic in the room that we don't care about. This is
-a problem with leaving after a "full" join; we don't seek to fix this with
-partial joins.
-
-(With that said: there's nothing machine-readable in the /send response. I don't
-think we can deduce "destination has left the room" from a failure to /send an
-event into that room?)
-
-#### Can we still do this during a partial join?
-
-We can create leave events and can choose what gets included in our auth events,
-so we can be sure that we pass check 4 on incoming events. For check 5, we might
-have an incorrect view of the state before an event.
-The only way we might erroneously think a leave is valid is if
-
- the partial state before the leave has `U` joined, invited or knocked, but
- the full state before the leave has `U` banned, left or not present,
-
-in which case the leave doesn't make anything worse: other HSes already consider
-us as not in the room, and will continue to do so after seeing the leave.
-
-The remaining obstacle is then: can we safely broadcast the leave event? We may
-miss servers or incorrectly think that a server is in the room. Or the
-destination server may be offline and miss the transaction containing our leave
-event.This should self-heal when they see an event whose `prev_events` descends
-from our leave.
-
-Another option we considered was to use federation `/send_leave` to ask a
-fully-joined server to send out the event on our behalf. But that introduces
-complexity without much benefit. Besides, as Rich put it,
-
-> sending out leaves is pretty best-effort currently
-
-so this is probably good enough as-is.
-
-#### Cleanup after the last leave
-
-**TODO**: what cleanup is necessary? Is it all just nice-to-have to save unused
-work?
-</details>
@@ -265,6 +265,33 @@ server_.

 If multiple modules implement this callback, Synapse runs them all in order.

+### `unbind_threepid`
+
+_First introduced in Synapse v1.74.0_
+
+```python
+async def unbind_threepid(
+    user_id: str, medium: str, address: str, identity_server: str
+) -> Tuple[bool, bool]:
+```
+
+Called before a threepid association is removed.
+
+The module is given the Matrix ID of the user to which an association is to be removed,
+as well as the medium (`email` or `msisdn`), address of the third-party identifier and
+the identity server where the threepid was successfully registered.
+
+A module can hence do its own custom unbinding, if for example it did also registered a custom
+binding logic with `on_threepid_bind`.
+
+It should return a tuple of 2 booleans:
+- first one should be `True` on a success calling the identity server, otherwise `False` if
+the identity server doesn't support unbinding (or no identity server found to contact).
+- second one should be `True` if unbind needs to stop there. In this case no other module
+unbind will be called, and the default unbind made to the IS that was used on bind will also be
+skipped. In any case the mapping will be removed from the Synapse 3pid remote table,
+except if an Exception was raised at some point.
+
 ## Example

 The example below is a module that implements the third-party rules callback
@@ -59,8 +59,8 @@ namespace (such as anything under `/_matrix/client` for example). It is strongly
 recommended that modules register their web resources under the `/_synapse/client`
 namespace.

-The provided resource is a Python class that implements Twisted's [IResource](https://docs.twistedmatrix.com/en/stable/api/twisted.web.resource.IResource.html)
-interface (such as [Resource](https://docs.twistedmatrix.com/en/stable/api/twisted.web.resource.Resource.html)).
+The provided resource is a Python class that implements Twisted's [IResource](https://twistedmatrix.com/documents/current/api/twisted.web.resource.IResource.html)
+interface (such as [Resource](https://twistedmatrix.com/documents/current/api/twisted.web.resource.Resource.html)).

 Only one resource can be registered for a given path. If several modules attempt to
 register a resource for the same path, the module that appears first in Synapse's
@@ -82,4 +82,4 @@ the callback name as the argument name and the function as its value. A
 `register_[...]_callbacks` method exists for each category.

 Callbacks for each category can be found on their respective page of the
-[Synapse documentation website](https://matrix-org.github.io/synapse).
+[Synapse documentation website](https://matrix-org.github.io/synapse).
@@ -88,41 +88,98 @@ oidc_providers:
        display_name_template: "{{ user.name }}"
 ```

-### Apple
+### Dex

-Configuring "Sign in with Apple" (SiWA) requires an Apple Developer account.
+[Dex][dex-idp] is a simple, open-source OpenID Connect Provider.
+Although it is designed to help building a full-blown provider with an
+external database, it can be configured with static passwords in a config file.

-You will need to create a new "Services ID" for SiWA, and create and download a
-private key with "SiWA" enabled.
+Follow the [Getting Started guide](https://dexidp.io/docs/getting-started/)
+to install Dex.

-As well as the private key file, you will need:
- * Client ID: the "identifier" you gave the "Services ID"
- * Team ID: a 10-character ID associated with your developer account.
- * Key ID: the 10-character identifier for the key.
-
-[Apple's developer documentation](https://help.apple.com/developer-account/?lang=en#/dev77c875b7e)
-has more information on setting up SiWA.
-
-The synapse config will look like this:
+Edit `examples/config-dev.yaml` config file from the Dex repo to add a client:

 ```yaml
-  - idp_id: apple
-    idp_name: Apple
-    issuer: "https://appleid.apple.com"
-    client_id: "your-client-id" # Set to the "identifier" for your "ServicesID"
-    client_auth_method: "client_secret_post"
-    client_secret_jwt_key:
-      key_file: "/path/to/AuthKey_KEYIDCODE.p8"  # point to your key file
-      jwt_header:
-        alg: ES256
-        kid: "KEYIDCODE"   # Set to the 10-char Key ID
-      jwt_payload:
-        iss: TEAMIDCODE    # Set to the 10-char Team ID
-    scopes: ["name", "email", "openid"]
-    authorization_endpoint: https://appleid.apple.com/auth/authorize?response_mode=form_post
+staticClients:
+- id: synapse
+  secret: secret
+  redirectURIs:
+  - '[synapse public baseurl]/_synapse/client/oidc/callback'
+  name: 'Synapse'
+```
+
+Run with `dex serve examples/config-dev.yaml`.
+
+Synapse config:
+
+```yaml
+oidc_providers:
+  - idp_id: dex
+    idp_name: "My Dex server"
+    skip_verification: true # This is needed as Dex is served on an insecure endpoint
+    issuer: "http://127.0.0.1:5556/dex"
+    client_id: "synapse"
+    client_secret: "secret"
+    scopes: ["openid", "profile"]
    user_mapping_provider:
      config:
-        email_template: "{{ user.email }}"
+        localpart_template: "{{ user.name }}"
+        display_name_template: "{{ user.name|capitalize }}"
+```
+### Keycloak
+
+[Keycloak][keycloak-idp] is an opensource IdP maintained by Red Hat.
+
+Keycloak supports OIDC Back-Channel Logout, which sends logout notification to Synapse, so that Synapse users get logged out when they log out from Keycloak.
+This can be optionally enabled by setting `backchannel_logout_enabled` to `true` in the Synapse configuration, and by setting the "Backchannel Logout URL" in Keycloak.
+
+Follow the [Getting Started Guide](https://www.keycloak.org/getting-started) to install Keycloak and set up a realm.
+
+1. Click `Clients` in the sidebar and click `Create`
+
+2. Fill in the fields as below:
+
+| Field | Value |
+|-----------|-----------|
+| Client ID | `synapse` |
+| Client Protocol | `openid-connect` |
+
+3. Click `Save`
+4. Fill in the fields as below:
+
+| Field | Value |
+|-----------|-----------|
+| Client ID | `synapse` |
+| Enabled | `On` |
+| Client Protocol | `openid-connect` |
+| Access Type | `confidential` |
+| Valid Redirect URIs | `[synapse public baseurl]/_synapse/client/oidc/callback` |
+| Backchannel Logout URL (optional) | `[synapse public baseurl]/_synapse/client/oidc/backchannel_logout` |
+| Backchannel Logout Session Required (optional) | `On` |
+
+5. Click `Save`
+6. On the Credentials tab, update the fields:
+
+| Field | Value |
+|-------|-------|
+| Client Authenticator | `Client ID and Secret` |
+
+7. Click `Regenerate Secret`
+8. Copy Secret
+
+```yaml
+oidc_providers:
+  - idp_id: keycloak
+    idp_name: "My KeyCloak server"
+    issuer: "https://127.0.0.1:8443/realms/{realm_name}"
+    client_id: "synapse"
+    client_secret: "copy secret generated from above"
+    scopes: ["openid", "profile"]
+    user_mapping_provider:
+      config:
+        localpart_template: "{{ user.preferred_username }}"
+        display_name_template: "{{ user.name }}"
+    backchannel_logout_enabled: true # Optional
 ```

 ### Auth0
@@ -205,43 +262,285 @@ oidc_providers:
        display_name_template: "{{ user.preferred_username|capitalize }}" # TO BE FILLED: If your users have names in Authentik and you want those in Synapse, this should be replaced with user.name|capitalize.
 ```

-### Dex
+### LemonLDAP

-[Dex][dex-idp] is a simple, open-source OpenID Connect Provider.
-Although it is designed to help building a full-blown provider with an
-external database, it can be configured with static passwords in a config file.
+[LemonLDAP::NG][lemonldap] is an open-source IdP solution.

-Follow the [Getting Started guide](https://dexidp.io/docs/getting-started/)
-to install Dex.
-
-Edit `examples/config-dev.yaml` config file from the Dex repo to add a client:
+1. Create an OpenID Connect Relying Parties in LemonLDAP::NG
+2. The parameters are:
+- Client ID under the basic menu of the new Relying Parties (`Options > Basic >
+  Client ID`)
+- Client secret (`Options > Basic > Client secret`)
+- JWT Algorithm: RS256 within the security menu of the new Relying Parties
+  (`Options > Security > ID Token signature algorithm` and `Options > Security >
+  Access Token signature algorithm`)
+- Scopes: OpenID, Email and Profile
+- Allowed redirection addresses for login (`Options > Basic > Allowed
+  redirection addresses for login` ) :
+  `[synapse public baseurl]/_synapse/client/oidc/callback`

+Synapse config:
 ```yaml
-staticClients:
- id: synapse
-  secret: secret
-  redirectURIs:
-  - '[synapse public baseurl]/_synapse/client/oidc/callback'
-  name: 'Synapse'
+oidc_providers:
+  - idp_id: lemonldap
+    idp_name: lemonldap
+    discover: true
+    issuer: "https://auth.example.org/" # TO BE FILLED: replace with your domain
+    client_id: "your client id" # TO BE FILLED
+    client_secret: "your client secret" # TO BE FILLED
+    scopes:
+      - "openid"
+      - "profile"
+      - "email"
+    user_mapping_provider:
+      config:
+        localpart_template: "{{ user.preferred_username }}}"
+        # TO BE FILLED: If your users have names in LemonLDAP::NG and you want those in Synapse, this should be replaced with user.name|capitalize or any valid filter.
+        display_name_template: "{{ user.preferred_username|capitalize }}"
 ```

-Run with `dex serve examples/config-dev.yaml`.
+### GitHub
+
+[GitHub][github-idp] is a bit special as it is not an OpenID Connect compliant provider, but
+just a regular OAuth2 provider.
+
+The [`/user` API endpoint](https://developer.github.com/v3/users/#get-the-authenticated-user)
+can be used to retrieve information on the authenticated user. As the Synapse
+login mechanism needs an attribute to uniquely identify users, and that endpoint
+does not return a `sub` property, an alternative `subject_claim` has to be set.
+
+1. Create a new OAuth application: [https://github.com/settings/applications/new](https://github.com/settings/applications/new).
+2. Set the callback URL to `[synapse public baseurl]/_synapse/client/oidc/callback`.

 Synapse config:

 ```yaml
 oidc_providers:
-  - idp_id: dex
-    idp_name: "My Dex server"
-    skip_verification: true # This is needed as Dex is served on an insecure endpoint
-    issuer: "http://127.0.0.1:5556/dex"
-    client_id: "synapse"
-    client_secret: "secret"
-    scopes: ["openid", "profile"]
+  - idp_id: github
+    idp_name: Github
+    idp_brand: "github"  # optional: styling hint for clients
+    discover: false
+    issuer: "https://github.com/"
+    client_id: "your-client-id" # TO BE FILLED
+    client_secret: "your-client-secret" # TO BE FILLED
+    authorization_endpoint: "https://github.com/login/oauth/authorize"
+    token_endpoint: "https://github.com/login/oauth/access_token"
+    userinfo_endpoint: "https://api.github.com/user"
+    scopes: ["read:user"]
    user_mapping_provider:
      config:
-        localpart_template: "{{ user.name }}"
-        display_name_template: "{{ user.name|capitalize }}"
+        subject_claim: "id"
+        localpart_template: "{{ user.login }}"
+        display_name_template: "{{ user.name }}"
+```
+
+### Google
+
+[Google][google-idp] is an OpenID certified authentication and authorisation provider.
+
+1. Set up a project in the Google API Console (see 
+   [documentation](https://developers.google.com/identity/protocols/oauth2/openid-connect#appsetup)).
+3. Add an "OAuth Client ID" for a Web Application under "Credentials".
+4. Copy the Client ID and Client Secret, and add the following to your synapse config:
+   ```yaml
+   oidc_providers:
+     - idp_id: google
+       idp_name: Google
+       idp_brand: "google"  # optional: styling hint for clients
+       issuer: "https://accounts.google.com/"
+       client_id: "your-client-id" # TO BE FILLED
+       client_secret: "your-client-secret" # TO BE FILLED
+       scopes: ["openid", "profile", "email"] # email is optional, read below
+       user_mapping_provider:
+         config:
+           localpart_template: "{{ user.given_name|lower }}"
+           display_name_template: "{{ user.name }}"
+           email_template: "{{ user.email }}" # needs "email" in scopes above
+   ```
+4. Back in the Google console, add this Authorized redirect URI: `[synapse
+   public baseurl]/_synapse/client/oidc/callback`.
+
+### Twitch
+
+1. Setup a developer account on [Twitch](https://dev.twitch.tv/)
+2. Obtain the OAuth 2.0 credentials by [creating an app](https://dev.twitch.tv/console/apps/)
+3. Add this OAuth Redirect URL: `[synapse public baseurl]/_synapse/client/oidc/callback`
+
+Synapse config:
+
+```yaml
+oidc_providers:
+  - idp_id: twitch
+    idp_name: Twitch
+    issuer: "https://id.twitch.tv/oauth2/"
+    client_id: "your-client-id" # TO BE FILLED
+    client_secret: "your-client-secret" # TO BE FILLED
+    client_auth_method: "client_secret_post"
+    user_mapping_provider:
+      config:
+        localpart_template: "{{ user.preferred_username }}"
+        display_name_template: "{{ user.name }}"
+```
+
+### GitLab
+
+1. Create a [new application](https://gitlab.com/profile/applications).
+2. Add the `read_user` and `openid` scopes.
+3. Add this Callback URL: `[synapse public baseurl]/_synapse/client/oidc/callback`
+
+Synapse config:
+
+```yaml
+oidc_providers:
+  - idp_id: gitlab
+    idp_name: Gitlab
+    idp_brand: "gitlab"  # optional: styling hint for clients
+    issuer: "https://gitlab.com/"
+    client_id: "your-client-id" # TO BE FILLED
+    client_secret: "your-client-secret" # TO BE FILLED
+    client_auth_method: "client_secret_post"
+    scopes: ["openid", "read_user"]
+    user_profile_method: "userinfo_endpoint"
+    user_mapping_provider:
+      config:
+        localpart_template: '{{ user.nickname }}'
+        display_name_template: '{{ user.name }}'
+```
+
+### Facebook
+
+0. You will need a Facebook developer account. You can register for one
+   [here](https://developers.facebook.com/async/registration/).
+1. On the [apps](https://developers.facebook.com/apps/) page of the developer
+   console, "Create App", and choose "Build Connected Experiences".
+2. Once the app is created, add "Facebook Login" and choose "Web". You don't
+   need to go through the whole form here.
+3. In the left-hand menu, open "Products"/"Facebook Login"/"Settings".
+   * Add `[synapse public baseurl]/_synapse/client/oidc/callback` as an OAuth Redirect
+     URL.
+4. In the left-hand menu, open "Settings/Basic". Here you can copy the "App ID"
+   and "App Secret" for use below.
+
+Synapse config:
+
+```yaml
+  - idp_id: facebook
+    idp_name: Facebook
+    idp_brand: "facebook"  # optional: styling hint for clients
+    discover: false
+    issuer: "https://www.facebook.com"
+    client_id: "your-client-id" # TO BE FILLED
+    client_secret: "your-client-secret" # TO BE FILLED
+    scopes: ["openid", "email"]
+    authorization_endpoint: "https://facebook.com/dialog/oauth"
+    token_endpoint: "https://graph.facebook.com/v9.0/oauth/access_token"
+    jwks_uri: "https://www.facebook.com/.well-known/oauth/openid/jwks/"
+    user_mapping_provider:
+      config:
+        display_name_template: "{{ user.name }}"
+        email_template: "{{ user.email }}"
+```
+
+Relevant documents:
+ * [Manually Build a Login Flow](https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow)
+ * [Using Facebook's Graph API](https://developers.facebook.com/docs/graph-api/using-graph-api/)
+ * [Reference to the User endpoint](https://developers.facebook.com/docs/graph-api/reference/user)
+
+Facebook do have an [OIDC discovery endpoint](https://www.facebook.com/.well-known/openid-configuration),
+but it has a `response_types_supported` which excludes "code" (which we rely on, and
+is even mentioned in their [documentation](https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow#login)),
+so we have to disable discovery and configure the URIs manually.
+
+### Gitea
+
+Gitea is, like Github, not an OpenID provider, but just an OAuth2 provider.
+
+The [`/user` API endpoint](https://try.gitea.io/api/swagger#/user/userGetCurrent)
+can be used to retrieve information on the authenticated user. As the Synapse
+login mechanism needs an attribute to uniquely identify users, and that endpoint
+does not return a `sub` property, an alternative `subject_claim` has to be set.
+
+1. Create a new application.
+2. Add this Callback URL: `[synapse public baseurl]/_synapse/client/oidc/callback`
+
+Synapse config:
+
+```yaml
+oidc_providers:
+  - idp_id: gitea
+    idp_name: Gitea
+    discover: false
+    issuer: "https://your-gitea.com/"
+    client_id: "your-client-id" # TO BE FILLED
+    client_secret: "your-client-secret" # TO BE FILLED
+    client_auth_method: client_secret_post
+    scopes: [] # Gitea doesn't support Scopes
+    authorization_endpoint: "https://your-gitea.com/login/oauth/authorize"
+    token_endpoint: "https://your-gitea.com/login/oauth/access_token"
+    userinfo_endpoint: "https://your-gitea.com/api/v1/user"
+    user_mapping_provider:
+      config:
+        subject_claim: "id"
+        localpart_template: "{{ user.login }}"
+        display_name_template: "{{ user.full_name }}"
+```
+
+### XWiki
+
+Install [OpenID Connect Provider](https://extensions.xwiki.org/xwiki/bin/view/Extension/OpenID%20Connect/OpenID%20Connect%20Provider/) extension in your [XWiki](https://www.xwiki.org) instance.
+
+Synapse config:
+
+```yaml
+oidc_providers:
+  - idp_id: xwiki
+    idp_name: "XWiki"
+    issuer: "https://myxwikihost/xwiki/oidc/"
+    client_id: "your-client-id" # TO BE FILLED
+    client_auth_method: none
+    scopes: ["openid", "profile"]
+    user_profile_method: "userinfo_endpoint"
+    user_mapping_provider:
+      config:
+        localpart_template: "{{ user.preferred_username }}"
+        display_name_template: "{{ user.name }}"
+```
+
+### Apple
+
+Configuring "Sign in with Apple" (SiWA) requires an Apple Developer account.
+
+You will need to create a new "Services ID" for SiWA, and create and download a
+private key with "SiWA" enabled.
+
+As well as the private key file, you will need:
+ * Client ID: the "identifier" you gave the "Services ID"
+ * Team ID: a 10-character ID associated with your developer account.
+ * Key ID: the 10-character identifier for the key.
+
+[Apple's developer documentation](https://help.apple.com/developer-account/?lang=en#/dev77c875b7e)
+has more information on setting up SiWA.
+
+The synapse config will look like this:
+
+```yaml
+  - idp_id: apple
+    idp_name: Apple
+    issuer: "https://appleid.apple.com"
+    client_id: "your-client-id" # Set to the "identifier" for your "ServicesID"
+    client_auth_method: "client_secret_post"
+    client_secret_jwt_key:
+      key_file: "/path/to/AuthKey_KEYIDCODE.p8"  # point to your key file
+      jwt_header:
+        alg: ES256
+        kid: "KEYIDCODE"   # Set to the 10-char Key ID
+      jwt_payload:
+        iss: TEAMIDCODE    # Set to the 10-char Team ID
+    scopes: ["name", "email", "openid"]
+    authorization_endpoint: https://appleid.apple.com/auth/authorize?response_mode=form_post
+    user_mapping_provider:
+      config:
+        email_template: "{{ user.email }}"
 ```

 ### Django OAuth Toolkit
@@ -292,263 +591,6 @@ oidc_providers:
        email_template: "{{ user.email }}"
 ```

-### Facebook
-
-0. You will need a Facebook developer account. You can register for one
-   [here](https://developers.facebook.com/async/registration/).
-1. On the [apps](https://developers.facebook.com/apps/) page of the developer
-   console, "Create App", and choose "Build Connected Experiences".
-2. Once the app is created, add "Facebook Login" and choose "Web". You don't
-   need to go through the whole form here.
-3. In the left-hand menu, open "Products"/"Facebook Login"/"Settings".
-   * Add `[synapse public baseurl]/_synapse/client/oidc/callback` as an OAuth Redirect
-     URL.
-4. In the left-hand menu, open "Settings/Basic". Here you can copy the "App ID"
-   and "App Secret" for use below.
-
-Synapse config:
-
-```yaml
-  - idp_id: facebook
-    idp_name: Facebook
-    idp_brand: "facebook"  # optional: styling hint for clients
-    discover: false
-    issuer: "https://www.facebook.com"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    scopes: ["openid", "email"]
-    authorization_endpoint: "https://facebook.com/dialog/oauth"
-    token_endpoint: "https://graph.facebook.com/v9.0/oauth/access_token"
-    jwks_uri: "https://www.facebook.com/.well-known/oauth/openid/jwks/"
-    user_mapping_provider:
-      config:
-        display_name_template: "{{ user.name }}"
-        email_template: "{{ user.email }}"
-```
-
-Relevant documents:
- * [Manually Build a Login Flow](https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow)
- * [Using Facebook's Graph API](https://developers.facebook.com/docs/graph-api/using-graph-api/)
- * [Reference to the User endpoint](https://developers.facebook.com/docs/graph-api/reference/user)
-
-Facebook do have an [OIDC discovery endpoint](https://www.facebook.com/.well-known/openid-configuration),
-but it has a `response_types_supported` which excludes "code" (which we rely on, and
-is even mentioned in their [documentation](https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow#login)),
-so we have to disable discovery and configure the URIs manually.
-
-### GitHub
-
-[GitHub][github-idp] is a bit special as it is not an OpenID Connect compliant provider, but
-just a regular OAuth2 provider.
-
-The [`/user` API endpoint](https://developer.github.com/v3/users/#get-the-authenticated-user)
-can be used to retrieve information on the authenticated user. As the Synapse
-login mechanism needs an attribute to uniquely identify users, and that endpoint
-does not return a `sub` property, an alternative `subject_claim` has to be set.
-
-1. Create a new OAuth application: [https://github.com/settings/applications/new](https://github.com/settings/applications/new).
-2. Set the callback URL to `[synapse public baseurl]/_synapse/client/oidc/callback`.
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: github
-    idp_name: Github
-    idp_brand: "github"  # optional: styling hint for clients
-    discover: false
-    issuer: "https://github.com/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    authorization_endpoint: "https://github.com/login/oauth/authorize"
-    token_endpoint: "https://github.com/login/oauth/access_token"
-    userinfo_endpoint: "https://api.github.com/user"
-    scopes: ["read:user"]
-    user_mapping_provider:
-      config:
-        subject_claim: "id"
-        localpart_template: "{{ user.login }}"
-        display_name_template: "{{ user.name }}"
-```
-
-### GitLab
-
-1. Create a [new application](https://gitlab.com/profile/applications).
-2. Add the `read_user` and `openid` scopes.
-3. Add this Callback URL: `[synapse public baseurl]/_synapse/client/oidc/callback`
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: gitlab
-    idp_name: Gitlab
-    idp_brand: "gitlab"  # optional: styling hint for clients
-    issuer: "https://gitlab.com/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    client_auth_method: "client_secret_post"
-    scopes: ["openid", "read_user"]
-    user_profile_method: "userinfo_endpoint"
-    user_mapping_provider:
-      config:
-        localpart_template: '{{ user.nickname }}'
-        display_name_template: '{{ user.name }}'
-```
-
-### Gitea
-
-Gitea is, like Github, not an OpenID provider, but just an OAuth2 provider.
-
-The [`/user` API endpoint](https://try.gitea.io/api/swagger#/user/userGetCurrent)
-can be used to retrieve information on the authenticated user. As the Synapse
-login mechanism needs an attribute to uniquely identify users, and that endpoint
-does not return a `sub` property, an alternative `subject_claim` has to be set.
-
-1. Create a new application.
-2. Add this Callback URL: `[synapse public baseurl]/_synapse/client/oidc/callback`
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: gitea
-    idp_name: Gitea
-    discover: false
-    issuer: "https://your-gitea.com/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    client_auth_method: client_secret_post
-    scopes: [] # Gitea doesn't support Scopes
-    authorization_endpoint: "https://your-gitea.com/login/oauth/authorize"
-    token_endpoint: "https://your-gitea.com/login/oauth/access_token"
-    userinfo_endpoint: "https://your-gitea.com/api/v1/user"
-    user_mapping_provider:
-      config:
-        subject_claim: "id"
-        localpart_template: "{{ user.login }}"
-        display_name_template: "{{ user.full_name }}"
-```
-
-### Google
-
-[Google][google-idp] is an OpenID certified authentication and authorisation provider.
-
-1. Set up a project in the Google API Console (see 
-   [documentation](https://developers.google.com/identity/protocols/oauth2/openid-connect#appsetup)).
-3. Add an "OAuth Client ID" for a Web Application under "Credentials".
-4. Copy the Client ID and Client Secret, and add the following to your synapse config:
-   ```yaml
-   oidc_providers:
-     - idp_id: google
-       idp_name: Google
-       idp_brand: "google"  # optional: styling hint for clients
-       issuer: "https://accounts.google.com/"
-       client_id: "your-client-id" # TO BE FILLED
-       client_secret: "your-client-secret" # TO BE FILLED
-       scopes: ["openid", "profile", "email"] # email is optional, read below
-       user_mapping_provider:
-         config:
-           localpart_template: "{{ user.given_name|lower }}"
-           display_name_template: "{{ user.name }}"
-           email_template: "{{ user.email }}" # needs "email" in scopes above
-   ```
-4. Back in the Google console, add this Authorized redirect URI: `[synapse
-   public baseurl]/_synapse/client/oidc/callback`.
-
-### Keycloak
-
-[Keycloak][keycloak-idp] is an opensource IdP maintained by Red Hat.
-
-Keycloak supports OIDC Back-Channel Logout, which sends logout notification to Synapse, so that Synapse users get logged out when they log out from Keycloak.
-This can be optionally enabled by setting `backchannel_logout_enabled` to `true` in the Synapse configuration, and by setting the "Backchannel Logout URL" in Keycloak.
-
-Follow the [Getting Started Guide](https://www.keycloak.org/guides) to install Keycloak and set up a realm.
-
-1. Click `Clients` in the sidebar and click `Create`
-
-2. Fill in the fields as below:
-
-| Field | Value |
-|-----------|-----------|
-| Client ID | `synapse` |
-| Client Protocol | `openid-connect` |
-
-3. Click `Save`
-4. Fill in the fields as below:
-
-| Field | Value |
-|-----------|-----------|
-| Client ID | `synapse` |
-| Enabled | `On` |
-| Client Protocol | `openid-connect` |
-| Access Type | `confidential` |
-| Valid Redirect URIs | `[synapse public baseurl]/_synapse/client/oidc/callback` |
-| Backchannel Logout URL (optional) | `[synapse public baseurl]/_synapse/client/oidc/backchannel_logout` |
-| Backchannel Logout Session Required (optional) | `On` |
-
-5. Click `Save`
-6. On the Credentials tab, update the fields:
-
-| Field | Value |
-|-------|-------|
-| Client Authenticator | `Client ID and Secret` |
-
-7. Click `Regenerate Secret`
-8. Copy Secret
-
-```yaml
-oidc_providers:
-  - idp_id: keycloak
-    idp_name: "My KeyCloak server"
-    issuer: "https://127.0.0.1:8443/realms/{realm_name}"
-    client_id: "synapse"
-    client_secret: "copy secret generated from above"
-    scopes: ["openid", "profile"]
-    user_mapping_provider:
-      config:
-        localpart_template: "{{ user.preferred_username }}"
-        display_name_template: "{{ user.name }}"
-    backchannel_logout_enabled: true # Optional
-```
-
-### LemonLDAP
-
-[LemonLDAP::NG][lemonldap] is an open-source IdP solution.
-
-1. Create an OpenID Connect Relying Parties in LemonLDAP::NG
-2. The parameters are:
- Client ID under the basic menu of the new Relying Parties (`Options > Basic >
-  Client ID`)
- Client secret (`Options > Basic > Client secret`)
- JWT Algorithm: RS256 within the security menu of the new Relying Parties
-  (`Options > Security > ID Token signature algorithm` and `Options > Security >
-  Access Token signature algorithm`)
- Scopes: OpenID, Email and Profile
- Allowed redirection addresses for login (`Options > Basic > Allowed
-  redirection addresses for login` ) :
-  `[synapse public baseurl]/_synapse/client/oidc/callback`
-
-Synapse config:
-```yaml
-oidc_providers:
-  - idp_id: lemonldap
-    idp_name: lemonldap
-    discover: true
-    issuer: "https://auth.example.org/" # TO BE FILLED: replace with your domain
-    client_id: "your client id" # TO BE FILLED
-    client_secret: "your client secret" # TO BE FILLED
-    scopes:
-      - "openid"
-      - "profile"
-      - "email"
-    user_mapping_provider:
-      config:
-        localpart_template: "{{ user.preferred_username }}}"
-        # TO BE FILLED: If your users have names in LemonLDAP::NG and you want those in Synapse, this should be replaced with user.name|capitalize or any valid filter.
-        display_name_template: "{{ user.preferred_username|capitalize }}"
-```
-
 ### Mastodon

 [Mastodon](https://docs.joinmastodon.org/) instances provide an [OAuth API](https://docs.joinmastodon.org/spec/oauth/), allowing those instances to be used as a single sign-on provider for Synapse.
@@ -589,81 +631,3 @@ oidc_providers:
 ```

 Note that the fields `client_id` and `client_secret` are taken from the CURL response above.
-
-### Twitch
-
-1. Setup a developer account on [Twitch](https://dev.twitch.tv/)
-2. Obtain the OAuth 2.0 credentials by [creating an app](https://dev.twitch.tv/console/apps/)
-3. Add this OAuth Redirect URL: `[synapse public baseurl]/_synapse/client/oidc/callback`
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: twitch
-    idp_name: Twitch
-    issuer: "https://id.twitch.tv/oauth2/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    client_auth_method: "client_secret_post"
-    user_mapping_provider:
-      config:
-        localpart_template: "{{ user.preferred_username }}"
-        display_name_template: "{{ user.name }}"
-```
-
-### Twitter
-
-*Using Twitter as an identity provider requires using Synapse 1.75.0 or later.*
-
-1. Setup a developer account on [Twitter](https://developer.twitter.com/en/portal/dashboard)
-2. Create a project & app.
-3. Enable user authentication and under "Type of App" choose "Web App, Automated App or Bot".
-4. Under "App info" set the callback URL to `[synapse public baseurl]/_synapse/client/oidc/callback`.
-5. Obtain the OAuth 2.0 credentials under the "Keys and tokens" tab, copy the "OAuth 2.0 Client ID and Client Secret"
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: twitter
-    idp_name: Twitter
-    idp_brand: "twitter"  # optional: styling hint for clients
-    discover: false  # Twitter is not OpenID compliant.
-    issuer: "https://twitter.com/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    pkce_method: "always"
-    # offline.access providers refresh tokens, tweet.read and users.read needed for userinfo request.
-    scopes: ["offline.access", "tweet.read", "users.read"]
-    authorization_endpoint: https://twitter.com/i/oauth2/authorize
-    token_endpoint: https://api.twitter.com/2/oauth2/token
-    userinfo_endpoint: https://api.twitter.com/2/users/me?user.fields=profile_image_url
-    user_mapping_provider:
-      config:
-        subject_template: "{{ user.data.id }}"
-        localpart_template: "{{ user.data.username }}"
-        display_name_template: "{{ user.data.name }}"
-        picture_template: "{{ user.data.profile_image_url }}"
-```
-
-### XWiki
-
-Install [OpenID Connect Provider](https://extensions.xwiki.org/xwiki/bin/view/Extension/OpenID%20Connect/OpenID%20Connect%20Provider/) extension in your [XWiki](https://www.xwiki.org) instance.
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: xwiki
-    idp_name: "XWiki"
-    issuer: "https://myxwikihost/xwiki/oidc/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_auth_method: none
-    scopes: ["openid", "profile"]
-    user_profile_method: "userinfo_endpoint"
-    user_mapping_provider:
-      config:
-        localpart_template: "{{ user.preferred_username }}"
-        display_name_template: "{{ user.name }}"
-```
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Mathieu Velten	4dc7b444bd	Fix import order	2022-12-13 10:25:19 +01:00
Mathieu Velten	6073c0ecb1	Adress comments	2022-12-12 23:31:09 +01:00
Mathieu Velten	4ccade636e	Merge remote-tracking branch 'origin/develop' into mv/unbind-callback	2022-12-12 17:48:27 +01:00
Mathieu Velten	5ff0ba261c	Merge remote-tracking branch 'origin/develop' into mv/unbind-callback	2022-11-28 12:57:35 +01:00
Mathieu Velten	9b4c0e79d8	Adress comments	2022-08-03 16:24:44 +02:00
Mathieu Velten	97f991ed2e	Apply suggestions from code review Co-authored-by: Brendan Abolivier <babolivier@matrix.org>	2022-08-03 14:52:20 +02:00
Mathieu Velten	0274a7f2f5	Add 3pid unbind callback to module API	2022-08-03 14:52:20 +02:00
				`@@ -0,0 +1 @@`
				`Add a module callback for unbinding a 3PID.`
				`@@ -0,0 +1 @@`
				`Optimise push badge count calculations. Contributed by Nick @ Beeper (@fizzadar).`
				`@@ -0,0 +1 @@`
				`Fix a long-standing bug where a device list update might not be sent to clients in certain circumstances.`
				`@@ -0,0 +1 @@`
				`Improve user search for international display names.`
				`@@ -0,0 +1 @@`
				`Faster remote room joins: stream the un-partial-stating of rooms over replication.`
				`@@ -0,0 +1 @@`
				Stop using deprecated `keyIds` parameter when calling `/_matrix/key/v2/server`.
				`@@ -0,0 +1 @@`
				Update worker settings for `pusher` and `federation_sender` functionality.
				`@@ -0,0 +1 @@`
				`Add links to third party package repositories, and point to the bug which highlights Ubuntu's out-of-date packages.`
				`@@ -0,0 +1 @@`
				Share the `ClientRestResource` for both workers and the main process.