Fix twisted trunk with sytest

Install twisted trunk into the correct virtual env and use offline mode
so that twisted trunk doesn't get overridden with the pinned version of
twisted.

Signed-off-by: Sean Quah <seanq@element.io>

.ci/patch_for_twisted_trunk.sh

@@ -1,8 +1,22 @@
 #!/bin/sh
 
-# replaces the dependency on Twisted in `python_dependencies` with trunk.
+# replaces the dependency on Twisted with Twisted's trunk.
 
-set -e
+set -xe
 cd "$(dirname "$0")"/..
 
-sed -i -e 's#"Twisted.*"#"Twisted @ git+https://github.com/twisted/twisted"#' synapse/python_dependencies.py
+# We could make full use of the poetry machinery (rather than just `pip install ...`)
+# so that we can catch dependency resolver problems that could arise from twisted
+# bumping its dependencies...
+# sed -ibackup -e 's!^Twisted = .*!Twisted = { git = "https://github.com/twisted/twisted.git", rev = "trunk" }!' pyproject.toml
+# poetry lock --no-update
+# poetry install --no-interaction --extras "all test"
+
+# ...except we run into https://github.com/python-poetry/poetry/issues/5311, where
+# poetry insists on installing an old version of treq, which isn't actually compatible
+# with recent twisted releases. So let's just install twisted trunk using pip.
+poetry install --no-interaction --extras "all test"
+poetry run pip install git+https://github.com/twisted/twisted.git@trunk
+
+# Confirm the version of twisted in use
+poetry run pip show twisted

.ci/scripts/test_export_data_command.sh

@@ -2,29 +2,23 @@
 
 # Test for the export-data admin command against sqlite and postgres
 
+# Expects Synapse to have been already installed with `poetry install -e .[postgres]
+
 set -xe
 cd "$(dirname "$0")/../.."
 
-echo "--- Install dependencies"
-
-# Install dependencies for this test.
-pip install psycopg2
-
-# Install Synapse itself. This won't update any libraries.
-pip install -e .
-
 echo "--- Generate the signing key"
 
 # Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
+poetry run synapse_homeserver --generate-keys -c .ci/sqlite-config.yaml
 
 echo "--- Prepare test database"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
-update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # Run the export-data command on the sqlite test database
-python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
+poetry run python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
 --output-directory /tmp/export_data
 
 # Test that the output directory exists and contains the rooms directory
@@ -37,14 +31,14 @@ else
 fi
 
 # Create the PostgreSQL database.
-.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
+poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
 # Port the SQLite databse to postgres so we can check command works against postgres
 echo "+++ Port SQLite3 databse to postgres"
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 # Run the export-data command on postgres database
-python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
+poetry run python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
 --output-directory /tmp/export_data2
 
 # Test that the output directory exists and contains the rooms directory

.ci/scripts/test_old_deps.sh

@@ -9,12 +9,66 @@ set -ex
 
 apt-get update
 apt-get install -y \
-        python3 python3-dev python3-pip python3-venv \
-        libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox libjpeg-dev libwebp-dev
+        python3 python3-dev python3-pip python3-venv pipx \
+        libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
 
 export LANG="C.UTF-8"
 
 # Prevent virtualenv from auto-updating pip to an incompatible version
 export VIRTUALENV_NO_DOWNLOAD=1
 
-exec tox -e py3-old
+# I'd prefer to use something like this
+#   https://github.com/python-poetry/poetry/issues/3527
+#   https://github.com/pypa/pip/issues/8085
+# rather than this sed script. But that's an Opinion.
+
+# patch the project definitions in-place
+# replace all lower bounds with exact bounds
+# delete all lines referring to psycopg2 --- so no postgres support
+# but make the pyopenssl 17.0, which can work against an
+# OpenSSL 1.1 compiled cryptography (as older ones don't compile on Travis).
+
+sed -i-backup \
+   -e "s/[~>]=/==/g" \
+   -e "/psycopg2/d" \
+   -e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \
+   -e '/psycopg2/d' \
+   -e '/systemd/d' \
+   pyproject.toml
+
+pip install --user toml
+
+REMOVE_DEV_DEPENDENCIES="
+import toml
+with open('pyproject.toml', 'r') as f:
+    data = toml.loads(f.read())
+
+from pprint import pprint
+pprint(data)
+
+del data['tool']['poetry']['dev-dependencies']
+
+with open('pyproject.toml', 'w') as f:
+    data = toml.dump(data, f)
+"
+python3 -c "$REMOVE_DEV_DEPENDENCIES"
+
+# There are almost certainly going to be dependency conflicts there, so I'm going to
+# use plain pip to install rather than poetry.
+
+# Can't pip install with -e. Error message:
+# > A "pyproject.toml" file was found, but editable mode currently requires a setup.py based build.
+# Needs PEP 660 support in poetry, sigh. See
+# https://github.com/python-poetry/poetry/issues/34#issuecomment-1055142428
+# So instead, make a virtualenv and install in there.
+
+# At the time of writing, this notes:
+#  ERROR: ldap3 2.9.1 has requirement pyasn1>=0.4.6, but you'll have pyasn1 0.1.9 which is incompatible.
+# Helpfully, pip doesn't indicate the error; it returns 0.
+# TODO: bump pyasn to >=0.4.6?
+pipx install poetry==1.1.12
+~/.local/bin/poetry run pip install .[all,test]
+
+# Run the tests. Since we didn't install in editable mode, we have to invoke trial
+# with `python -m` so that the working directory contains `tests/`.
+~/.local/bin/poetry run python -m twisted.trial -j2 tests

.ci/scripts/test_synapse_port_db.sh

@@ -5,39 +5,32 @@
 #   - runs the port script on a prepopulated test sqlite db
 #   - also runs it against an new sqlite db
 
+# Expects Synapse to have been already installed with `poetry install -e .[postgres]
 
 set -xe
 cd "$(dirname "$0")/../.."
 
-echo "--- Install dependencies"
-
-# Install dependencies for this test.
-pip install psycopg2 coverage coverage-enable-subprocess
-
-# Install Synapse itself. This won't update any libraries.
-pip install -e .
-
 echo "--- Generate the signing key"
 
 # Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
+poetry run synapse_homeserver --generate-keys -c .ci/sqlite-config.yaml
 
 echo "--- Prepare test database"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
-update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # Create the PostgreSQL database.
-.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
+poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against test database"
 # TODO: this invocation of synapse_port_db (and others below) used to be prepended with `coverage run`,
 # but coverage seems unable to find the entrypoints installed by `pip install -e .`.
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 # We should be able to run twice against the same database.
 echo "+++ Run synapse_port_db a second time"
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 #####
 
@@ -48,12 +41,12 @@ echo "--- Prepare empty SQLite database"
 # we do this by deleting the sqlite db, and then doing the same again.
 rm .ci/test_db.db
 
-update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # re-create the PostgreSQL database.
-.ci/scripts/postgres_exec.py \
+poetry run .ci/scripts/postgres_exec.py \
   "DROP DATABASE synapse" \
   "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against empty database"
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

.dockerignore

@@ -4,8 +4,8 @@
 # things to include
 !docker
 !synapse
-!MANIFEST.in
 !README.rst
-!setup.py
+!pyproject.toml
+!poetry.lock
 
 **/__pycache__

.github/workflows/tests.yml

@@ -20,20 +20,7 @@ jobs:
       - run: scripts-dev/config-lint.sh
 
   lint:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        toxenv:
-          - "check_codestyle"
-          - "check_isort"
-          - "mypy"
-          - "packaging"
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install tox
-      - run: tox -e ${{ matrix.toxenv }}
+    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
 
   lint-crlf:
     runs-on: ubuntu-latest
@@ -51,6 +38,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
       - uses: actions/setup-python@v2
+      # TODO: version guard comes from #3425. Unexplained. May as well add it to pyproject.toml tho.
       - run: "pip install 'towncrier>=18.6.0rc1'"
       - run: scripts-dev/check-newsfragment.sh
         env:
@@ -72,23 +60,23 @@ jobs:
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10"]
         database: ["sqlite"]
-        toxenv: ["py"]
+        extras: ["all"]
         include:
           # Newest Python without optional deps
           - python-version: "3.10"
-            toxenv: "py-noextras"
+            extras: ""
 
           # Oldest Python with PostgreSQL
           - python-version: "3.7"
             database: "postgres"
             postgres-version: "10"
-            toxenv: "py"
+            extras: "all"
 
           # Newest Python with newest PostgreSQL
           - python-version: "3.10"
             database: "postgres"
             postgres-version: "14"
-            toxenv: "py"
+            extras: "all"
 
     steps:
       - uses: actions/checkout@v2
@@ -100,17 +88,16 @@ jobs:
             -e POSTGRES_PASSWORD=postgres \
             -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
             postgres:${{ matrix.postgres-version }}
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@dmr/try-extras
         with:
           python-version: ${{ matrix.python-version }}
-      - run: pip install tox
+          extras: ${{ matrix.extras }}
       - name: Await PostgreSQL
         if: ${{ matrix.postgres-version }}
         timeout-minutes: 2
         run: until pg_isready -h localhost; do sleep 1; done
-      - run: tox -e ${{ matrix.toxenv }}
+      - run: poetry run trial --jobs=2 tests
         env:
-          TRIAL_FLAGS: "--jobs=2"
           SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
           SYNAPSE_POSTGRES_HOST: localhost
           SYNAPSE_POSTGRES_USER: postgres
@@ -129,18 +116,18 @@ jobs:
           || true
 
   trial-olddeps:
+    # Note: sqlite only; no postgres
     if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
+#    needs: linting-done
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - name: Test with old deps
         uses: docker://ubuntu:focal # For old python and sqlite
+        # Note: focal seems to be using 3.8, but the oldest is 3.7?
         with:
           workdir: /github/workspace
           entrypoint: .ci/scripts/test_old_deps.sh
-        env:
-          TRIAL_FLAGS: "--jobs=2"
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
         if: ${{ always() }}
@@ -156,23 +143,24 @@ jobs:
 
   trial-pypy:
     # Very slow; only run if the branch name includes 'pypy'
+    # Note: sqlite only; no postgres
     if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
     needs: linting-done
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["pypy-3.7"]
+        extras: ["all"]
 
     steps:
       - uses: actions/checkout@v2
+      # TODO: why is this list different from  above?
       - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@dmr/try-extras
         with:
           python-version: ${{ matrix.python-version }}
-      - run: pip install tox
-      - run: tox -e py
-        env:
-          TRIAL_FLAGS: "--jobs=2"
+          extras: ${{ matrix.extras }}
+      - run: poetry run trial --jobs=2 tests
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
         if: ${{ always() }}
@@ -271,9 +259,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@dmr/try-extras
         with:
-          python-version: "3.9"
+          python-version: ${{ matrix.python-version }}
+          extras: "postgres"
       - run: .ci/scripts/test_export_data_command.sh
 
   portdb:
@@ -308,9 +297,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@dmr/try-extras
         with:
           python-version: ${{ matrix.python-version }}
+          extras: "postgres"
       - run: .ci/scripts/test_synapse_port_db.sh
 
   complement:

.github/workflows/twisted_trunk.yml

@@ -5,6 +5,11 @@ on:
     - cron: 0 8 * * *
 
   workflow_dispatch:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   mypy:
@@ -12,10 +17,12 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@dmr/try-extras
+        with:
+          python-version: "3.x"
+          extras: "all"
       - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e mypy
+      - run: poetry run mypy
 
   trial:
     runs-on: ubuntu-latest
@@ -23,14 +30,12 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@dmr/try-extras
         with:
-          python-version: 3.7
+          python-version: "3.x"
+          extras: "all test"
       - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e py
-        env:
-          TRIAL_FLAGS: "--jobs=2"
+      - run: poetry run trial --jobs 2 tests
 
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
@@ -55,11 +60,18 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Patch dependencies
-        run: .ci/patch_for_twisted_trunk.sh
+        run: >-
+          ln -s -T /venv /src/.venv &&
+          .ci/patch_for_twisted_trunk.sh &&
+          unlink /src/.venv
         working-directory: /src
       - name: Run SyTest
         run: /bootstrap.sh synapse
         working-directory: /src
+        env:
+          # Use offline mode to avoid reinstalling the pinned version of
+          # twisted.
+          OFFLINE: 1
       - name: Summarise results.tap
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
@@ -72,21 +84,21 @@ jobs:
             /logs/results.tap
             /logs/**/*.log*
 
-  # open an issue if the build fails, so we know about it.
-  open-issue:
-    if: failure()
-    needs:
-      - mypy
-      - trial
-      - sytest
-
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          update_existing: true
-          filename: .ci/twisted_trunk_build_failed_issue_template.md
+#  # open an issue if the build fails, so we know about it.
+#  open-issue:
+#    if: failure()
+#    needs:
+#      - mypy
+#      - trial
+#      - sytest
+#
+#    runs-on: ubuntu-latest
+#
+#    steps:
+#      - uses: actions/checkout@v2
+#      - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06
+#        env:
+#          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+#        with:
+#          update_existing: true
+#          filename: .ci/twisted_trunk_build_failed_issue_template.md

.gitignore

@@ -15,6 +15,10 @@ _trial_temp*/
 .DS_Store
 __pycache__/
 
+# We do want the poetry lockfile. TODO: is there a good reason for ignoring
+# '*.lock' above? If not, let's nuke it.
+!poetry.lock
+
 # stuff that is likely to exist when you run a server locally
 /*.db
 /*.log
@@ -30,6 +34,9 @@ __pycache__/
 /media_store/
 /uploads
 
+# For direnv uses
+/.envrc
+
 # IDEs
 /.idea/
 /.ropeproject/

changelog.d/12049.misc

@@ -0,0 +1 @@
+Use poetry to manage Synapse's dependencies.

docker/Dockerfile

@@ -16,18 +16,31 @@
 
 ARG PYTHON_VERSION=3.9
 
+FROM docker.io/python:${PYTHON_VERSION}-slim as base
+
 ###
 ### Stage 0: builder
 ###
-FROM docker.io/python:${PYTHON_VERSION}-slim as builder
 
-# install the OS build deps
-#
+# Irritatingly, there is no blessed guide on how to distribute an application with its
+# poetry-managed environment in a docker image. For a while,
+# `poetry export | pip install -r /dev/stdin` seemed plausible but is limited by bugs
+# in `poetry export` whose fixes (scheduled for poetry 1.2) have yet to be released.
+# The best references I could find are
+#     https://github.com/python-poetry/poetry/discussions/1879#discussioncomment-216865
+#     https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker?answertab=scoredesc#tab-top
+FROM base as builder
+
 # RUN --mount is specific to buildkit and is documented at
 # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
-# Here we use it to set up a cache for apt, to improve rebuild speeds on
-# slow connections.
-#
+# Here we use it to set up a cache for pip (below, for apt and poetry), to improve
+# rebuild speeds on slow connections.
+# We install poetry as --user so that it doesn't end up in the system-wide python
+# installation. That gets copied later into the runtime image.
+RUN --mount=type=cache,target=/root/.cache/pip \
+  pip install poetry==1.1.12
+
+# install the OS build deps
 RUN \
    --mount=type=cache,target=/var/cache/apt,sharing=locked \
    --mount=type=cache,target=/var/lib/apt,sharing=locked \
@@ -45,33 +58,36 @@ RUN \
     zlib1g-dev \
     && rm -rf /var/lib/apt/lists/*
 
-# Copy just what we need to pip install
-COPY MANIFEST.in README.rst setup.py /synapse/
-COPY synapse/__init__.py /synapse/synapse/__init__.py
-COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py
+WORKDIR /synapse
 
+# Copy just what we need to poetry install
+COPY pyproject.toml poetry.lock README.rst /synapse/
+
+# Install to the Python installation which hosts `pip`. In this case, it's the system
+# Python.
+ENV POETRY_VIRTUALENVS_IN_PROJECT=true \
+    POETRY_VIRTUALENVS_CREATE=true \
+    POETRY_HOME=/opt/poetry
 # To speed up rebuilds, install all of the dependencies before we copy over
-# the whole synapse project so that we this layer in the Docker cache can be
+# the whole synapse project, so that this layer in the Docker cache can be
 # used while you develop on the source
-#
-# This is aiming at installing the `install_requires` and `extras_require` from `setup.py`
-RUN --mount=type=cache,target=/root/.cache/pip \
-  pip install --prefix="/install" --no-warn-script-location \
-    /synapse[all]
+RUN --mount=type=cache,target=/opt/poetry/artifacts \
+    --mount=type=cache,target=/opt/poetry/.cache/pypoetry/cache \
+  poetry install --no-dev --no-root --no-interaction --no-ansi --extras all
 
-# Copy over the rest of the project
+# Copy over the synapse source code.
 COPY synapse /synapse/synapse/
 
-# Install the synapse package itself and all of its children packages.
-#
-# This is aiming at installing only the `packages=find_packages(...)` from `setup.py
-RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
+# Install the synapse package itself, by omitting the --no-root argument
+RUN --mount=type=cache,target=/opt/poetry/artifacts \
+    --mount=type=cache,target=/opt/poetry/cache \
+  poetry install --no-dev --no-interaction --no-ansi --extras all
 
 ###
 ### Stage 1: runtime
 ###
 
-FROM docker.io/python:${PYTHON_VERSION}-slim
+FROM base
 
 LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
 LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'
@@ -93,7 +109,7 @@ RUN \
     openssl \
     && rm -rf /var/lib/apt/lists/*
 
-COPY --from=builder /install /usr/local
+COPY --from=builder /synapse/ /synapse
 COPY ./docker/start.py /start.py
 COPY ./docker/conf /conf
 

docker/start.py

@@ -1,4 +1,4 @@
-#!/usr/local/bin/python
+#!/synapse/.venv/bin/python
 
 import codecs
 import glob
@@ -9,6 +9,8 @@ import sys
 
 import jinja2
 
+VIRTUALENV_INTERPRETER = "/synapse/.venv/bin/python"
+
 
 # Utility functions
 def log(txt):
@@ -108,7 +110,7 @@ def generate_config_from_template(config_dir, config_path, environ, ownership):
 
     # Hopefully we already have a signing key, but generate one if not.
     args = [
-        "python",
+        VIRTUALENV_INTERPRETER,
         "-m",
         "synapse.app.homeserver",
         "--config-path",
@@ -158,7 +160,7 @@ def run_generate_config(environ, ownership):
 
     # generate the main config file, and a signing key.
     args = [
-        "python",
+        VIRTUALENV_INTERPRETER,
         "-m",
         "synapse.app.homeserver",
         "--server-name",
@@ -175,7 +177,7 @@ def run_generate_config(environ, ownership):
         "--open-private-ports",
     ]
     # log("running %s" % (args, ))
-    os.execv("/usr/local/bin/python", args)
+    os.execv(VIRTUALENV_INTERPRETER, args)
 
 
 def main(args, environ):
@@ -254,12 +256,12 @@ running with 'migrate_config'. See the README for more details.
 
     log("Starting synapse with args " + " ".join(args))
 
-    args = ["python"] + args
+    args = [VIRTUALENV_INTERPRETER] + args
     if ownership is not None:
         args = ["gosu", ownership] + args
         os.execve("/usr/sbin/gosu", args, environ)
     else:
-        os.execve("/usr/local/bin/python", args, environ)
+        os.execve(VIRTUALENV_INTERPRETER, args, environ)
 
 
 if __name__ == "__main__":

docs/code_style.md

@@ -24,7 +24,7 @@ pip install -e ".[lint,mypy]"
     functionality) with:
 
     ```sh
-    black . --exclude="\.tox|build|env"
+    black .
     ```
 
 -   **flake8**
@@ -35,7 +35,7 @@ pip install -e ".[lint,mypy]"
     Check all application and test code with:
 
     ```sh
-    flake8 synapse tests
+    flake8 .
     ```
 
 -   **isort**
@@ -46,11 +46,9 @@ pip install -e ".[lint,mypy]"
     Auto-fix imports with:
 
     ```sh
-    isort -rc synapse tests
+    isort .
     ```
 
-    `-rc` means to recursively search the given directories.
-
 It's worth noting that modern IDEs and text editors can run these tools
 automatically on save. It may be worth looking into whether this
 functionality is supported in your editor for a more convenient

mypy.ini

@@ -12,7 +12,6 @@ no_implicit_optional = True
 
 files =
   scripts-dev/,
-  setup.py,
   synapse/,
   tests/
 

pyproject.toml

@@ -36,24 +36,9 @@
 
 [tool.black]
 target-version = ['py37', 'py38', 'py39', 'py310']
-exclude = '''
-
-(
-  /(
-      \.eggs         # exclude a few common directories in the
-    | \.git          # root of the project
-    | \.tox
-    | \.venv
-    | \.env
-    | env
-    | _build
-    | _trial_temp.*
-    | build
-    | dist
-    | debian
-  )/
-)
-'''
+# black ignores everything in .gitignore by default, see
+# https://black.readthedocs.io/en/stable/usage_and_configuration/file_collection_and_discovery.html#gitignore
+# Use `extend-exclude` if you want to exclude something in addition to this.
 
 [tool.isort]
 line_length = 88
@@ -65,4 +50,229 @@ known_twisted = ["twisted", "OpenSSL"]
 multi_line_output = 3
 include_trailing_comma = true
 combine_as_imports = true
+skip_gitignore = true
 
+[tool.poetry]
+name = "matrix-synapse"
+version = "1.54.0rc1"
+description = "Homeserver for the Matrix decentralised comms protocol"
+authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
+license = "Apache-2.0"
+readme = "README.rst"
+repository = "https://github.com/matrix-org/synapse"
+packages = [
+    { include = "synapse" },
+]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Topic :: Communications :: Chat",
+]
+include = [
+    { path = "AUTHORS.rst", format = "sdist" },
+    { path = "book.toml", format = "sdist" },
+    { path = "changelog.d", format = "sdist" },
+    { path = "CHANGES.md", format = "sdist" },
+    { path = "CONTRIBUTING.md", format = "sdist" },
+    { path = "demo", format = "sdist" },
+    { path = "docs", format = "sdist" },
+    { path = "INSTALL.md", format = "sdist" },
+    { path = "mypy.ini", format = "sdist" },
+    { path = "scripts-dev", format = "sdist" },
+    { path = "synmark", format="sdist" },
+    { path = "sytest-blacklist", format = "sdist" },
+    { path = "tests", format = "sdist" },
+    { path = "tox.ini", format = "sdist" },
+    { path = "UPGRADE.rst", format = "sdist" },
+]
+
+[tool.poetry.scripts]
+synapse_homeserver = "synapse.app.homeserver:main"
+synapse_worker = "synapse.app.generic_worker:main"
+
+export_signing_key = "synapse._scripts.export_signing_key:main"
+generate_config = "synapse._scripts.generate_config:main"
+generate_log_config = "synapse._scripts.generate_log_config:main"
+generate_signing_key = "synapse._scripts.generate_signing_key:main"
+hash_password = "synapse._scripts.hash_password:main"
+register_new_matrix_user = "synapse._scripts.register_new_matrix_user:main"
+synapse_port_db = "synapse._scripts.synapse_port_db:main"
+synapse_review_recent_signups = "synapse._scripts.review_recent_signups:main"
+update_synapse_database = "synapse._scripts.update_synapse_database:main"
+
+# TODO add synctl
+
+[tool.poetry.dependencies]
+python = "^3.7"
+
+# Mandatory Dependencies
+# ----------------------
+# we use the TYPE_CHECKER.redefine method added in jsonschema 3.0.0
+jsonschema = ">=3.0.0"
+# frozendict 2.1.2 is broken on Debian 10: https://github.com/Marco-Sulla/python-frozendict/issues/41
+frozendict = ">=1,!=2.1.2"
+# We require 2.1.0 or higher for type hints. Previous guard was >= 1.1.0
+unpaddedbase64 = ">=2.1.0"
+canonicaljson = ">=1.4.0"
+# we use the type definitions added in signedjson 1.1.
+signedjson = ">=1.1.0"
+PyNaCl = ">=1.2.1"
+idna = ">=2.5"
+# validating SSL certs for IP addresses requires service_identity 18.1.
+service-identity = ">=18.1.0"
+# Twisted 18.9 introduces some logger improvements that the structured
+# logger utilises
+Twisted = ">=18.9.0"
+treq = ">=15.1"
+# Twisted has required pyopenssl 16.0 since about Twisted 16.6.
+pyOpenSSL = ">=16.0.0"
+PyYAML = ">=3.11"
+pyasn1 = ">=0.1.9"
+pyasn1-modules = ">=0.0.7"
+bcrypt = ">=3.1.0"
+Pillow = ">=5.4.0"
+sortedcontainers = ">=1.4.4"
+pymacaroons = ">=0.13.0"
+msgpack = ">=0.5.2"
+phonenumbers = ">=8.2.0"
+# we use GaugeHistogramMetric, which was added in prom-client 0.4.0.
+prometheus-client = ">=0.4.0"
+# we use `order`, which arrived in attrs 19.2.0.
+# Note: 21.1.0 broke `/sync`, see #9936
+attrs = ">=19.2.0,!=21.1.0"
+netaddr = ">=0.7.18"
+# We require 3.x or higher for type hints. Previous guard was >= 2.9.
+# Jinja2 3.1.0 removes the deprecated jinja2.Markup class, which we rely on.
+Jinja2 = ">=3.0.0,<3.1.0"
+bleach = ">=1.4.3"
+# We use `ParamSpec`, which was added in `typing-extensions` 3.10.0.0.
+typing-extensions = ">=3.10.0"
+# We enforce that we have a `cryptography` version that bundles an `openssl`
+# with the latest security patches.
+cryptography = ">=3.4.7"
+# ijson 3.1.4 fixes a bug with "." in property names
+ijson = ">=3.1.4"
+matrix-common = "~=1.1.0"
+# We need packaging.requirements.Requirement, added in 16.1.
+packaging = ">=16.1"
+
+
+# Optional Dependencies
+# ---------------------
+matrix-synapse-ldap3 = { version = ">=0.1", optional = true }
+psycopg2 = { version = ">=2.8", markers = "platform_python_implementation != 'PyPy'", optional = true }
+psycopg2cffi = { version = ">=2.8", markers = "platform_python_implementation == 'PyPy'", optional = true }
+psycopg2cffi-compat = { version = "==1.1", markers = "platform_python_implementation == 'PyPy'", optional = true }
+pysaml2 = { version = ">=4.5.0", optional = true }
+authlib = { version = ">=0.14.0", optional = true }
+# Note: systemd-python 231 appears to have been yanked from pypi
+systemd-python = { version = ">=231", optional = true }
+lxml = { version = ">=4.2.0", optional = true }
+sentry-sdk = { version = ">=0.7.2", optional = true }
+opentracing = { version = ">=2.2.0", optional = true }
+jaeger-client = { version = ">=4.0.0", optional = true }
+pyjwt = { version = ">=1.6.4", optional = true }
+txredisapi = { version = ">=1.4.7", optional = true }
+hiredis = { version = "*", optional = true }
+Pympler = { version = "*", optional = true }
+parameterized = { version = ">=0.7.4", optional = true }
+
+## Developer dependencies should not get included in "all".
+##
+[tool.poetry.extras]
+# NB: Packages that should be part of `pip install matrix-synapse[all]` need to be specified
+# twice: once here, and once in the `all` extra.
+matrix-synapse-ldap3 = ["matrix-synapse-ldap3"]
+postgres = ["psycopg2", "psycopg2cffi", "psycopg2cffi-compat"]
+saml2 = ["pysaml2"]
+oidc = ["authlib"]
+# systemd-python is necessary for logging to the systemd journal via
+# `systemd.journal.JournalHandler`, as is documented in
+# `contrib/systemd/log_config.yaml`.
+systemd = ["systemd-python"]
+url_preview = ["lxml"]
+sentry = ["sentry-sdk"]
+opentracing = ["jaeger-client", "opentracing"]
+jwt = ["pyjwt"]
+# hiredis is not a *strict* dependency, but it makes things much faster.
+# (if it is not installed, we fall back to slow code.)
+redis = ["txredisapi", "hiredis"]
+# Required to use experimental `caches.track_memory_usage` config option.
+cache_memory = ["pympler"]
+test = ["parameterized"]
+
+# The duplication here is awful. I hate hate hate hate hate it. However, for now I want
+# to ensure you can still `pip install matrix-synapse[all]` like today. Two motivations:
+# 1) for new installations, I want instructions in existing documentation and tutorials
+#    out there to still work.
+# 2) I don't want to hard-code a list of extras into CI if I can help it. The ideal
+#    solution here would be something like https://github.com/python-poetry/poetry/issues/3413
+# Poetry 1.2's dependency groups might make this easier. But I'm not trying that out
+# until there's a stable release of 1.2.
+#
+# NB: the strings in this list must be *package* names, not extra names.
+# Some of our extra names _are_ package names, which can lead to great confusion.
+all = [
+    # matrix-synapse-ldap3
+    "matrix-synapse-ldap3",
+    # postgres
+    "psycopg2", "psycopg2cffi", "psycopg2cffi-compat",
+    # saml2
+    "pysaml2",
+    # oidc
+    "authlib",
+    # systemd, omitted: this is a system-based requirement
+    # url_preview
+    "lxml",
+    # sentry
+    "sentry-sdk",
+    # opentracing
+    "jaeger-client", "opentracing",
+    # jwt
+    "pyjwt",
+    #redis
+    "txredisapi", "hiredis"
+    # cache_memory, omitted: this is an experimental option
+    # test, omitted: it's useful to have this separate from dev deps in the olddeps job
+]
+
+[tool.poetry.dev-dependencies]
+## We pin black so that our tests don't start failing on new releases.
+isort = "==5.7.0"
+black = "==21.12b0"
+flake8-comprehensions = "*"
+flake8-bugbear = "==21.3.2"
+flake8 = "*"
+
+# Typechecking
+mypy = "==0.931"
+mypy-zope = "==0.3.5"
+types-bleach = ">=4.1.0"
+types-jsonschema = ">=3.2.0"
+types-opentracing = ">=2.4.2"
+types-Pillow = ">=8.3.4"
+types-psycopg2 = ">=2.9.9"
+types-pyOpenSSL = ">=20.0.7"
+types-PyYAML = ">=5.4.10"
+types-requests = ">=2.26.0"
+types-setuptools = ">=57.4.0"
+
+# Dependencies which are exclusively required by unit test code. This is
+# NOT a list of all modules that are necessary to run the unit tests.
+# Tests assume that all optional dependencies are installed.
+# parameterized<0.7.4 can create classes with names that would normally be invalid
+# identifiers. trial really does not like this when running with multiple workers.
+parameterized = ">=0.7.4"
+
+# The following are used by the release script
+click = "==7.1.2"
+redbaron = "==0.9.2"
+GitPython = "==3.1.14"
+commonmark = "==0.9.1"
+pygithub = "==1.55"
+# The following are executed as commands by the release script.
+twine = "*"
+towncrier = "*"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

scripts-dev/lint.sh

@@ -80,13 +80,7 @@ else
   # then lint everything!
   if [[ -z ${files+x} ]]; then
     # Lint all source code files and directories
-    # Note: this list aims to mirror the one in tox.ini
-      files=(
-          "synapse" "docker" "tests"
-          # annoyingly, black doesn't find these so we have to list them
-          "scripts-dev"
-          "contrib" "setup.py" "synmark" "stubs" ".ci"
-      )
+      files=( "." )
   fi
 fi
 

setup.cfg

@@ -1,9 +0,0 @@
-[check-manifest]
-ignore =
-    .git-blame-ignore-revs
-    contrib
-    contrib/*
-    docs/*
-    pylint.cfg
-    tox.ini
-

synapse/__init__.py

@@ -20,6 +20,8 @@ import json
 import os
 import sys
 
+from matrix_common.versionstring import get_distribution_version_string
+
 # Check that we're not running on an unsupported Python version.
 if sys.version_info < (3, 7):
     print("Synapse requires Python 3.7 or above.")
@@ -68,11 +70,8 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.55.2"
+__version__ = get_distribution_version_string("matrix-synapse")
 
-if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
-    # We import here so that we don't have to install a bunch of deps when
-    # running the packaging tox test.
-    from synapse.util.patch_inline_callbacks import do_patch
+from synapse.util.patch_inline_callbacks import do_patch
 
-    do_patch()
+do_patch()

tox.ini

@@ -1,14 +1,14 @@
 [tox]
-envlist = packaging, py37, py38, py39, py310, check_codestyle, check_isort
+envlist = py37, py38, py39, py310
 
 # we require tox>=2.3.2 for the fix to https://github.com/tox-dev/tox/issues/208
-minversion = 2.3.2
+minversion = 3.3.0
 
 # the tox-venv plugin makes tox use python's built-in `venv` module rather than
 # the legacy `virtualenv` tool. `virtualenv` embeds its own `pip`, `setuptools`,
 # etc, and ends up being rather unreliable.
 requires = tox-venv
-
+isolated_build = true
 [base]
 deps =
     python-subunit
@@ -32,19 +32,6 @@ deps =
     # install the "enum34" dependency of cryptography.
     pip>=10
 
-# directories/files we run the linters on.
-# if you update this list, make sure to do the same in scripts-dev/lint.sh
-lint_targets =
-    setup.py
-    synapse
-    tests
-    # annoyingly, black doesn't find these so we have to list them
-    scripts-dev
-    stubs
-    contrib
-    synmark
-    .ci
-    docker
 
 # default settings for all tox environments
 [testenv]
@@ -122,10 +109,10 @@ commands =
     # Make all greater-thans equals so we test the oldest version of our direct
     # dependencies, but make the pyopenssl 17.0, which can work against an
     # OpenSSL 1.1 compiled cryptography (as older ones don't compile on Travis).
-    /bin/sh -c 'python -m synapse.python_dependencies | sed -e "s/>=/==/g" -e "/psycopg2/d" -e "s/pyopenssl==16.0.0/pyopenssl==17.0.0/" | xargs -d"\n" pip install'
+    /bin/sh 'sed -ibackup -e "s/>=/==/g" -e "/psycopg2/d" -e "s/pyopenssl==16.0.0/pyopenssl==17.0.0/" pyproject.toml'
 
     # Install Synapse itself. This won't update any libraries.
-    pip install -e ".[test]"
+    pip install ".[all, test]"
 
     {[testenv]commands}
 
@@ -138,26 +125,3 @@ setenv =
 commands =
     python -m synmark {posargs:}
 
-[testenv:packaging]
-skip_install = true
-usedevelop = false
-deps =
-    check-manifest
-commands =
-    check-manifest
-
-[testenv:check_codestyle]
-extras = lint
-commands =
-    python -m black --check --diff {[base]lint_targets}
-    flake8 {[base]lint_targets} {env:PEP8SUFFIX:}
-
-[testenv:check_isort]
-extras = lint
-commands = isort -c --df {[base]lint_targets}
-
-[testenv:mypy]
-deps =
-    {[base]deps}
-extras = all,mypy
-commands = mypy