Dirty duplication to make all work

poetry.lock

@@ -1565,7 +1565,7 @@ docs = ["sphinx", "repoze.sphinx.autointerface"]
 test = ["zope.i18nmessageid", "zope.testing", "zope.testrunner"]
 
 [extras]
-all = ["matrix-synapse-ldap3", "opentracing"]
+all = ["matrix-synapse-ldap3", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pysaml2", "authlib", "lxml", "sentry-sdk", "jaeger-client", "opentracing", "pyjwt", "txredisapi", "hiredis"]
 cache_memory = ["Pympler"]
 jwt = ["pyjwt"]
 matrix-synapse-ldap3 = ["matrix-synapse-ldap3"]
@@ -1581,7 +1581,7 @@ url_preview = ["lxml"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.7"
-content-hash = "f943431a14333dcd7eccd53c940b86e2aea1a80ba4c2be0427f9923891754f7b"
+content-hash = "568b9ec860636ec456031d3b424a67bf8de3121c292a81818a2f259757fe5236"
 
 [metadata.files]
 appdirs = [

pyproject.toml

@@ -188,6 +188,8 @@ Pympler = { version = "*", optional = true }
 ## Developer dependencies should not get included in "all".
 ##
 [tool.poetry.extras]
+# NB: Packages that should be part of `pip install matrix-synapse[all]` need to be specified
+# twice: once here, and once in the `all` extra.
 matrix-synapse-ldap3 = ["matrix-synapse-ldap3"]
 postgres = ["psycopg2", "psycopg2cffi", "psycopg2cffi-compat"]
 saml2 = ["pysaml2"]
@@ -206,9 +208,39 @@ redis = ["txredisapi", "hiredis"]
 # Required to use experimental `caches.track_memory_usage` config option.
 cache_memory = ["pympler"]
 
-# TODO: is there a better way to avoid the duplication here?
-# See also https://github.com/python-poetry/poetry/issues/3413#issue-750319958
-all = ["matrix-synapse-ldap3", "postgres", "saml2", "oidc", "systemd", "url_preview", "sentry", "opentracing", "jwt", "redis", "cache_memory"]
+# The duplication here is awful. I hate hate hate hate hate it. However, for now I want
+# to ensure you can still `pip install matrix-synapse[all]` like today. Two motivations:
+# 1) for new installations, I want instructions in existing documentation and tutorials
+#    out there to still work.
+# 2) I don't want to hard-code a list of extras into CI if I can help it. The ideal
+#    solution here would be something like https://github.com/python-poetry/poetry/issues/3413
+# Poetry 1.2's dependency groups might make this easier. But I'm not trying that out
+# until there's a stable release of 1.2.
+#
+# NB: the strings in this list must be *package* names, not extra names.
+# Some of our extra names _are_ package names, which can lead to great confusion.
+all = [
+    # matrix-synapse-ldap3
+    "matrix-synapse-ldap3",
+    # postgres
+    "psycopg2", "psycopg2cffi", "psycopg2cffi-compat",
+    # saml2
+    "pysaml2",
+    # oidc
+    "authlib",
+    # systemd, omitted: this is a system-based requirement
+    # url_preview
+    "lxml",
+    # sentry
+    "sentry-sdk",
+    # opentracing
+    "jaeger-client", "opentracing",
+    # jwt
+    "pyjwt",
+    #redis
+    "txredisapi", "hiredis"
+    # cache_memory, omitted: this is an experimental option
+]
 
 [tool.poetry.dev-dependencies]
 ## We pin black so that our tests don't start failing on new releases.