Merge remote-tracking branch 'origin/develop' into dmr/stateres/debug

scripts-dev/build_debian_packages.py

@@ -26,7 +26,6 @@ DISTS = (
     "debian:bookworm",
     "debian:sid",
     "ubuntu:focal",  # 20.04 LTS (our EOL forced by Py38 on 2024-10-14)
-    "ubuntu:impish",  # 21.10  (EOL 2022-07)
     "ubuntu:jammy",  # 22.04 LTS (EOL 2027-04)
 )
 

scripts-dev/complement.sh

@@ -14,12 +14,18 @@
 # By default Synapse is run in monolith mode. This can be overridden by
 # setting the WORKERS environment variable.
 #
-# A regular expression of test method names can be supplied as the first
-# argument to the script. Complement will then only run those tests. If
-# no regex is supplied, all tests are run. For example;
+# You can optionally give a "-f" argument (for "fast") before any to skip
+# rebuilding the docker images, if you just want to rerun the tests.
+#
+# Remaining commandline arguments are passed through to `go test`. For example,
+# you can supply a regular expression of test method names via the "-run"
+# argument:
 #
 # ./complement.sh -run "TestOutboundFederation(Profile|Send)"
 #
+# Specifying TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 will cause `poetry export`
+# to not emit any hashes when building the Docker image. This then means that
+# you can use 'unverifiable' sources such as git repositories as dependencies.
 
 # Exit if a line returns a non-zero exit code
 set -e
@@ -32,6 +38,47 @@ echo_if_github() {
   fi
 }
 
+# Helper to print out the usage instructions
+usage() {
+    cat >&2 <<EOF
+Usage: $0 [-f] <go test arguments>...
+Run the complement test suite on Synapse.
+
+  -f, --fast
+        Skip rebuilding the docker images, and just use the most recent
+        'complement-synapse:latest' image.
+        Conflicts with --build-only.
+
+  --build-only
+        Only build the Docker images. Don't actually run Complement.
+        Conflicts with -f/--fast.
+
+For help on arguments to 'go test', run 'go help testflag'.
+EOF
+}
+
+# parse our arguments
+skip_docker_build=""
+skip_complement_run=""
+while [ $# -ge 1 ]; do
+    arg=$1
+    case "$arg" in
+        "-h")
+            usage
+            exit 1
+            ;;
+        "-f"|"--fast")
+            skip_docker_build=1
+            ;;
+        "--build-only")
+            skip_complement_run=1
+            ;;
+        *)
+            # unknown arg: presumably an argument to gotest. break the loop.
+            break
+    esac
+    shift
+done
 
 # enable buildkit for the docker builds
 export DOCKER_BUILDKIT=1
@@ -49,21 +96,30 @@ if [[ -z "$COMPLEMENT_DIR" ]]; then
   echo "Checkout available at 'complement-${COMPLEMENT_REF}'"
 fi
 
-# Build the base Synapse image from the local checkout
-echo_if_github "::group::Build Docker image: matrixdotorg/synapse"
-docker build -t matrixdotorg/synapse -f "docker/Dockerfile" .
-echo_if_github "::endgroup::"
+if [ -z "$skip_docker_build" ]; then
+    # Build the base Synapse image from the local checkout
+    echo_if_github "::group::Build Docker image: matrixdotorg/synapse"
+    docker build -t matrixdotorg/synapse \
+      --build-arg TEST_ONLY_SKIP_DEP_HASH_VERIFICATION \
+      -f "docker/Dockerfile" .
+    echo_if_github "::endgroup::"
 
-# Build the workers docker image (from the base Synapse image we just built).
-echo_if_github "::group::Build Docker image: matrixdotorg/synapse-workers"
-docker build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" .
-echo_if_github "::endgroup::"
+    # Build the workers docker image (from the base Synapse image we just built).
+    echo_if_github "::group::Build Docker image: matrixdotorg/synapse-workers"
+    docker build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" .
+    echo_if_github "::endgroup::"
 
-# Build the unified Complement image (from the worker Synapse image we just built).
-echo_if_github "::group::Build Docker image: complement/Dockerfile"
-docker build -t complement-synapse \
-  -f "docker/complement/Dockerfile" "docker/complement"
-echo_if_github "::endgroup::"
+    # Build the unified Complement image (from the worker Synapse image we just built).
+    echo_if_github "::group::Build Docker image: complement/Dockerfile"
+    docker build -t complement-synapse \
+           -f "docker/complement/Dockerfile" "docker/complement"
+    echo_if_github "::endgroup::"
+fi
+
+if [ -n "$skip_complement_run" ]; then
+    echo "Skipping Complement run as requested."
+    exit
+fi
 
 export COMPLEMENT_BASE_IMAGE=complement-synapse
 
@@ -104,6 +160,18 @@ else
   test_tags="$test_tags,faster_joins"
 fi
 
+
+if [[ -n "$SYNAPSE_TEST_LOG_LEVEL" ]]; then
+  # Set the log level to what is desired
+  export PASS_SYNAPSE_LOG_LEVEL="$SYNAPSE_TEST_LOG_LEVEL"
+
+  # Allow logging sensitive things (currently SQL queries & parameters).
+  # (This won't have any effect if we're not logging at DEBUG level overall.)
+  # Since this is just a test suite, this is fine and won't reveal anyone's
+  # personal information
+  export PASS_SYNAPSE_LOG_SENSITIVE=1
+fi
+
 # Run the tests!
 echo "Images built; running complement"
 cd "$COMPLEMENT_DIR"