diff --git a/changelog.d/19209.misc b/changelog.d/19209.misc new file mode 100644 index 0000000000..e64ca85d1d --- /dev/null +++ b/changelog.d/19209.misc @@ -0,0 +1 @@ +Refactor `scripts-dev/complement.sh` logic to avoid `exit` to facilitate being able to source it from other scripts (composable). diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index c4d678b142..adb5807325 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -72,153 +72,154 @@ For help on arguments to 'go test', run 'go help testflag'. EOF } -# parse our arguments -skip_docker_build="" -skip_complement_run="" -while [ $# -ge 1 ]; do +main() { + # parse our arguments + skip_docker_build="" + skip_complement_run="" + while [ $# -ge 1 ]; do arg=$1 case "$arg" in - "-h") - usage - exit 1 - ;; - "-f"|"--fast") - skip_docker_build=1 - ;; - "--build-only") - skip_complement_run=1 - ;; - "-e"|"--editable") - use_editable_synapse=1 - ;; - "--rebuild-editable") - rebuild_editable_synapse=1 - ;; - *) - # unknown arg: presumably an argument to gotest. break the loop. - break + "-h") + usage + return 1 + ;; + "-f"|"--fast") + skip_docker_build=1 + ;; + "--build-only") + skip_complement_run=1 + ;; + "-e"|"--editable") + use_editable_synapse=1 + ;; + "--rebuild-editable") + rebuild_editable_synapse=1 + ;; + *) + # unknown arg: presumably an argument to gotest. break the loop. + break esac shift -done + done -# enable buildkit for the docker builds -export DOCKER_BUILDKIT=1 + # enable buildkit for the docker builds + export DOCKER_BUILDKIT=1 -# Determine whether to use the docker or podman container runtime. -if [ -n "$PODMAN" ]; then - export CONTAINER_RUNTIME=podman - export DOCKER_HOST=unix://$XDG_RUNTIME_DIR/podman/podman.sock - export BUILDAH_FORMAT=docker - export COMPLEMENT_HOSTNAME_RUNNING_COMPLEMENT=host.containers.internal -else - export CONTAINER_RUNTIME=docker -fi + # Determine whether to use the docker or podman container runtime. + if [ -n "$PODMAN" ]; then + export CONTAINER_RUNTIME=podman + export DOCKER_HOST=unix://$XDG_RUNTIME_DIR/podman/podman.sock + export BUILDAH_FORMAT=docker + export COMPLEMENT_HOSTNAME_RUNNING_COMPLEMENT=host.containers.internal + else + export CONTAINER_RUNTIME=docker + fi -# Change to the repository root -cd "$(dirname $0)/.." + # Change to the repository root + cd "$(dirname $0)/.." -# Check for a user-specified Complement checkout -if [[ -z "$COMPLEMENT_DIR" ]]; then - COMPLEMENT_REF=${COMPLEMENT_REF:-main} - echo "COMPLEMENT_DIR not set. Fetching Complement checkout from ${COMPLEMENT_REF}..." - wget -Nq https://github.com/matrix-org/complement/archive/${COMPLEMENT_REF}.tar.gz - tar -xzf ${COMPLEMENT_REF}.tar.gz - COMPLEMENT_DIR=complement-${COMPLEMENT_REF} - echo "Checkout available at 'complement-${COMPLEMENT_REF}'" -fi + # Check for a user-specified Complement checkout + if [[ -z "$COMPLEMENT_DIR" ]]; then + COMPLEMENT_REF=${COMPLEMENT_REF:-main} + echo "COMPLEMENT_DIR not set. Fetching Complement checkout from ${COMPLEMENT_REF}..." + wget -Nq https://github.com/matrix-org/complement/archive/${COMPLEMENT_REF}.tar.gz + tar -xzf ${COMPLEMENT_REF}.tar.gz + COMPLEMENT_DIR=complement-${COMPLEMENT_REF} + echo "Checkout available at 'complement-${COMPLEMENT_REF}'" + fi -if [ -n "$use_editable_synapse" ]; then + if [ -n "$use_editable_synapse" ]; then if [[ -e synapse/synapse_rust.abi3.so ]]; then - # In an editable install, back up the host's compiled Rust module to prevent - # inconvenience; the container will overwrite the module with its own copy. - mv -n synapse/synapse_rust.abi3.so synapse/synapse_rust.abi3.so~host - # And restore it on exit: - synapse_pkg=`realpath synapse` - trap "mv -f '$synapse_pkg/synapse_rust.abi3.so~host' '$synapse_pkg/synapse_rust.abi3.so'" EXIT + # In an editable install, back up the host's compiled Rust module to prevent + # inconvenience; the container will overwrite the module with its own copy. + mv -n synapse/synapse_rust.abi3.so synapse/synapse_rust.abi3.so~host + # And restore it on exit: + synapse_pkg=`realpath synapse` + trap "mv -f '$synapse_pkg/synapse_rust.abi3.so~host' '$synapse_pkg/synapse_rust.abi3.so'" EXIT fi editable_mount="$(realpath .):/editable-src:z" if [ -n "$rebuild_editable_synapse" ]; then - unset skip_docker_build + unset skip_docker_build elif $CONTAINER_RUNTIME inspect complement-synapse-editable &>/dev/null; then - # complement-synapse-editable already exists: see if we can still use it: - # - The Rust module must still be importable; it will fail to import if the Rust source has changed. - # - The Poetry lock file must be the same (otherwise we assume dependencies have changed) + # complement-synapse-editable already exists: see if we can still use it: + # - The Rust module must still be importable; it will fail to import if the Rust source has changed. + # - The Poetry lock file must be the same (otherwise we assume dependencies have changed) - # First set up the module in the right place for an editable installation. - $CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'cp' complement-synapse-editable -- /synapse_rust.abi3.so.bak /editable-src/synapse/synapse_rust.abi3.so + # First set up the module in the right place for an editable installation. + $CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'cp' complement-synapse-editable -- /synapse_rust.abi3.so.bak /editable-src/synapse/synapse_rust.abi3.so - if ($CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'python' complement-synapse-editable -c 'import synapse.synapse_rust' \ - && $CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'diff' complement-synapse-editable --brief /editable-src/poetry.lock /poetry.lock.bak); then - skip_docker_build=1 - else - echo "Editable Synapse image is stale. Will rebuild." - unset skip_docker_build - fi + if ($CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'python' complement-synapse-editable -c 'import synapse.synapse_rust' \ + && $CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'diff' complement-synapse-editable --brief /editable-src/poetry.lock /poetry.lock.bak); then + skip_docker_build=1 + else + echo "Editable Synapse image is stale. Will rebuild." + unset skip_docker_build + fi fi -fi + fi -if [ -z "$skip_docker_build" ]; then + if [ -z "$skip_docker_build" ]; then if [ -n "$use_editable_synapse" ]; then - # Build a special image designed for use in development with editable - # installs. - $CONTAINER_RUNTIME build -t synapse-editable \ - -f "docker/editable.Dockerfile" . + # Build a special image designed for use in development with editable + # installs. + $CONTAINER_RUNTIME build -t synapse-editable \ + -f "docker/editable.Dockerfile" . - $CONTAINER_RUNTIME build -t synapse-workers-editable \ - --build-arg FROM=synapse-editable \ - -f "docker/Dockerfile-workers" . + $CONTAINER_RUNTIME build -t synapse-workers-editable \ + --build-arg FROM=synapse-editable \ + -f "docker/Dockerfile-workers" . - $CONTAINER_RUNTIME build -t complement-synapse-editable \ - --build-arg FROM=synapse-workers-editable \ - -f "docker/complement/Dockerfile" "docker/complement" + $CONTAINER_RUNTIME build -t complement-synapse-editable \ + --build-arg FROM=synapse-workers-editable \ + -f "docker/complement/Dockerfile" "docker/complement" - # Prepare the Rust module - $CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'cp' complement-synapse-editable -- /synapse_rust.abi3.so.bak /editable-src/synapse/synapse_rust.abi3.so + # Prepare the Rust module + $CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'cp' complement-synapse-editable -- /synapse_rust.abi3.so.bak /editable-src/synapse/synapse_rust.abi3.so else - # Build the base Synapse image from the local checkout - echo_if_github "::group::Build Docker image: matrixdotorg/synapse" - $CONTAINER_RUNTIME build -t matrixdotorg/synapse \ - --build-arg TEST_ONLY_SKIP_DEP_HASH_VERIFICATION \ - --build-arg TEST_ONLY_IGNORE_POETRY_LOCKFILE \ - -f "docker/Dockerfile" . - echo_if_github "::endgroup::" + # Build the base Synapse image from the local checkout + echo_if_github "::group::Build Docker image: matrixdotorg/synapse" + $CONTAINER_RUNTIME build -t matrixdotorg/synapse \ + --build-arg TEST_ONLY_SKIP_DEP_HASH_VERIFICATION \ + --build-arg TEST_ONLY_IGNORE_POETRY_LOCKFILE \ + -f "docker/Dockerfile" . + echo_if_github "::endgroup::" - # Build the workers docker image (from the base Synapse image we just built). - echo_if_github "::group::Build Docker image: matrixdotorg/synapse-workers" - $CONTAINER_RUNTIME build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" . - echo_if_github "::endgroup::" + # Build the workers docker image (from the base Synapse image we just built). + echo_if_github "::group::Build Docker image: matrixdotorg/synapse-workers" + $CONTAINER_RUNTIME build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" . + echo_if_github "::endgroup::" - # Build the unified Complement image (from the worker Synapse image we just built). - echo_if_github "::group::Build Docker image: complement/Dockerfile" - $CONTAINER_RUNTIME build -t complement-synapse \ - `# This is the tag we end up pushing to the registry (see` \ - `# .github/workflows/push_complement_image.yml) so let's just label it now` \ - `# so people can reference it by the same name locally.` \ - -t ghcr.io/element-hq/synapse/complement-synapse \ - -f "docker/complement/Dockerfile" "docker/complement" - echo_if_github "::endgroup::" + # Build the unified Complement image (from the worker Synapse image we just built). + echo_if_github "::group::Build Docker image: complement/Dockerfile" + $CONTAINER_RUNTIME build -t complement-synapse \ + `# This is the tag we end up pushing to the registry (see` \ + `# .github/workflows/push_complement_image.yml) so let's just label it now` \ + `# so people can reference it by the same name locally.` \ + -t ghcr.io/element-hq/synapse/complement-synapse \ + -f "docker/complement/Dockerfile" "docker/complement" + echo_if_github "::endgroup::" fi -fi + fi -if [ -n "$skip_complement_run" ]; then - echo "Skipping Complement run as requested." - exit -fi + if [ -n "$skip_complement_run" ]; then + echo "Skipping Complement run as requested." + return 0 + fi -export COMPLEMENT_BASE_IMAGE=complement-synapse -if [ -n "$use_editable_synapse" ]; then + export COMPLEMENT_BASE_IMAGE=complement-synapse + if [ -n "$use_editable_synapse" ]; then export COMPLEMENT_BASE_IMAGE=complement-synapse-editable export COMPLEMENT_HOST_MOUNTS="$editable_mount" -fi + fi -extra_test_args=() + extra_test_args=() -test_packages=( + test_packages=( ./tests/csapi ./tests ./tests/msc3874 @@ -231,71 +232,80 @@ test_packages=( ./tests/msc4140 ./tests/msc4155 ./tests/msc4306 -) + ) -# Enable dirty runs, so tests will reuse the same container where possible. -# This significantly speeds up tests, but increases the possibility of test pollution. -export COMPLEMENT_ENABLE_DIRTY_RUNS=1 + # Enable dirty runs, so tests will reuse the same container where possible. + # This significantly speeds up tests, but increases the possibility of test pollution. + export COMPLEMENT_ENABLE_DIRTY_RUNS=1 -# All environment variables starting with PASS_ will be shared. -# (The prefix is stripped off before reaching the container.) -export COMPLEMENT_SHARE_ENV_PREFIX=PASS_ + # All environment variables starting with PASS_ will be shared. + # (The prefix is stripped off before reaching the container.) + export COMPLEMENT_SHARE_ENV_PREFIX=PASS_ -# It takes longer than 10m to run the whole suite. -extra_test_args+=("-timeout=60m") + # It takes longer than 10m to run the whole suite. + extra_test_args+=("-timeout=60m") -if [[ -n "$WORKERS" ]]; then - # Use workers. - export PASS_SYNAPSE_COMPLEMENT_USE_WORKERS=true + if [[ -n "$WORKERS" ]]; then + # Use workers. + export PASS_SYNAPSE_COMPLEMENT_USE_WORKERS=true - # Pass through the workers defined. If none, it will be an empty string - export PASS_SYNAPSE_WORKER_TYPES="$WORKER_TYPES" + # Pass through the workers defined. If none, it will be an empty string + export PASS_SYNAPSE_WORKER_TYPES="$WORKER_TYPES" - # Workers can only use Postgres as a database. - export PASS_SYNAPSE_COMPLEMENT_DATABASE=postgres - - # And provide some more configuration to complement. - - # It can take quite a while to spin up a worker-mode Synapse for the first - # time (the main problem is that we start 14 python processes for each test, - # and complement likes to do two of them in parallel). - export COMPLEMENT_SPAWN_HS_TIMEOUT_SECS=120 -else - export PASS_SYNAPSE_COMPLEMENT_USE_WORKERS= - if [[ -n "$POSTGRES" ]]; then + # Workers can only use Postgres as a database. export PASS_SYNAPSE_COMPLEMENT_DATABASE=postgres + + # And provide some more configuration to complement. + + # It can take quite a while to spin up a worker-mode Synapse for the first + # time (the main problem is that we start 14 python processes for each test, + # and complement likes to do two of them in parallel). + export COMPLEMENT_SPAWN_HS_TIMEOUT_SECS=120 else - export PASS_SYNAPSE_COMPLEMENT_DATABASE=sqlite + export PASS_SYNAPSE_COMPLEMENT_USE_WORKERS= + if [[ -n "$POSTGRES" ]]; then + export PASS_SYNAPSE_COMPLEMENT_DATABASE=postgres + else + export PASS_SYNAPSE_COMPLEMENT_DATABASE=sqlite + fi fi + + if [[ -n "$ASYNCIO_REACTOR" ]]; then + # Enable the Twisted asyncio reactor + export PASS_SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR=true + fi + + if [[ -n "$UNIX_SOCKETS" ]]; then + # Enable full on Unix socket mode for Synapse, Redis and Postgresql + export PASS_SYNAPSE_USE_UNIX_SOCKET=1 + fi + + if [[ -n "$SYNAPSE_TEST_LOG_LEVEL" ]]; then + # Set the log level to what is desired + export PASS_SYNAPSE_LOG_LEVEL="$SYNAPSE_TEST_LOG_LEVEL" + + # Allow logging sensitive things (currently SQL queries & parameters). + # (This won't have any effect if we're not logging at DEBUG level overall.) + # Since this is just a test suite, this is fine and won't reveal anyone's + # personal information + export PASS_SYNAPSE_LOG_SENSITIVE=1 + fi + + # Log a few more useful things for a developer attempting to debug something + # particularly tricky. + export PASS_SYNAPSE_LOG_TESTING=1 + + # Run the tests! + echo "Images built; running complement with ${extra_test_args[@]} $@ ${test_packages[@]}" + cd "$COMPLEMENT_DIR" + + go test -v -tags "synapse_blacklist" -count=1 "${extra_test_args[@]}" "$@" "${test_packages[@]}" +} + +main "$@" +# For any non-zero exit code (indicating some sort of error happened), we want to exit +# with that code. +exit_code=$? +if [ $exit_code -ne 0 ]; then + exit $exit_code fi - -if [[ -n "$ASYNCIO_REACTOR" ]]; then - # Enable the Twisted asyncio reactor - export PASS_SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR=true -fi - -if [[ -n "$UNIX_SOCKETS" ]]; then - # Enable full on Unix socket mode for Synapse, Redis and Postgresql - export PASS_SYNAPSE_USE_UNIX_SOCKET=1 -fi - -if [[ -n "$SYNAPSE_TEST_LOG_LEVEL" ]]; then - # Set the log level to what is desired - export PASS_SYNAPSE_LOG_LEVEL="$SYNAPSE_TEST_LOG_LEVEL" - - # Allow logging sensitive things (currently SQL queries & parameters). - # (This won't have any effect if we're not logging at DEBUG level overall.) - # Since this is just a test suite, this is fine and won't reveal anyone's - # personal information - export PASS_SYNAPSE_LOG_SENSITIVE=1 -fi - -# Log a few more useful things for a developer attempting to debug something -# particularly tricky. -export PASS_SYNAPSE_LOG_TESTING=1 - -# Run the tests! -echo "Images built; running complement with ${extra_test_args[@]} $@ ${test_packages[@]}" -cd "$COMPLEMENT_DIR" - -go test -v -tags "synapse_blacklist" -count=1 "${extra_test_args[@]}" "$@" "${test_packages[@]}"