Fix tests workflow

Co-authored-by: Michael Telatynski <7t3chguy@gmail.com>

.codecov.yml

@@ -1,14 +0,0 @@
-comment: off
-
-coverage:
-  status:
-    project:
-      default:
-        target: 0  # Target % coverage, can be auto. Turned off for now
-        threshold: null
-        base: auto
-    patch:
-      default:
-        target: 0
-        threshold: null
-        base: auto

.coveragerc

@@ -6,3 +6,8 @@ data_file = $TOP/.coverage
 
 [report]
 precision = 2
+
+[paths]
+source =
+    synapse
+    /synapse/synapse

.github/workflows/sonarqube.yml

@@ -0,0 +1,32 @@
+name: SonarQube
+on:
+  workflow_run:
+    workflows: [ "Tests" ]
+    types:
+      - completed
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch }}
+  cancel-in-progress: true
+jobs:
+  sonarqube:
+    name: SonarQube
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: ${{ github.event.workflow_run.head_repository.full_name }}
+          ref: ${{ github.event.workflow_run.head_branch }} # checkout commit that triggered this workflow
+          fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
+      - uses: matrix-org/setup-python-poetry@v1
+      - name: SonarCloud Scan
+        uses: matrix-org/sonarcloud-workflow-action@v2.2
+        with:
+          is_pr: ${{ github.event.workflow_run.event == 'pull_request' }}
+          repository: ${{ github.event.workflow_run.head_repository.full_name }}
+          version_cmd: 'poetry version --short'
+          branch: ${{ github.event.workflow_run.head_branch }}
+          revision: ${{ github.event.workflow_run.head_sha }}
+          token: ${{ secrets.SONAR_TOKEN }}
+          coverage_run_id: ${{ github.event.workflow_run.id }}
+          coverage_workflow_name: tests.yml
+          skip_checkout: true

.github/workflows/tests.yml

@@ -10,65 +10,65 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  check-sampleconfig:
+#  check-sampleconfig:
-    runs-on: ubuntu-latest
+#    runs-on: ubuntu-latest
-    steps:
+#    steps:
-      - uses: actions/checkout@v2
+#      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+#      - uses: actions/setup-python@v2
-      - run: pip install .
+#      - run: pip install .
-      - run: scripts-dev/generate_sample_config.sh --check
+#      - run: scripts-dev/generate_sample_config.sh --check
-      - run: scripts-dev/config-lint.sh
+#      - run: scripts-dev/config-lint.sh
-
+#
-  check-schema-delta:
+#  check-schema-delta:
-    runs-on: ubuntu-latest
+#    runs-on: ubuntu-latest
-    steps:
+#    steps:
-      - uses: actions/checkout@v2
+#      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+#      - uses: actions/setup-python@v2
-      - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'"
+#      - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'"
-      - run: scripts-dev/check_schema_delta.py --force-colors
+#      - run: scripts-dev/check_schema_delta.py --force-colors
-
+#
-  lint:
+#  lint:
-    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
+#    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
-    with:
+#    with:
-      typechecking-extras: "all"
+#      typechecking-extras: "all"
-
+#
-  lint-crlf:
+#  lint-crlf:
-    runs-on: ubuntu-latest
+#    runs-on: ubuntu-latest
-    steps:
+#    steps:
-      - uses: actions/checkout@v2
+#      - uses: actions/checkout@v2
-      - name: Check line endings
+#      - name: Check line endings
-        run: scripts-dev/check_line_terminators.sh
+#        run: scripts-dev/check_line_terminators.sh
-
+#
-  lint-newsfile:
+#  lint-newsfile:
-    if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
+#    if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
-    runs-on: ubuntu-latest
+#    runs-on: ubuntu-latest
-    steps:
+#    steps:
-      - uses: actions/checkout@v2
+#      - uses: actions/checkout@v2
-        with:
+#        with:
-          ref: ${{ github.event.pull_request.head.sha }}
+#          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
+#          fetch-depth: 0
-      - uses: actions/setup-python@v2
+#      - uses: actions/setup-python@v2
-      - run: "pip install 'towncrier>=18.6.0rc1'"
+#      - run: "pip install 'towncrier>=18.6.0rc1'"
-      - run: scripts-dev/check-newsfragment.sh
+#      - run: scripts-dev/check-newsfragment.sh
-        env:
+#        env:
-          PULL_REQUEST_NUMBER: ${{ github.event.number }}
+#          PULL_REQUEST_NUMBER: ${{ github.event.number }}
-
+#
-  lint-pydantic:
+#  lint-pydantic:
-    runs-on: ubuntu-latest
+#    runs-on: ubuntu-latest
-    steps:
+#    steps:
-      - uses: actions/checkout@v2
+#      - uses: actions/checkout@v2
-        with:
+#        with:
-          ref: ${{ github.event.pull_request.head.sha }}
+#          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
+#          fetch-depth: 0
-      - uses: matrix-org/setup-python-poetry@v1
+#      - uses: matrix-org/setup-python-poetry@v1
-        with:
+#        with:
-          extras: "all"
+#          extras: "all"
-      - run: poetry run scripts-dev/check_pydantic_models.py
+#      - run: poetry run scripts-dev/check_pydantic_models.py
-
+#
   # Dummy step to gate other tests on without repeating the whole list
   linting-done:
     if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    needs: [lint, lint-crlf, lint-newsfile, lint-pydantic, check-sampleconfig, check-schema-delta]
+#    needs: [lint, lint-crlf, lint-newsfile, lint-pydantic, check-sampleconfig, check-schema-delta]
     runs-on: ubuntu-latest
     steps:
       - run: "true"
@@ -77,6 +77,8 @@ jobs:
     if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
     needs: linting-done
     runs-on: ubuntu-latest
+    env:
+      TOP: ${{ github.workspace }}
     strategy:
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10"]
@@ -127,7 +129,7 @@ jobs:
         if: ${{ matrix.postgres-version }}
         timeout-minutes: 2
         run: until pg_isready -h localhost; do sleep 1; done
-      - run: poetry run trial --jobs=2 tests
+      - run: poetry run coverage run --branch --parallel-mode -m twisted.trial --jobs=2 tests
         env:
           SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
           SYNAPSE_POSTGRES_HOST: localhost
@@ -145,33 +147,39 @@ jobs:
           -exec cat {} \;
           -exec echo "::endgroup::" \;
           || true
-
+      - name: Upload coverage
-  trial-olddeps:
+        uses: actions/upload-artifact@v3
-    # Note: sqlite only; no postgres
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Test with old deps
-        uses: docker://ubuntu:focal # For old python and sqlite
-        # Note: focal seems to be using 3.8, but the oldest is 3.7?
-        # See https://github.com/matrix-org/synapse/issues/12343
         with:
-          workdir: /github/workspace
+          name: coverage
-          entrypoint: .ci/scripts/test_old_deps.sh
+          path: .coverage.*
-      - name: Dump logs
+          retention-days: 1
-        # Logs are most useful when the command fails, always include them.
+
-        if: ${{ always() }}
+#  trial-olddeps:
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+#    # Note: sqlite only; no postgres
-        #       This keeps logs colocated with failing jobs
+#    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-        #       It also ignores find's exit code; this is a best effort affair
+#    needs: linting-done
-        run: >-
+#    runs-on: ubuntu-latest
-          find _trial_temp -name '*.log'
+#    steps:
-          -exec echo "::group::{}" \;
+#      - uses: actions/checkout@v2
-          -exec cat {} \;
+#      - name: Test with old deps
-          -exec echo "::endgroup::" \;
+#        uses: docker://ubuntu:focal # For old python and sqlite
-          || true
+#        # Note: focal seems to be using 3.8, but the oldest is 3.7?
+#        # See https://github.com/matrix-org/synapse/issues/12343
+#        with:
+#          workdir: /github/workspace
+#          entrypoint: .ci/scripts/test_old_deps.sh
+#      - name: Dump logs
+#        # Logs are most useful when the command fails, always include them.
+#        if: ${{ always() }}
+#        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+#        #       This keeps logs colocated with failing jobs
+#        #       It also ignores find's exit code; this is a best effort affair
+#        run: >-
+#          find _trial_temp -name '*.log'
+#          -exec echo "::group::{}" \;
+#          -exec cat {} \;
+#          -exec echo "::endgroup::" \;
+#          || true
 
   trial-pypy:
     # Very slow; only run if the branch name includes 'pypy'
@@ -221,7 +229,11 @@ jobs:
         WORKERS: ${{ matrix.workers && 1 }}
         REDIS: ${{ matrix.redis && 1 }}
         BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
-        TOP: ${{ github.workspace }}
+        # $TOP is usually defined as {{ github.workspace }}, however in this case we're
+        # operating in the context of the container, where the path defined by
+        # github.workspace doesn't exist, and the equivalent path is defined as /synapse
+        # in synapse_sytest.sh.
+        TOP: "/synapse"
 
     strategy:
       fail-fast: false
@@ -266,6 +278,13 @@ jobs:
           path: |
             /logs/results.tap
             /logs/**/*.log*
+      - name: Upload coverage
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage
+          path: /coverage/.coverage.*
+          retention-days: 1
+
 
   export-data:
     if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
@@ -367,20 +386,47 @@ jobs:
         shell: bash
         name: Run Complement Tests
 
+  # a job which combines the coverage reports from all trials runs, ready for upload to SonarQube.
+  coverage:
+    needs:
+      - trial
+      - sytest
+    runs-on: ubuntu-latest
+    env:
+      TOP: ${{ github.workspace }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v3
+        with:
+          name: coverage
+          path: .
+      - uses: matrix-org/setup-python-poetry@v1
+      - name: Combine coverage reports
+        run: poetry run coverage combine
+      - name: Write coverage to XML
+        run: poetry run coverage xml
+      - name: Upload combined coverage
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage
+          path: coverage.xml
+          retention-days: 1
+
   # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
   tests-done:
     if: ${{ always() }}
     needs:
-      - check-sampleconfig
+#      - check-sampleconfig
-      - lint
+#      - lint
-      - lint-crlf
+#      - lint-crlf
-      - lint-newsfile
+#      - lint-newsfile
       - trial
-      - trial-olddeps
+#      - trial-olddeps
       - sytest
       - export-data
       - portdb
       - complement
+      - coverage
     runs-on: ubuntu-latest
     steps:
       - uses: matrix-org/done-action@v2

changelog.d/12976.misc

@@ -0,0 +1 @@
+Wire up unit test coverage reporting to SonarCloud using Coverage.py.

poetry.lock

@@ -187,6 +187,17 @@ category = "main"
 optional = false
 python-versions = "*"
 
+[[package]]
+name = "coverage"
+version = "6.4.1"
+description = "Code coverage measurement for Python"
+category = "dev"
+optional = false
+python-versions = ">=3.7"
+
+[package.extras]
+toml = ["tomli"]
+
 [[package]]
 name = "cryptography"
 version = "36.0.1"
@@ -1739,6 +1750,49 @@ constantly = [
     {file = "constantly-15.1.0-py2.py3-none-any.whl", hash = "sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d"},
     {file = "constantly-15.1.0.tar.gz", hash = "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35"},
 ]
+coverage = [
+    {file = "coverage-6.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f1d5aa2703e1dab4ae6cf416eb0095304f49d004c39e9db1d86f57924f43006b"},
+    {file = "coverage-6.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4ce1b258493cbf8aec43e9b50d89982346b98e9ffdfaae8ae5793bc112fb0068"},
+    {file = "coverage-6.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83c4e737f60c6936460c5be330d296dd5b48b3963f48634c53b3f7deb0f34ec4"},
+    {file = "coverage-6.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84e65ef149028516c6d64461b95a8dbcfce95cfd5b9eb634320596173332ea84"},
+    {file = "coverage-6.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f69718750eaae75efe506406c490d6fc5a6161d047206cc63ce25527e8a3adad"},
+    {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e57816f8ffe46b1df8f12e1b348f06d164fd5219beba7d9433ba79608ef011cc"},
+    {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:01c5615d13f3dd3aa8543afc069e5319cfa0c7d712f6e04b920431e5c564a749"},
+    {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:75ab269400706fab15981fd4bd5080c56bd5cc07c3bccb86aab5e1d5a88dc8f4"},
+    {file = "coverage-6.4.1-cp310-cp310-win32.whl", hash = "sha256:a7f3049243783df2e6cc6deafc49ea123522b59f464831476d3d1448e30d72df"},
+    {file = "coverage-6.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:ee2ddcac99b2d2aec413e36d7a429ae9ebcadf912946b13ffa88e7d4c9b712d6"},
+    {file = "coverage-6.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fb73e0011b8793c053bfa85e53129ba5f0250fdc0392c1591fd35d915ec75c46"},
+    {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:106c16dfe494de3193ec55cac9640dd039b66e196e4641fa8ac396181578b982"},
+    {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87f4f3df85aa39da00fd3ec4b5abeb7407e82b68c7c5ad181308b0e2526da5d4"},
+    {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:961e2fb0680b4f5ad63234e0bf55dfb90d302740ae9c7ed0120677a94a1590cb"},
+    {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:cec3a0f75c8f1031825e19cd86ee787e87cf03e4fd2865c79c057092e69e3a3b"},
+    {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:129cd05ba6f0d08a766d942a9ed4b29283aff7b2cccf5b7ce279d50796860bb3"},
+    {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bf5601c33213d3cb19d17a796f8a14a9eaa5e87629a53979a5981e3e3ae166f6"},
+    {file = "coverage-6.4.1-cp37-cp37m-win32.whl", hash = "sha256:269eaa2c20a13a5bf17558d4dc91a8d078c4fa1872f25303dddcbba3a813085e"},
+    {file = "coverage-6.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f02cbbf8119db68455b9d763f2f8737bb7db7e43720afa07d8eb1604e5c5ae28"},
+    {file = "coverage-6.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ffa9297c3a453fba4717d06df579af42ab9a28022444cae7fa605af4df612d54"},
+    {file = "coverage-6.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:145f296d00441ca703a659e8f3eb48ae39fb083baba2d7ce4482fb2723e050d9"},
+    {file = "coverage-6.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d44996140af8b84284e5e7d398e589574b376fb4de8ccd28d82ad8e3bea13"},
+    {file = "coverage-6.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2bd9a6fc18aab8d2e18f89b7ff91c0f34ff4d5e0ba0b33e989b3cd4194c81fd9"},
+    {file = "coverage-6.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3384f2a3652cef289e38100f2d037956194a837221edd520a7ee5b42d00cc605"},
+    {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9b3e07152b4563722be523e8cd0b209e0d1a373022cfbde395ebb6575bf6790d"},
+    {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1480ff858b4113db2718848d7b2d1b75bc79895a9c22e76a221b9d8d62496428"},
+    {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:865d69ae811a392f4d06bde506d531f6a28a00af36f5c8649684a9e5e4a85c83"},
+    {file = "coverage-6.4.1-cp38-cp38-win32.whl", hash = "sha256:664a47ce62fe4bef9e2d2c430306e1428ecea207ffd68649e3b942fa8ea83b0b"},
+    {file = "coverage-6.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:26dff09fb0d82693ba9e6231248641d60ba606150d02ed45110f9ec26404ed1c"},
+    {file = "coverage-6.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9c80df769f5ec05ad21ea34be7458d1dc51ff1fb4b2219e77fe24edf462d6df"},
+    {file = "coverage-6.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:39ee53946bf009788108b4dd2894bf1349b4e0ca18c2016ffa7d26ce46b8f10d"},
+    {file = "coverage-6.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5b66caa62922531059bc5ac04f836860412f7f88d38a476eda0a6f11d4724f4"},
+    {file = "coverage-6.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd180ed867e289964404051a958f7cccabdeed423f91a899829264bb7974d3d3"},
+    {file = "coverage-6.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84631e81dd053e8a0d4967cedab6db94345f1c36107c71698f746cb2636c63e3"},
+    {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8c08da0bd238f2970230c2a0d28ff0e99961598cb2e810245d7fc5afcf1254e8"},
+    {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d42c549a8f41dc103a8004b9f0c433e2086add8a719da00e246e17cbe4056f72"},
+    {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:309ce4a522ed5fca432af4ebe0f32b21d6d7ccbb0f5fcc99290e71feba67c264"},
+    {file = "coverage-6.4.1-cp39-cp39-win32.whl", hash = "sha256:fdb6f7bd51c2d1714cea40718f6149ad9be6a2ee7d93b19e9f00934c0f2a74d9"},
+    {file = "coverage-6.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:342d4aefd1c3e7f620a13f4fe563154d808b69cccef415415aece4c786665397"},
+    {file = "coverage-6.4.1-pp36.pp37.pp38-none-any.whl", hash = "sha256:4803e7ccf93230accb928f3a68f00ffa80a88213af98ed338a57ad021ef06815"},
+    {file = "coverage-6.4.1.tar.gz", hash = "sha256:4321f075095a096e70aff1d002030ee612b65a205a0a0f5b815280d5dc58100c"},
+]
 cryptography = [
     {file = "cryptography-36.0.1-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:73bc2d3f2444bcfeac67dd130ff2ea598ea5f20b40e36d19821b4df8c9c5037b"},
     {file = "cryptography-36.0.1-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:2d87cdcb378d3cfed944dac30596da1968f88fb96d7fc34fdae30a99054b2e31"},

pyproject.toml

@@ -283,6 +283,7 @@ pygithub = "==1.55"
 twine = "*"
 # Towncrier min version comes from #3425. Rationale unclear.
 towncrier = ">=18.6.0rc1"
+coverage = "^6.4.1"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

sonar-project.properties

@@ -0,0 +1,15 @@
+sonar.projectKey=matrix-org_synapse
+sonar.organization=matrix-org
+
+# Encoding of the source code. Default is default system encoding
+#sonar.sourceEncoding=UTF-8
+
+sonar.sources=synapse
+sonar.tests=tests
+sonar.exclusions=changelog.d,debian,demo,docker,docs
+
+sonar.coverage.exclusions=tests/**/*
+
+sonar.python.coverage.reportPaths=coverage.xml
+sonar.python.version=3.7,3.8,3.9,3.10
+