From f8a7872ddbbb56763d3d59c5471da13f64801f5c Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 8 Jul 2025 13:30:36 +0200 Subject: [PATCH] Use native arm runners for building wheels (#18618) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This takes down the CI time to build wheels from 50 minutes to <10 minutes. **It also fixes macOS ARM builds, and includes more ARM builds in general** (we were ignoring pypy and musl before). This doesn't cost much for us to do this, reasons for not doing this is 1. space on PyPI and 2. keeping them 'officially' supported? This is the list of wheels this built (`+` are the ones added): ```diff matrix_synapse-1.133.0-cp39-abi3-macosx_10_9_x86_64.whl + matrix_synapse-1.133.0-cp39-abi3-macosx_11_0_arm64.whl matrix_synapse-1.133.0-cp39-abi3-manylinux_2_28_aarch64.whl matrix_synapse-1.133.0-cp39-abi3-manylinux_2_28_x86_64.whl + matrix_synapse-1.133.0-cp39-abi3-musllinux_1_2_aarch64.whl matrix_synapse-1.133.0-cp39-abi3-musllinux_1_2_x86_64.whl matrix_synapse-1.133.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl + matrix_synapse-1.133.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl + matrix_synapse-1.133.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl matrix_synapse-1.133.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl matrix_synapse-1.133.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl + matrix_synapse-1.133.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl + matrix_synapse-1.133.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl matrix_synapse-1.133.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl ``` And the numbers aaaaare 🥁 - [before](https://github.com/element-hq/synapse/actions/runs/16072488018): 54 minutes - [after](https://github.com/element-hq/synapse/actions/runs/16004034949?pr=18618): 10 minutes **Revert [e43b0f9](https://github.com/element-hq/synapse/pull/18618/commits/e43b0f9bd18f6d4ec9f9e9d992ecdcceaaf10982) before merging** --- .github/workflows/release-artifacts.yml | 33 +++++++++---------------- changelog.d/18618.misc | 1 + pyproject.toml | 7 ++---- 3 files changed, 14 insertions(+), 27 deletions(-) create mode 100644 changelog.d/18618.misc diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 89474dd4da..7bac012845 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -107,12 +107,15 @@ jobs: path: debs/* build-wheels: - name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }} + name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-24.04, macos-13] - arch: [x86_64, aarch64] + os: + - ubuntu-24.04 + - ubuntu-24.04-arm + - macos-13 # This uses x86-64 + - macos-14 # This uses arm64 # is_pr is a flag used to exclude certain jobs from the matrix on PRs. # It is not read by the rest of the workflow. is_pr: @@ -122,12 +125,11 @@ jobs: # Don't build macos wheels on PR CI. - is_pr: true os: "macos-13" - # Don't build aarch64 wheels on mac. - - os: "macos-13" - arch: aarch64 + - is_pr: true + os: "macos-14" # Don't build aarch64 wheels on PR CI. - is_pr: true - arch: aarch64 + os: "ubuntu-24.04-arm" steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -141,19 +143,9 @@ jobs: - name: Install cibuildwheel run: python -m pip install cibuildwheel==3.0.0 - - name: Set up QEMU to emulate aarch64 - if: matrix.arch == 'aarch64' - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 - with: - platforms: arm64 - - - name: Build aarch64 wheels - if: matrix.arch == 'aarch64' - run: echo 'CIBW_ARCHS_LINUX=aarch64' >> $GITHUB_ENV - - name: Only build a single wheel on PR if: startsWith(github.ref, 'refs/pull/') - run: echo "CIBW_BUILD="cp39-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV + run: echo "CIBW_BUILD="cp39-manylinux_*"" >> $GITHUB_ENV - name: Build wheels run: python -m cibuildwheel --output-dir wheelhouse @@ -161,13 +153,10 @@ jobs: # Skip testing for platforms which various libraries don't have wheels # for, and so need extra build deps. CIBW_TEST_SKIP: pp3*-* *i686* *musl* - # Fix Rust OOM errors on emulated aarch64: https://github.com/rust-lang/cargo/issues/10583 - CARGO_NET_GIT_FETCH_WITH_CLI: true - CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: Wheel-${{ matrix.os }}-${{ matrix.arch }} + name: Wheel-${{ matrix.os }} path: ./wheelhouse/*.whl build-sdist: diff --git a/changelog.d/18618.misc b/changelog.d/18618.misc new file mode 100644 index 0000000000..bd9aaf305d --- /dev/null +++ b/changelog.d/18618.misc @@ -0,0 +1 @@ +Speed up the building of arm-based wheels in CI. diff --git a/pyproject.toml b/pyproject.toml index 5b2a306f8d..a5977700ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -375,13 +375,10 @@ build-backend = "poetry.core.masonry.api" # Skip unsupported platforms (by us or by Rust). # See https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip for the list of build targets. # We skip: -# - CPython 3.6, 3.7 and 3.8: EOLed -# - PyPy 3.7 and 3.8: we only support Python 3.9+ +# - CPython and PyPy 3.8: EOLed # - musllinux i686: excluded to reduce number of wheels we build. # c.f. https://github.com/matrix-org/synapse/pull/12595#discussion_r963107677 -# - PyPy on Aarch64 and musllinux on aarch64: too slow to build. -# c.f. https://github.com/matrix-org/synapse/pull/14259 -skip = "cp36* cp37* cp38* pp37* pp38* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64" +skip = "cp38* pp38* *-musllinux_i686" # Enable non-default builds. # "pypy" used to be included by default up until cibuildwheel 3. enable = "pypy"