Wheel builder #516
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Workflow to build and test wheels. | |
| # To work on the wheel building infrastructure on a fork, comment out: | |
| # | |
| # if: github.repository == 'scipy/scipy' | |
| # | |
| # in the get_commit_message job include [wheel build] in your commit | |
| # message to trigger the build. All files related to wheel building are located | |
| # at tools/wheels/ | |
| name: Wheel builder | |
| on: | |
| schedule: | |
| # ┌───────────── minute (0 - 59) | |
| # │ ┌───────────── hour (0 - 23) | |
| # │ │ ┌───────────── day of the month (1 - 31) | |
| # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) | |
| # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) | |
| # │ │ │ │ │ | |
| - cron: "9 9 * * *" | |
| push: | |
| branches: | |
| - maintenance/** | |
| pull_request: | |
| branches: | |
| - main | |
| - maintenance/** | |
| workflow_dispatch: | |
| permissions: | |
| contents: read # to fetch code (actions/checkout) | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| get_commit_message: | |
| name: Get commit message | |
| runs-on: ubuntu-latest | |
| if: github.repository == 'scipy/scipy' | |
| outputs: | |
| message: ${{ steps.commit_message.outputs.message }} | |
| steps: | |
| - name: Checkout scipy | |
| uses: actions/[email protected] | |
| # Gets the correct commit message for pull request | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Get commit message | |
| id: commit_message | |
| run: | | |
| set -xe | |
| COMMIT_MSG=$(git log --no-merges -1) | |
| RUN="0" | |
| if [[ "$COMMIT_MSG" == *"[wheel build]"* ]]; then | |
| RUN="1" | |
| fi | |
| echo "message=$RUN" >> $GITHUB_OUTPUT | |
| echo github.ref ${{ github.ref }} | |
| build_wheels: | |
| name: Wheel, ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} | |
| ${{ matrix.buildplat[2] }} ${{ matrix.buildplat[3] }} | |
| ${{ matrix.buildplat[4] }} | |
| needs: get_commit_message | |
| if: >- | |
| contains(needs.get_commit_message.outputs.message, '1') || | |
| github.event_name == 'schedule' || | |
| github.event_name == 'workflow_dispatch' | |
| runs-on: ${{ matrix.buildplat[0] }} | |
| strategy: | |
| # Ensure that a wheel builder finishes even if another fails | |
| fail-fast: false | |
| matrix: | |
| # Github Actions doesn't support pairing matrix values together, let's improvise | |
| # https://github.com/github/feedback/discussions/7835#discussioncomment-1769026 | |
| buildplat: | |
| # should also be able to do multi-archs on a single entry, e.g. | |
| # [windows-2019, win*, "AMD64 x86"]. However, those two require a different compiler setup | |
| # so easier to separate out here. | |
| - [ubuntu-22.04, manylinux, x86_64, "", ""] | |
| - [ubuntu-22.04, musllinux, x86_64, "", ""] | |
| - [macos-12, macosx, x86_64, openblas, "10.13"] | |
| - [macos-13, macosx, x86_64, accelerate, "14.0"] | |
| - [macos-14, macosx, arm64, openblas, "12.3"] | |
| - [macos-14, macosx, arm64, accelerate, "14.0"] | |
| - [windows-2019, win, AMD64, "", ""] | |
| python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"], ["cp313", "3.13"]] | |
| # python[0] is used to specify the python versions made by cibuildwheel | |
| env: | |
| IS_32_BIT: ${{ matrix.buildplat[2] == 'x86' }} | |
| # upload to staging if it's a push to a maintenance branch and the last | |
| # commit message contains '[wheel build]' | |
| IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/heads/maintenance') && contains(needs.get_commit_message.outputs.message, '1') }} | |
| IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| steps: | |
| - name: Checkout scipy | |
| uses: actions/[email protected] | |
| with: | |
| submodules: true | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11 | |
| - name: win_amd64 - install rtools | |
| run: | | |
| # mingw-w64 | |
| choco install rtools -y --no-progress --force --version=4.0.0.20220206 | |
| echo "c:\rtools40\ucrt64\bin;" >> $env:GITHUB_PATH | |
| if: ${{ runner.os == 'Windows' && env.IS_32_BIT == 'false' }} | |
| - name: windows - set PKG_CONFIG_PATH | |
| if: ${{ runner.os == 'Windows' }} | |
| run: | | |
| $env:CIBW = "${{ github.workspace }}" | |
| # It seems somewhere in the env passing, `\` is not | |
| # passed through, so convert it to '/' | |
| $env:CIBW=$env:CIBW.replace("\","/") | |
| echo "CIBW_ENVIRONMENT=PKG_CONFIG_PATH=$env:CIBW" >> $env:GITHUB_ENV | |
| - name: Setup macOS | |
| if: startsWith( matrix.buildplat[0], 'macos-' ) | |
| run: | | |
| if [[ ${{ matrix.buildplat[3] }} == 'accelerate' ]]; then | |
| echo CIBW_CONFIG_SETTINGS=\"setup-args=-Dblas=accelerate\" >> "$GITHUB_ENV" | |
| # Always use preinstalled gfortran for Accelerate builds | |
| ln -s $(which gfortran-13) gfortran | |
| export PATH=$PWD:$PATH | |
| echo "PATH=$PATH" >> "$GITHUB_ENV" | |
| LIB_PATH=$(dirname $(gfortran --print-file-name libgfortran.dylib)) | |
| fi | |
| if [[ ${{ matrix.buildplat[4] }} == '10.13' ]]; then | |
| # 20240621 macos-12 images span Xcode 13.1-->14.2 | |
| XCODE_VER='13.4.1' | |
| else | |
| XCODE_VER='15.2' | |
| fi | |
| CIBW="sudo xcode-select -s /Applications/Xcode_${XCODE_VER}.app" | |
| echo "CIBW_BEFORE_ALL=$CIBW" >> $GITHUB_ENV | |
| # setting SDKROOT necessary when using the gfortran compiler | |
| # installed in cibw_before_build_macos.sh | |
| sudo xcode-select -s /Applications/Xcode_${XCODE_VER}.app | |
| CIBW="MACOSX_DEPLOYMENT_TARGET=${{ matrix.buildplat[4] }}\ | |
| SDKROOT=$(xcrun --sdk macosx --show-sdk-path)\ | |
| PKG_CONFIG_PATH=${{ github.workspace }}" | |
| echo "CIBW_ENVIRONMENT=$CIBW" >> "$GITHUB_ENV" | |
| echo "REPAIR_PATH=$LIB_PATH" >> "$GITHUB_ENV" | |
| PREFIX=DYLD_LIBRARY_PATH="\$(dirname \$(gfortran --print-file-name libgfortran.dylib))" | |
| # remove libgfortran from location used for linking (if any), to | |
| # check wheel has bundled things correctly and all tests pass without | |
| # needing installed gfortran | |
| POSTFIX=" sudo rm -rf /opt/gfortran-darwin-x86_64-native &&\ | |
| sudo rm -rf /usr/local/gfortran/lib" | |
| CIBW="$PREFIX delocate-listdeps -d {wheel} && echo "-----------" &&\ | |
| $PREFIX delocate-wheel -v $EXCLUDE --require-archs \ | |
| {delocate_archs} -w {dest_dir} {wheel} && echo "-----------" &&\ | |
| delocate-listdeps -d {dest_dir}/*.whl && echo "-----------" &&\ | |
| $POSTFIX" | |
| # Rename x86 Accelerate wheel to test on macOS 13 runner | |
| if [[ ${{ matrix.buildplat[0] }} == 'macos-13' && ${{ matrix.buildplat[4] }} == '14.0' ]]; then | |
| CIBW+=" && mv {dest_dir}/\$(basename {wheel}) \ | |
| {dest_dir}/\$(echo \$(basename {wheel}) | sed 's/14_0/13_0/')" | |
| fi | |
| # macos-arm64-openblas wheels that target macos-12 need a | |
| # MACOS_DEPLOYMENT_TARGET of 12.3 otherwise delocate complains. | |
| # Unclear of cause, possibly build tool related. | |
| # This results in wheels that have 12_3 in their name. Since Python | |
| # has no concept of minor OS versions in packaging land rename the | |
| # wheel back to 12. | |
| if [[ ${{ matrix.buildplat[0] }} == 'macos-14' && ${{ matrix.buildplat[4] }} == '12.3' ]]; then | |
| CIBW+=" && echo \$(ls {dest_dir}) && \ | |
| mv {dest_dir}/*.whl \$(find {dest_dir} -type f -name '*.whl' | sed 's/12_3/12_0/')" | |
| fi | |
| echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=$CIBW" >> "$GITHUB_ENV" | |
| - name: Inject environment variable for python dev version | |
| if: matrix.python[1] == '3.13' | |
| shell: bash | |
| run: | | |
| # For dev versions of python need to use wheels from scientific-python-nightly-wheels | |
| # When the python version is released please comment out this section, but do not remove | |
| # (there will soon be another dev version to target). | |
| DEPS0="pip install --pre -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy" | |
| DEPS1="pip install ninja meson-python pybind11 pythran cython" | |
| CIBW="$DEPS0;$DEPS1;bash {project}/tools/wheels/cibw_before_build_linux.sh {project}" | |
| echo "CIBW_BEFORE_BUILD_LINUX=$CIBW" >> "$GITHUB_ENV" | |
| CIBW="$DEPS0 && $DEPS1 && bash {project}/tools/wheels/cibw_before_build_win.sh {project}" | |
| echo "CIBW_BEFORE_BUILD_WINDOWS=$CIBW" >> "$GITHUB_ENV" | |
| CIBW="$DEPS0;$DEPS1;bash {project}/tools/wheels/cibw_before_build_macos.sh {project}" | |
| echo "CIBW_BEFORE_BUILD_MACOS=$CIBW" >> "$GITHUB_ENV" | |
| echo "CIBW_BEFORE_TEST=$DEPS0" >> "$GITHUB_ENV" | |
| CIBW="pip; args: --no-build-isolation" | |
| echo "CIBW_BUILD_FRONTEND=$CIBW" >> "$GITHUB_ENV" | |
| - name: Build wheels | |
| uses: pypa/[email protected] | |
| env: | |
| CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}* | |
| CIBW_ARCHS: ${{ matrix.buildplat[2] }} | |
| CIBW_PRERELEASE_PYTHONS: True | |
| - name: Rename macOS wheels | |
| if: startsWith( matrix.buildplat[0], 'macos-' ) | |
| run: | | |
| # macos-x86_64-accelerate wheels targeting macos-14 were renamed to 13 | |
| # so they could be tested. Shift wheel name back to targeting 14. | |
| if [[ ${{ matrix.buildplat[0] }} == 'macos-13' && ${{ matrix.buildplat[4] }} == '14.0' ]]; then | |
| mv ./wheelhouse/*.whl $(find ./wheelhouse -type f -name '*.whl' | sed 's/13_0/14_0/') | |
| fi | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| path: ./wheelhouse/*.whl | |
| name: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} | |
| ${{ matrix.buildplat[2] }} ${{ matrix.buildplat[3] }} | |
| ${{ matrix.buildplat[4] }} | |
| - uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| # for installation of anaconda-client, required for upload to | |
| # anaconda.org | |
| # default (and activated) environment name is test | |
| # Note that this step is *after* specific pythons have been used to | |
| # build and test the wheel | |
| auto-update-conda: true | |
| python-version: "3.10" | |
| miniconda-version: "latest" | |
| - name: Upload wheels | |
| if: success() | |
| shell: bash -el {0} | |
| # see https://github.com/marketplace/actions/setup-miniconda for why | |
| # `-el {0}` is required. | |
| env: | |
| SCIPY_STAGING_UPLOAD_TOKEN: ${{ secrets.SCIPY_STAGING_UPLOAD_TOKEN }} | |
| SCIPY_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.SCIPY_NIGHTLY_UPLOAD_TOKEN }} | |
| run: | | |
| conda install -y anaconda-client | |
| source tools/wheels/upload_wheels.sh | |
| set_upload_vars | |
| # For cron jobs (restricted to main branch) or "Run workflow" trigger | |
| # an upload to: | |
| # | |
| # https://anaconda.org/scientific-python-nightly-wheels/scipy | |
| # | |
| # Pushes to a maintenance branch that contain '[wheel build]' will | |
| # cause wheels to be built and uploaded to: | |
| # | |
| # https://anaconda.org/multibuild-wheels-staging/scipy | |
| # | |
| # The tokens were originally generated at anaconda.org | |
| upload_wheels |