Skip to content

use sdist for wheel builds #159

use sdist for wheel builds

use sdist for wheel builds #159

Workflow file for this run

---
name: PyYAML CI
on:
push:
# pull_request:
# types: [opened, synchronize, reopened]
workflow_dispatch:
inputs:
libyaml_repo:
type: string
default: https://github.com/yaml/libyaml
libyaml_ref:
type: string
default: 0.2.5
skip_artifact_upload:
type: boolean
default: true
skip_ci_redundant_jobs:
type: boolean
default: true
skip_slow_jobs:
type: boolean
default: true
env:
LIBYAML_REPO: ${{ inputs.libyaml_repo || 'https://github.com/yaml/libyaml' }} # FIXME: can we ref the input.default value?
LIBYAML_REF: ${{ inputs.libyaml_ref || '0.2.5' }} # FIXME: can we ref the input.default value?
skip_ci_redundant_jobs: ${{ inputs.skip_ci_redundant_jobs || github.event_name == 'pull_request' || github.event_name == 'push' }}
skip_slow_jobs: ${{ inputs.skip_slow_jobs || github.event_name == 'pull_request' || github.event_name == 'push' }}
skip_artifact_upload: ${{ inputs.skip_artifact_upload || true }} # ${{ github.event_name == 'pull_request' }}
jobs:
python_sdist:
name: pyyaml sdist
runs-on: ubuntu-22.04
outputs:
artifact_name: ${{ steps.build_sdist.outputs.artifact_name }}
steps:
- name: Checkout PyYAML
uses: actions/checkout@v4
- name: Install a python
uses: actions/setup-python@v5
with:
python-version: 3.x
- name: Build sdist
id: build_sdist
env:
PYYAML_FORCE_CYTHON: 1
PYYAML_FORCE_LIBYAML: 0
run: |
python -V
python -m pip install build
python -m build -s .
echo "artifact_name=$(ls ./dist)" >> "$GITHUB_OUTPUT"
#
#
# - name: Test sdist
# run: |
# # Install some libyaml headers.
# # TODO Should we smoke test the sdist against the libyaml we built?
# sudo apt update
# sudo apt install libyaml-dev -y
#
# mkdir distout && cd distout
# tar zxvf $(ls ../dist/*.tar.gz)
# python -m build -w ./pyyaml*
#
# # FIXME: smoketest the built wheel
#
# cd ..
#
# # Pass no extra args.
# # We should auto-install with libyaml since it's present.
# python -m pip install dist/*.tar.gz -v
#
# python packaging/build/smoketest.py
- name: Upload sdist artifact
uses: actions/upload-artifact@v4
with:
name: ${{ steps.build_sdist.outputs.artifact_name }}
path: dist/${{ steps.build_sdist.outputs.artifact_name }}
if-no-files-found: error
# always upload the sdist artifact- all the wheel build jobs require it
make_linux_libyaml_matrix:
runs-on: ubuntu-22.04
outputs:
matrix_json: ${{ steps.make_matrix.outputs.matrix_json }}
steps:
- uses: actions/checkout@v4
- name: make a matrix
id: make_matrix
uses: ./.github/actions/dynamatrix
with:
matrix_yaml: |
include:
- { platform: manylinux1, arch: x86_64 }
- { platform: manylinux2014, arch: x86_64 }
- { platform: manylinux2014, arch: aarch64, omit: ${{ env.skip_slow_jobs }} }
- { platform: manylinux2014, arch: s390x, omit: ${{ env.skip_slow_jobs }} }
- { platform: musllinux_1_1, arch: x86_64 }
- { platform: musllinux_1_1, arch: aarch64, omit: ${{ env.skip_slow_jobs }} }
linux_libyaml:
needs: [make_linux_libyaml_matrix]
name: libyaml ${{ matrix.platform }} ${{ matrix.arch }}
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.make_linux_libyaml_matrix.outputs.matrix_json) }}
env:
DOCKER_IMAGE: quay.io/pypa/${{ matrix.platform }}_${{ matrix.arch }}
steps:
- name: Check cached libyaml state
id: cached_libyaml
uses: actions/cache@v4
with:
path: libyaml
key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }}
- name: configure docker foreign arch support
uses: docker/setup-qemu-action@v3
if: matrix.arch != 'x86_64' && steps.cached_libyaml.outputs.cache-hit != 'true'
- name: Checkout pyyaml
uses: actions/checkout@v4
if: steps.cached_libyaml.outputs.cache-hit != 'true'
- name: Build libyaml
run: >
docker run --rm
--volume "$(pwd):/io"
--env LIBYAML_REF
--env LIBYAML_REPO
--workdir /io
"$DOCKER_IMAGE"
/io/packaging/build/libyaml.sh
if: steps.cached_libyaml.outputs.cache-hit != 'true'
- name: ensure output is world readable (or cache fill fails with Permission Denied)
run: >
sudo chmod -R a+r ./libyaml/
if: steps.cached_libyaml.outputs.cache-hit != 'true'
make_linux_pyyaml_matrix:
runs-on: ubuntu-22.04
outputs:
matrix_json: ${{ steps.make_matrix.outputs.matrix_json }}
steps:
- uses: actions/checkout@v4
- name: make a matrix
id: make_matrix
uses: ./.github/actions/dynamatrix
with:
matrix_yaml: |
include:
- { platform: manylinux1, arch: x86_64, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs }} }
- { platform: manylinux1, arch: x86_64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs }} }
- { platform: manylinux2014, arch: x86_64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs }} }
- { platform: manylinux2014, arch: x86_64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs }} }
- { platform: manylinux2014, arch: x86_64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs }} }
- { platform: manylinux2014, arch: x86_64, spec: cp313, cibw_version: "https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip" }
- { platform: manylinux2014, arch: aarch64, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
- { platform: manylinux2014, arch: aarch64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
- { platform: manylinux2014, arch: aarch64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
- { platform: manylinux2014, arch: aarch64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
- { platform: manylinux2014, arch: aarch64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
- { platform: manylinux2014, arch: aarch64, spec: cp313, cibw_version: "https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip", omit: ${{ env.skip_slow_jobs }} }
- { platform: manylinux2014, arch: s390x, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
- { platform: manylinux2014, arch: s390x, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
- { platform: manylinux2014, arch: s390x, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
- { platform: manylinux2014, arch: s390x, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
- { platform: manylinux2014, arch: s390x, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
- { platform: manylinux2014, arch: s390x, spec: cp313, cibw_version: "https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip", omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
- { platform: musllinux_1_1, arch: x86_64, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs }} }
- { platform: musllinux_1_1, arch: x86_64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs }} }
- { platform: musllinux_1_1, arch: x86_64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs }} }
- { platform: musllinux_1_1, arch: x86_64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs }} }
- { platform: musllinux_1_1, arch: x86_64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs }} }
- { platform: musllinux_1_1, arch: x86_64, spec: cp313, cibw_version: "https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip", omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
- { platform: musllinux_1_1, arch: aarch64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
- { platform: musllinux_1_1, arch: aarch64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
- { platform: musllinux_1_1, arch: aarch64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
- { platform: musllinux_1_1, arch: aarch64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
- { platform: musllinux_1_1, arch: aarch64, spec: cp313, cibw_version: "https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip", omit: ${{ env.skip_slow_jobs }} }
linux_pyyaml:
needs: [python_sdist, linux_libyaml, make_linux_pyyaml_matrix]
name: pyyaml ${{matrix.spec}}
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.make_linux_pyyaml_matrix.outputs.matrix_json) }}
steps:
# - name: Checkout PyYAML
# uses: actions/checkout@v4
- name: fetch sdist artifact
id: fetch_sdist
uses: actions/download-artifact@v4
with:
name: ${{ needs.build_sdist.outputs.artifact_name }}
- name: Fetch cached libyaml
id: cached_libyaml
uses: actions/cache/restore@v4
with:
path: libyaml
key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}}
fail-on-cache-miss: true
- name: configure docker foreign arch support
uses: docker/setup-qemu-action@v3
if: matrix.arch != 'x86_64'
- name: Build/Test/Package
env:
CIBW_ARCHS: all
# HACK: ick, maybe deconstruct the matrix a bit or query cibuildwheel for its default target *linux spec first?
CIBW_BUILD: ${{matrix.spec}}-${{ contains(matrix.platform, 'musllinux') && 'musllinux' || 'manylinux' }}_${{matrix.arch}}
CIBW_BUILD_VERBOSITY: 1
# containerized Linux builds require explicit CIBW_ENVIRONMENT
CIBW_ENVIRONMENT: >
C_INCLUDE_PATH=libyaml/include
LIBRARY_PATH=libyaml/src/.libs
LD_LIBRARY_PATH=libyaml/src/.libs
PYYAML_FORCE_CYTHON=1
PYYAML_FORCE_LIBYAML=1
CIBW_TEST_COMMAND: cd {project}; pytest
CIBW_TEST_REQUIRES: pytest
run: |
set -eux
python3 -V
python3 -m pip install -U --user ${{ matrix.cibw_version || 'cibuildwheel' }}
CIBW_BEFORE_BUILD="cp /host$(pwd)/libyaml {project}" python3 -m cibuildwheel --platform auto --output-dir ./dist ${{ steps.fetch_sdist.outputs.download-path }}/pyyaml*.tar.gz/pyyaml*.tar.gz
echo "artifact_name=$(ls ./dist/)" >> "$GITHUB_OUTPUT"
# FIXME: ensure exactly one artifact
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: ${{ steps.build.outputs.artifact_name }}
path: dist/*.whl
if-no-files-found: error
if: ${{ ! env.skip_artifact_upload }}
#
# macos_libyaml:
# name: libyaml macos ${{matrix.arch}}
# strategy:
# matrix:
# include:
# - arch: x86_64
# runs-on: macos-13
# - arch: arm64
# deployment_target: '11.0'
# run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
# defaults:
# run:
# shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }}
# runs-on: ${{ matrix.runs_on || 'macos-14' }}
# steps:
# - name: Check cached libyaml state
# id: cached_libyaml
# uses: actions/cache@v4
# with:
# path: libyaml
# key: libyaml_macos_${{matrix.arch}}_${{env.LIBYAML_REF}}
#
# - name: Checkout PyYAML
# uses: actions/checkout@v4
# if: steps.cached_libyaml.outputs.cache-hit != 'true'
#
# - name: Build libyaml
# env:
# MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }}
# SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
# run: |
# set -eux
# brew install automake coreutils m4
# bash ./packaging/build/libyaml.sh
# echo "finished artifact arch is $(lipo -archs libyaml/src/.libs/libyaml.a)"
# if: steps.cached_libyaml.outputs.cache-hit != 'true'
#
#
# macos_pyyaml:
# needs: macos_libyaml
# name: pyyaml ${{ matrix.spec }}
# runs-on: ${{ matrix.runs_on || 'macos-14' }}
# defaults:
# run:
# shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }}
# strategy:
# matrix:
# include:
# - spec: cp38-macosx_x86_64
# cibw_version: cibuildwheel==2.11.1
# runs_on: [macos-13]
# - spec: cp39-macosx_x86_64
# runs_on: [macos-13]
## - spec: cp310-macosx_x86_64
## runs_on: [macos-13]
## - spec: cp311-macosx_x86_64
## runs_on: [macos-13]
## - spec: cp312-macosx_x86_64
## runs_on: [macos-13]
# - spec: cp313-macosx_x86_64
# cibw_version: https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip
# runs_on: [macos-13]
#
# # build for arm64 under a hacked macOS 12 self-hosted x86_64-on-arm64 runner until arm64 is fully supported
# - spec: cp39-macosx_arm64
# deployment_target: '11.0'
# arch: arm64
# run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
#
# - spec: cp310-macosx_arm64
# deployment_target: '11.0'
# arch: arm64
# run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
#
# - spec: cp311-macosx_arm64
# deployment_target: '11.0'
# arch: arm64
# run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
#
# - spec: cp312-macosx_arm64
# deployment_target: '11.0'
# arch: arm64
# run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
#
# - spec: cp313-macosx_arm64
# deployment_target: '11.0'
# arch: arm64
# run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
# cibw_version: https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip
#
# steps:
# - name: Checkout PyYAML
# uses: actions/checkout@v4
#
# - name: Get cached libyaml state
# id: cached_libyaml
# uses: actions/cache/restore@v4
# with:
# path: libyaml
# key: libyaml_macos_${{ matrix.arch || 'x86_64' }}_${{env.LIBYAML_REF}}
# fail-on-cache-miss: true
#
# - name: Build/Test/Package
# env:
# C_INCLUDE_PATH: libyaml/include
# CIBW_BUILD: ${{matrix.spec}}
# CIBW_BUILD_VERBOSITY: 1
# CIBW_TEST_COMMAND: cd {project}; pytest
# CIBW_TEST_REQUIRES: pytest
# LIBRARY_PATH: libyaml/src/.libs
# MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }}
# PYYAML_FORCE_CYTHON: 1
# PYYAML_FORCE_LIBYAML: 1
# SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
# run: |
# python3 -V
# python3 -m pip install -U --user ${{ matrix.cibw_version || 'cibuildwheel' }}
# python3 -m cibuildwheel --platform auto --output-dir dist .
#
# - name: Upload artifacts
# uses: actions/upload-artifact@v3
# with:
# name: dist
# path: dist/*.whl
# if-no-files-found: error
# if: ${{ ! env.skip_artifact_upload }}
#
# windows_libyaml:
# name: libyaml windows ${{ matrix.arch }}
# runs-on: ${{ matrix.platform || 'windows-2022' }}
# strategy:
# matrix:
# include:
# - arch: x64
# - arch: win32
# steps:
# - name: Get cached libyaml state
# id: cached_libyaml
# uses: actions/cache@v4
# with:
# path: libyaml
# key: libyaml_${{ 'windows' }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }}
#
# - name: Build libyaml
# shell: bash
# if: steps.cached_libyaml.outputs.cache-hit != 'true'
# run: |
# # git spews all over stderr unless we tell it not to
# export GIT_REDIRECT_STDERR="2>&1"
#
# if [[ ! -d ./libyaml ]]; then
# git clone -b ${{ env.LIBYAML_REF }} ${{ env.LIBYAML_REPO }} 2>&1
# fi
#
# pushd libyaml
# git clean -fdx
# popd
#
# mkdir libyaml/build
#
# pushd libyaml/build
# cmake.exe -G "Visual Studio 17 2022" -A ${{ matrix.arch }} -DYAML_STATIC_LIB_NAME=yaml ..
# cmake.exe --build . --config Release
# popd
#
# make_windows_pyyaml_matrix:
# runs-on: ubuntu-22.04
# outputs:
# matrix_json: ${{ steps.make_matrix.outputs.matrix_json }}
# steps:
# - uses: actions/checkout@v4
# - name: make a matrix
# id: make_matrix
# uses: ./.github/actions/dynamatrix
# with:
# matrix_yaml: |
# include:
# - spec: cp38-win_amd64
# # omit: ${{ env.skip_ci_redundant_jobs }}
#
# - spec: cp39-win_amd64
# omit: ${{ env.skip_ci_redundant_jobs }}
#
# - spec: cp310-win_amd64
# omit: ${{ env.skip_ci_redundant_jobs }}
#
# - spec: cp311-win_amd64
# omit: ${{ env.skip_ci_redundant_jobs }}
#
# - spec: cp312-win_amd64
# omit: ${{ env.skip_ci_redundant_jobs }}
#
# - spec: cp313-win_amd64
# cibw_version: https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip
#
# - spec: cp38-win32
# omit: ${{ env.skip_ci_redundant_jobs }}
#
# - spec: cp39-win32
# omit: ${{ env.skip_ci_redundant_jobs }}
#
# - spec: cp310-win32
# omit: ${{ env.skip_ci_redundant_jobs }}
#
# - spec: cp311-win32
# omit: ${{ env.skip_ci_redundant_jobs }}
#
# - spec: cp312-win32
# omit: ${{ env.skip_ci_redundant_jobs }}
#
# - spec: cp313-win32
# cibw_version: https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip
# # omit: ${{ env.skip_ci_redundant_jobs }}
#
# windows_pyyaml:
# needs: [python_sdist, windows_libyaml, make_windows_pyyaml_matrix]
# name: pyyaml ${{matrix.spec}}
# runs-on: ${{ matrix.runs-on || 'windows-2022' }}
# strategy:
# fail-fast: false
# matrix: ${{ fromJSON(needs.make_windows_pyyaml_matrix.outputs.matrix_json) }}
# steps:
# # autocrlf screws up tests under Windows
# - name: Set git to use LF
# run: |
# git config --global core.autocrlf false
# git config --global core.eol lf
#
# - name: Checkout pyyaml
# uses: actions/checkout@v4
#
# - name: Get cached libyaml state
# id: cached_libyaml
# uses: actions/cache/restore@v4
# with:
# path: libyaml
# key: libyaml_${{'windows'}}_${{ contains(matrix.spec, 'win_amd64') && 'x64' || 'win32' }}_${{env.LIBYAML_REF}}
# fail-on-cache-miss: true
#
# - name: Install python
# uses: actions/setup-python@v5
# with:
# python-version: 3.x
#
# - name: Build/Test/Package
# id: build
# shell: bash
# env:
# CIBW_BUILD: ${{matrix.spec}}
# CIBW_BUILD_VERBOSITY: 1
# CIBW_TEST_COMMAND: pytest {project}/tests
# CIBW_TEST_REQUIRES: pytest
# CIBW_CONFIG_SETTINGS: |
# pyyaml_build_config='{"include_dirs": ["libyaml/include"], "library_dirs": ["libyaml/build/Release"], "define": [["YAML_DECLARE_STATIC", 1]], "force": 1}'
# run: |
# set -eux
# python -V
# python -m pip install -U --user ${{ matrix.cibw_version || 'cibuildwheel' }}
# python -m cibuildwheel --output-dir dist .
# echo "ARTIFACT_NAME=$(ls ./dist/)" >> "$GITHUB_OUTPUT"
# # FIXME: ensure exactly one artifact
#
# - name: Upload artifacts
# uses: actions/upload-artifact@v4
# with:
# name: ${{ steps.build.outputs.ARTIFACT_NAME }}
# path: dist/*.whl
# if-no-files-found: error
# if: ${{ ! env.skip_artifact_upload }}
# FIXME: artifact combine job?
#
# check:
# if: always()
# needs:
# - sdist
# - linux
# - macos
# - windows
# runs-on: ubuntu-latest
# steps:
# - name: Verify all previous jobs succeeded (provides a single check to sample for gating purposes)
# uses: re-actors/alls-green@release/v1
# with:
# jobs: ${{ toJSON(needs) }}
...