refactor: CI/CD #26
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - '**' | |
| pull_request: {} | |
| env: | |
| COLUMNS: 150 | |
| jobs: | |
| coverage: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: install rust nightly | |
| uses: dtolnay/rust-toolchain@nightly | |
| - id: cache-rust | |
| name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: coverage-v2 | |
| - name: Check | |
| run: cargo check --no-default-features | |
| - name: Build | |
| run: cargo build --verbose | |
| - name: Run tests | |
| run: cargo test --verbose | |
| - name: Check formatting | |
| run: cargo fmt -- --check | |
| - name: code-coverage | |
| env: | |
| TOKEN: ${{secrets.CODECOV_TOKEN}} | |
| run: | | |
| cargo install cargo-tarpaulin --force --git https://github.com/xd009642/tarpaulin --branch develop | |
| cargo tarpaulin --out xml | |
| bash <(curl -s https://codecov.io/bash) | |
| # See https://github.com/PyO3/pyo3/discussions/2781 | |
| # tests intermittently segfault with pypy and cpython 3.7 when using `coverage run ...`, hence separate job | |
| test-python: | |
| name: test ${{ matrix.python-version }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - '3.7' | |
| - '3.8' | |
| - '3.9' | |
| - '3.10' | |
| - '3.11' | |
| - '3.12' | |
| - 'pypy3.7' | |
| - 'pypy3.8' | |
| - 'pypy3.9' | |
| - 'pypy3.10' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: test-v3 | |
| - name: set up python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - run: pip install -r tests/requirements.txt | |
| - run: pip install -e . | |
| env: | |
| RUST_BACKTRACE: 1 | |
| - run: pip freeze | |
| - run: pytest | |
| test-os: | |
| name: test on ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu, macos, windows] | |
| runs-on: ${{ matrix.os }}-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: ${{ matrix.os }}-v1 | |
| - name: set up python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - run: pip install -r tests/requirements.txt | |
| - run: pip install -e . | |
| env: | |
| RUST_BACKTRACE: 1 | |
| - run: pip freeze | |
| - run: pytest | |
| - run: cargo test | |
| # https://github.com/marketplace/actions/alls-green#why used for branch protection checks | |
| check: | |
| if: always() | |
| needs: [coverage, test-python, test-os] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Decide whether the needed jobs succeeded or failed | |
| uses: re-actors/alls-green@release/v1 | |
| with: | |
| jobs: ${{ toJSON(needs) }} | |
| allowed-failures: coverage | |
| build-sdist: | |
| name: build sdist | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: PyO3/maturin-action@v1 | |
| with: | |
| command: sdist | |
| args: --out dist | |
| rust-toolchain: stable | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: pypi_files | |
| path: dist | |
| build: | |
| name: build on ${{ matrix.os }} (${{ matrix.target }} - ${{ matrix.interpreter || 'all' }}${{ matrix.os == 'linux' && format(' - {0}', matrix.manylinux == 'auto' && 'manylinux' || matrix.manylinux) || '' }}) | |
| # only run on push to main and on release | |
| if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [linux, macos, windows] | |
| target: [x86_64, aarch64] | |
| manylinux: [auto] | |
| include: | |
| # manylinux for various platforms, plus x86_64 pypy | |
| - os: linux | |
| manylinux: auto | |
| target: i686 | |
| - os: linux | |
| manylinux: auto | |
| target: aarch64 | |
| - os: linux | |
| manylinux: auto | |
| target: armv7 | |
| interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 | |
| - os: linux | |
| manylinux: auto | |
| target: ppc64le | |
| interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 | |
| - os: linux | |
| manylinux: auto | |
| target: s390x | |
| interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 | |
| - os: linux | |
| manylinux: auto | |
| target: x86_64 | |
| interpreter: pypy3.7 pypy3.8 pypy3.9 pypy3.10 | |
| # musllinux | |
| - os: linux | |
| manylinux: musllinux_1_1 | |
| target: x86_64 | |
| - os: linux | |
| manylinux: musllinux_1_1 | |
| target: aarch64 | |
| # macos; | |
| # all versions x86_64 | |
| # arm pypy and older pythons which can't be run on the arm hardware for PGO | |
| - os: macos | |
| target: x86_64 | |
| - os: macos | |
| target: aarch64 | |
| interpreter: 3.7 3.8 3.9 pypy3.8 pypy3.9 pypy3.10 | |
| # windows; | |
| # x86_64 pypy builds are not PGO optimized | |
| # i686 not supported by pypy | |
| # aarch64 only 3.11 and up, also not PGO optimized | |
| - os: windows | |
| target: x86_64 | |
| interpreter: pypy3.8 pypy3.9 pypy3.10 | |
| - os: windows | |
| target: i686 | |
| python-architecture: x86 | |
| interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 | |
| - os: windows | |
| target: aarch64 | |
| interpreter: 3.11 3.12 | |
| runs-on: ${{ (matrix.os == 'linux' && 'ubuntu') || matrix.os }}-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| architecture: ${{ matrix.python-architecture || 'x64' }} | |
| - run: pip install -U twine | |
| - name: build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.target }} | |
| manylinux: ${{ matrix.manylinux == 'manylinux' && 'auto' || matrix.manylinux }} | |
| args: --release --out dist --interpreter ${{ matrix.interpreter || '3.7 3.8 3.9 3.10 3.11 3.12 pypy3.7 pypy3.8 pypy3.9 pypy3.10' }} | |
| rust-toolchain: stable | |
| docker-options: -e CI | |
| - run: ${{ (matrix.os == 'windows' && 'dir') || 'ls -lh' }} dist/ | |
| - run: twine check --strict dist/* | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: pypi_files | |
| path: dist | |
| build-pgo: | |
| name: build pgo-optimized on ${{ matrix.os }} / ${{ matrix.interpreter }} | |
| # only run on push to main and on release | |
| if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [linux, windows, macos] | |
| interpreter: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] | |
| include: | |
| # standard runners with override for macos arm | |
| - os: linux | |
| runs-on: ubuntu-latest | |
| - os: windows | |
| ls: dir | |
| runs-on: windows-latest | |
| - os: macos | |
| runs-on: macos-latest-xlarge | |
| exclude: | |
| # macos arm only supported from 3.10 and up | |
| - os: macos | |
| interpreter: '3.7' | |
| - os: macos | |
| interpreter: '3.8' | |
| - os: macos | |
| interpreter: '3.9' | |
| runs-on: ${{ matrix.runs-on }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.interpreter }} | |
| - name: install rust stable | |
| id: rust-toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: llvm-tools | |
| - run: rustc --version --verbose | |
| - name: build initial wheel | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| manylinux: auto | |
| args: > | |
| --release | |
| --out pgo-wheel | |
| --interpreter ${{ matrix.interpreter }} | |
| rust-toolchain: stable | |
| docker-options: -e CI | |
| env: | |
| RUSTFLAGS: "-Cprofile-generate=${{ github.workspace }}/profdata" | |
| - name: detect rust host | |
| run: echo RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) >> "$GITHUB_ENV" | |
| shell: bash | |
| - name: generate pgo data | |
| run: | | |
| pip install -U pip | |
| pip install -r tests/requirements.txt | |
| pip install powerboxes --no-index --no-deps --find-links pgo-wheel --force-reinstall | |
| pytest tests/test_speed.py | |
| rustup run stable bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/${{ env.RUST_HOST }}/bin/llvm-profdata >> "$GITHUB_ENV"' | |
| - name: merge pgo data | |
| run: ${{ env.LLVM_PROFDATA }} merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata | |
| - name: build pgo-optimized wheel | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| manylinux: auto | |
| args: > | |
| --release | |
| --out dist | |
| --interpreter ${{ matrix.interpreter }} | |
| rust-toolchain: stable | |
| docker-options: -e CI | |
| env: | |
| RUSTFLAGS: "-Cprofile-use=${{ github.workspace }}/merged.profdata" | |
| - run: ${{ matrix.ls || 'ls -lh' }} dist/ | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: pypi_files_pgo | |
| path: dist | |
| inspect-pypi-assets: | |
| needs: [build, build-sdist, build-pgo] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi_files | |
| path: dist | |
| - name: list dist files before PGO builds | |
| run: | | |
| ls -lh dist/ | |
| ls -l dist/ | |
| echo "`ls dist | wc -l` files" | |
| - name: get PGO dist artifacts (comes after "get dist artifacts" to so these files override the non-PGO builds) | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi_files_pgo | |
| path: dist | |
| - name: list dist files with PGO builds | |
| run: | | |
| ls -lh dist/ | |
| ls -l dist/ | |
| echo "`ls dist | wc -l` files" | |
| - name: extract and list sdist file | |
| run: | | |
| mkdir sdist-files | |
| tar -xvf dist/*.tar.gz -C sdist-files | |
| tree -a sdist-files | |
| - name: extract and list wheel file | |
| run: | | |
| ls dist/*cp310-manylinux*x86_64.whl | head -n 1 | |
| python -m zipfile --list `ls dist/*cp310-manylinux*x86_64.whl | head -n 1` | |
| test-builds-arch: | |
| name: test build on ${{ matrix.target }}-${{ matrix.distro }} | |
| needs: [build] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: [aarch64, armv7, s390x, ppc64le] | |
| distro: ['ubuntu22.04'] | |
| include: | |
| - target: aarch64 | |
| distro: alpine_latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi_files | |
| path: dist | |
| - name: get PGO dist artifacts (comes after "get dist artifacts" to so these files override the non-PGO builds) | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi_files_pgo | |
| path: dist | |
| - uses: uraimo/run-on-arch-action@v2.5.1 | |
| name: install & test | |
| with: | |
| arch: ${{ matrix.target }} | |
| distro: ${{ matrix.distro }} | |
| githubToken: ${{ github.token }} | |
| install: | | |
| set -x | |
| if command -v apt-get &> /dev/null; then | |
| echo "installing python & pip with apt-get..." | |
| apt-get update | |
| apt-get install -y --no-install-recommends python3 python3-pip python3-venv | |
| else | |
| echo "installing python & pip with apk..." | |
| apk update | |
| apk add python3 py3-pip | |
| fi | |
| run: | | |
| set -x | |
| python3 -m venv venv | |
| source venv/bin/activate | |
| python3 -m pip install -r tests/requirements.txt | |
| python3 -m pip install powermetrics --no-index --no-deps --find-links dist --force-reinstall | |
| python3 -m pytest --ignore=tests/test_docstrings.py | |
| python3 -c 'import pydantic_core._pydantic_core; print(pydantic_core._pydantic_core.__version__)' | |
| test-builds-os: | |
| name: test build on ${{ matrix.os }} | |
| needs: [build, build-pgo] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu, macos, windows] | |
| runs-on: ${{ matrix.os }}-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi_files | |
| path: dist | |
| - name: get PGO dist artifacts (comes after "get dist artifacts" to so these files override the non-PGO builds) | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi_files_pgo | |
| path: dist | |
| - run: pip install typing-extensions | |
| - run: pip install -r tests/requirements.txt | |
| - run: pip install powerboxes --no-index --no-deps --find-links dist --force-reinstall | |
| - run: pytest | |
| release: | |
| needs: [test-builds-arch, test-builds-os, build-sdist, check] | |
| if: success() && startsWith(github.ref, 'refs/tags/') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - run: pip install -U twine | |
| - name: check package version | |
| run: python .github/check_version.py | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi_files | |
| path: dist | |
| - name: get PGO dist artifacts (comes after "get dist artifacts" to so these files override the non-PGO builds) | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi_files_pgo | |
| path: dist | |
| - run: twine check --strict dist/* | |
| - name: upload to pypi | |
| run: twine upload dist/* | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} | |
| - name: get wasm dist artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wasm_wheels | |
| path: wasm | |
| - name: upload to github release | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| files: | | |
| wasm/*.whl | |
| prerelease: ${{ contains(github.ref, 'alpha') || contains(github.ref, 'beta') }} |