Fix pyo3 as an optional dependency (#160)
#669
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - "**" | |
| pull_request: {} | |
| jobs: | |
| resolve: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| MSRV: ${{ steps.resolve-msrv.outputs.MSRV }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - name: resolve MSRV | |
| id: resolve-msrv | |
| run: echo MSRV=`python -c 'import tomllib; print(tomllib.load(open("Cargo.toml", "rb"))["workspace"]["package"]["rust-version"])'` >> $GITHUB_OUTPUT | |
| test-linux: | |
| needs: [resolve] | |
| name: test rust-${{ matrix.rust-version }} on linux | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| rust-version: [stable, nightly] | |
| include: | |
| - rust-version: ${{ needs.resolve.outputs.MSRV }} | |
| runs-on: ubuntu-latest | |
| env: | |
| RUNS_ON: ubuntu-latest | |
| RUST_VERSION: ${{ matrix.rust-version }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: ${{ matrix.rust-version }} | |
| - id: cache-rust | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| prefix-key: "v1-rust" | |
| - run: cargo install cargo-careful | |
| if: steps.cache-rust.outputs.cache-hit != 'true' | |
| - run: cargo test -F python | |
| - run: cargo careful t -F python | |
| if: matrix.rust-version == 'nightly' | |
| - uses: taiki-e/install-action@cargo-llvm-cov | |
| - run: cargo llvm-cov -F python --codecov --output-path codecov.json | |
| - uses: codecov/codecov-action@v4 | |
| with: | |
| env_vars: RUNS_ON,RUST_VERSION | |
| files: codecov.json | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| test-macos: | |
| name: test on ${{ matrix.runs-on }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runs-on: [macos-latest, macos-latest-xlarge] | |
| runs-on: ${{ matrix.runs-on }} | |
| env: | |
| RUNS_ON: ${{ matrix.runs-on }} | |
| RUST_VERSION: stable | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - uses: dtolnay/rust-toolchain@stable | |
| - id: cache-rust | |
| uses: Swatinem/rust-cache@v2 | |
| - run: cargo test -F python | |
| env: | |
| RUST_BACKTRACE: 1 | |
| - uses: taiki-e/install-action@cargo-llvm-cov | |
| - run: cargo llvm-cov -F python --codecov --output-path codecov.json | |
| - uses: codecov/codecov-action@v4 | |
| with: | |
| env_vars: RUNS_ON,RUST_VERSION | |
| files: codecov.json | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| test-python: | |
| name: test jiter-python | |
| runs-on: ubuntu-latest | |
| env: | |
| RUNS_ON: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - uses: dtolnay/rust-toolchain@stable | |
| - id: cache-rust | |
| uses: Swatinem/rust-cache@v2 | |
| - run: cargo install rustfilt coverage-prepare | |
| if: steps.cache-rust.outputs.cache-hit != 'true' | |
| - run: rustup component add llvm-tools-preview | |
| - run: make python-install | |
| - run: pip install -e crates/jiter-python | |
| env: | |
| RUSTFLAGS: "-C instrument-coverage" | |
| - run: pytest crates/jiter-python/tests | |
| env: | |
| RUST_BACKTRACE: 1 | |
| - run: python crates/jiter-python/bench.py --fast | |
| - run: coverage-prepare lcov $(python -c 'import jiter.jiter;print(jiter.jiter.__file__)') | |
| - uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| bench: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: moonrepo/setup-rust@v1 | |
| with: | |
| channel: stable | |
| cache-target: release | |
| bins: cargo-codspeed | |
| - run: python crates/jiter/benches/generate_big.py | |
| - run: cargo codspeed build -F python -p jiter | |
| - uses: CodSpeedHQ/action@v2 | |
| with: | |
| run: cargo codspeed run | |
| token: ${{ secrets.CODSPEED_TOKEN }} | |
| fuzz: | |
| name: fuzz on ${{ matrix.runs-on }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runs-on: [ubuntu-latest, macos-latest-xlarge] | |
| runs-on: ${{ matrix.runs-on }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: moonrepo/setup-rust@v1 | |
| with: | |
| channel: nightly | |
| cache-target: release | |
| bins: cargo-fuzz | |
| - run: | | |
| # cargo fuzz defaults to musl targets, which is seeming incomatible with sanitizers according to CI failures | |
| RUST_TARGET=$(rustc -Vv | grep host | cut -d ' ' -f 2) | |
| cargo fuzz run --target=$RUST_TARGET --fuzz-dir crates/fuzz compare_to_serde --release -- -max_total_time=300s | |
| fuzz-skip: | |
| name: fuzz skip | |
| # we only run this on ubuntu since architecture should make no difference | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: moonrepo/setup-rust@v1 | |
| with: | |
| channel: nightly | |
| cache-target: release | |
| bins: cargo-fuzz | |
| - run: | | |
| # cargo fuzz defaults to musl targets, which is seeming incomatible with sanitizers according to CI failures | |
| RUST_TARGET=$(rustc -Vv | grep host | cut -d ' ' -f 2) | |
| cargo fuzz run --target=$RUST_TARGET --fuzz-dir crates/fuzz compare_skip --release -- -max_total_time=300s | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - uses: moonrepo/setup-rust@v1 | |
| with: | |
| channel: stable | |
| components: rustfmt, clippy | |
| - uses: pre-commit/[email protected] | |
| with: | |
| extra_args: --all-files --verbose | |
| env: | |
| PRE_COMMIT_COLOR: always | |
| SKIP: test | |
| - run: cargo doc -F python | |
| build-sdist: | |
| name: build sdist | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - uses: PyO3/maturin-action@v1 | |
| with: | |
| # version pinned until https://github.com/PyO3/maturin/issues/2154 is fixed | |
| maturin-version: v1.6.0 | |
| command: sdist | |
| args: --out dist | |
| rust-toolchain: stable | |
| working-directory: crates/jiter-python | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: pypi_files_sdist | |
| path: crates/jiter-python/dist | |
| build: | |
| name: build on ${{ matrix.os }} (${{ matrix.target }} - ${{ matrix.interpreter || 'all' }}${{ matrix.os == 'linux' && format(' - {0}', matrix.manylinux == 'auto' && 'manylinux' || matrix.manylinux) || '' }}) | |
| # only run on push to main and on release | |
| if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [linux] # windows and macos builds added manually | |
| target: [x86_64, aarch64] | |
| manylinux: [auto] | |
| include: | |
| # manylinux for various platforms | |
| - os: linux | |
| manylinux: auto | |
| target: i686 | |
| - os: linux | |
| manylinux: auto | |
| target: aarch64 | |
| - os: linux | |
| manylinux: auto | |
| target: armv7 | |
| interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| - os: linux | |
| manylinux: auto | |
| target: ppc64le | |
| interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| - os: linux | |
| manylinux: auto | |
| target: s390x | |
| interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| # musllinux | |
| - os: linux | |
| manylinux: musllinux_1_1 | |
| target: x86_64 | |
| - os: linux | |
| manylinux: musllinux_1_1 | |
| target: aarch64 | |
| # macos; | |
| # all versions x86_64 | |
| # older pythons which can't be run on the arm hardware for PGO | |
| - os: macos | |
| target: x86_64 | |
| - os: macos | |
| target: aarch64 | |
| interpreter: 3.8 3.9 | |
| # windows; | |
| # aarch64 only 3.11 and up, also not PGO optimized | |
| - os: windows | |
| target: i686 | |
| python-architecture: x86 | |
| interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| # FIXME had link failures, needs investigation | |
| # - os: windows | |
| # target: aarch64 | |
| # interpreter: 3.11 3.12 | |
| exclude: | |
| # PGO optimized below | |
| - os: linux | |
| manylinux: auto | |
| target: x86_64 | |
| runs-on: ${{ (matrix.os == 'linux' && 'ubuntu') || matrix.os }}-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| architecture: ${{ matrix.python-architecture || 'x64' }} | |
| - name: build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.target }} | |
| manylinux: ${{ matrix.manylinux }} | |
| args: --release --out dist --interpreter ${{ matrix.interpreter || '3.8 3.9 3.10 3.11 3.12 3.13' }} | |
| rust-toolchain: stable | |
| working-directory: crates/jiter-python | |
| - run: ${{ (matrix.os == 'windows' && 'dir') || 'ls -lh' }} crates/jiter-python/dist/ | |
| - run: | | |
| pip install -U twine | |
| twine check --strict crates/jiter-python/dist/* | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: pypi_files_${{ matrix.os }}_${{ matrix.target }}_${{ matrix.interpreter || 'all' }}_${{ matrix.manylinux }} | |
| path: crates/jiter-python/dist | |
| build-pgo: | |
| name: build pgo-optimized on ${{ matrix.os }} / ${{ matrix.interpreter }} | |
| # only run on push to main and on release | |
| if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [linux, windows, macos] | |
| interpreter: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] | |
| include: | |
| # standard runners with override for macos arm | |
| - os: linux | |
| runs-on: ubuntu-latest | |
| - os: windows | |
| ls: dir | |
| runs-on: windows-latest | |
| - os: macos | |
| runs-on: macos-latest | |
| exclude: | |
| # macos arm only supported from 3.10 and up | |
| - os: macos | |
| interpreter: "3.8" | |
| - os: macos | |
| interpreter: "3.9" | |
| runs-on: ${{ matrix.runs-on }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.interpreter }} | |
| allow-prereleases: true | |
| - name: install rust stable | |
| id: rust-toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: llvm-tools | |
| - run: rustc --version --verbose | |
| - name: build initial wheel | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| manylinux: auto | |
| args: > | |
| --release | |
| --out pgo-wheel | |
| --interpreter ${{ matrix.interpreter }} | |
| rust-toolchain: stable | |
| working-directory: crates/jiter-python | |
| env: | |
| RUSTFLAGS: "-Cprofile-generate=${{ github.workspace }}/profdata" | |
| - name: detect rust host | |
| run: echo RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) >> "$GITHUB_ENV" | |
| shell: bash | |
| - name: generate pgo data | |
| run: | | |
| cd crates/jiter-python | |
| pip install -U pip | |
| pip install -r tests/requirements.txt | |
| pip install jiter --no-index --no-deps --find-links pgo-wheel --force-reinstall | |
| python bench.py jiter jiter-cache | |
| rustup run stable bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/${{ env.RUST_HOST }}/bin/llvm-profdata >> "$GITHUB_ENV"' | |
| - name: merge pgo data | |
| run: | | |
| cd crates/jiter-python | |
| ${{ env.LLVM_PROFDATA }} merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata | |
| - name: build pgo-optimized wheel | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| manylinux: auto | |
| args: > | |
| --release | |
| --out dist | |
| --interpreter ${{ matrix.interpreter }} | |
| rust-toolchain: stable | |
| working-directory: crates/jiter-python | |
| env: | |
| RUSTFLAGS: "-Cprofile-use=${{ github.workspace }}/merged.profdata" | |
| - run: ${{ matrix.ls || 'ls -lh' }} crates/jiter-python/dist/ | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: pypi_files_${{ matrix.os }}_${{ matrix.interpreter }} | |
| path: crates/jiter-python/dist | |
| inspect-pypi-assets: | |
| needs: [build, build-sdist, build-pgo] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: pypi_files_* | |
| merge-multiple: true | |
| path: dist | |
| - name: list dist files | |
| run: | | |
| ls -lh dist/ | |
| ls -l dist/ | |
| echo "`ls dist | wc -l` files" | |
| - name: extract and list sdist file | |
| run: | | |
| mkdir sdist-files | |
| tar -xvf dist/*.tar.gz -C sdist-files | |
| tree -a sdist-files | |
| - name: extract and list wheel file | |
| run: | | |
| ls dist/*cp310-manylinux*x86_64.whl | head -n 1 | |
| python -m zipfile --list `ls dist/*cp310-manylinux*x86_64.whl | head -n 1` | |
| test-builds-arch: | |
| name: test build on ${{ matrix.target }}-${{ matrix.distro }} | |
| needs: [build] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: [aarch64, armv7, s390x, ppc64le] | |
| distro: ["ubuntu22.04"] | |
| include: | |
| - target: aarch64 | |
| distro: alpine_latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: pypi_files_linux_* | |
| merge-multiple: true | |
| path: crates/jiter-python/dist | |
| - uses: uraimo/run-on-arch-action@v2 | |
| name: install & test | |
| with: | |
| arch: ${{ matrix.target }} | |
| distro: ${{ matrix.distro }} | |
| githubToken: ${{ github.token }} | |
| install: | | |
| set -x | |
| if command -v apt-get &> /dev/null; then | |
| echo "installing python & pip with apt-get..." | |
| apt-get update | |
| apt-get install -y --no-install-recommends python3 python3-pip python3-venv git | |
| else | |
| echo "installing python & pip with apk..." | |
| apk update | |
| apk add python3 py3-pip git | |
| fi | |
| run: | | |
| cd crates/jiter-python | |
| set -x | |
| # typing-extensions isn't automatically installed because of `--no-index --no-deps` | |
| python3 -m venv venv | |
| source venv/bin/activate | |
| python3 -m pip install -U pip -r tests/requirements.txt | |
| python3 -m pip install jiter --no-index --no-deps --find-links dist --force-reinstall | |
| python3 -c 'import jiter; print(jiter.__version__)' | |
| test-builds-os: | |
| name: test build on ${{ matrix.os }} | |
| needs: [build, build-pgo] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu, macos, windows] | |
| runs-on: ${{ matrix.os }}-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: pypi_files_* | |
| merge-multiple: true | |
| path: crates/jiter-python/dist | |
| - name: run tests | |
| run: | | |
| python3 -m pip install -U pip -r tests/requirements.txt | |
| python3 -m pip install jiter --no-index --no-deps --find-links dist --force-reinstall | |
| python3 -m pytest | |
| working-directory: crates/jiter-python | |
| # https://github.com/marketplace/actions/alls-green#why used for branch protection checks | |
| check: | |
| if: always() | |
| needs: [test-linux, test-macos, test-python, bench, fuzz, fuzz-skip, lint] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Decide whether the needed jobs succeeded or failed | |
| uses: re-actors/alls-green@release/v1 | |
| with: | |
| jobs: ${{ toJSON(needs) }} | |
| release: | |
| needs: [check] | |
| if: "success() && startsWith(github.ref, 'refs/tags/')" | |
| runs-on: ubuntu-latest | |
| environment: release | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - run: cargo publish -p jiter | |
| env: | |
| CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} | |
| release-python: | |
| needs: [test-builds-arch, test-builds-os, build-sdist, check] | |
| if: "success() && startsWith(github.ref, 'refs/tags/')" | |
| runs-on: ubuntu-latest | |
| environment: release-python | |
| permissions: | |
| id-token: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: pypi_files_* | |
| merge-multiple: true | |
| path: dist | |
| - run: pip install -U twine | |
| - run: ls -l dist/ | |
| - run: twine check --strict dist/* | |
| - name: upload to pypi | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| packages-dir: dist/ |