From 6bb7b6e14b0c91aaa8fd6d8f574f1e3fe126e974 Mon Sep 17 00:00:00 2001 From: geisserml Date: Mon, 30 Oct 2023 23:40:55 +0100 Subject: [PATCH] Conda CI --- .github/workflows/conda.yaml | 149 ++++++++++++++++++ .github/workflows/gh_pages.yaml | 4 - .../{build_packages.yaml => main.yaml} | 26 ++- .github/workflows/test_release.yaml | 2 +- .github/workflows/trigger_conda_raw.yaml | 27 ++++ ...trigger_release.yaml => trigger_main.yaml} | 11 +- README.md | 37 ++++- conda/bundle/recipe/meta.yaml | 2 - conda/helpers/recipe/meta.yaml | 4 +- conda/raw/recipe/meta.yaml | 2 - setupsrc/pypdfium2_setup/craft_packages.py | 8 +- 11 files changed, 242 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/conda.yaml rename .github/workflows/{build_packages.yaml => main.yaml} (91%) create mode 100644 .github/workflows/trigger_conda_raw.yaml rename .github/workflows/{trigger_release.yaml => trigger_main.yaml} (91%) diff --git a/.github/workflows/conda.yaml b/.github/workflows/conda.yaml new file mode 100644 index 000000000..49d223fb4 --- /dev/null +++ b/.github/workflows/conda.yaml @@ -0,0 +1,149 @@ +# SPDX-FileCopyrightText: 2023 geisserml +# SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause + +name: Conda packaging +on: + workflow_dispatch: + inputs: + package: + type: choice + default: raw + options: + - raw + - helpers + test: + default: true + type: boolean + publish: + default: false + type: boolean + py_version: + default: '3.11' + type: string + +# This is required for setup-miniconda / conda init +# see https://github.com/conda-incubator/setup-miniconda#important +defaults: + run: + shell: bash -el {0} + +jobs: + + build: + + runs-on: ubuntu-latest + steps: + + - name: Check out repository + uses: actions/checkout@v4 + with: + repository: ${{ github.repository }} + fetch-depth: 0 + + - name: Miniconda setup + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ inputs.py_version }} + channels: bblanchon,pypdfium2-team + channel-priority: strict + + - name: Prepare + run: | + python -VV + conda install -y conda-build conda-verify + git config --global user.email "geisserml@gmail.com" + git config --global user.name "geisserml" + python -m pip install -U -r req/setup.txt + + - name: Build package + run: ./run craft conda_${{ inputs.package }} + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: conda_package + path: conda/${{ inputs.package }}/out/ + + test: + + if: ${{ inputs.test }} + needs: build + + strategy: + fail-fast: false + matrix: + os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] + py: ['3.8', '3.9', '3.10', '3.11'] + + runs-on: ${{ matrix.os }} + + steps: + + - name: Check out repository + uses: actions/checkout@v4 + with: + repository: ${{ github.repository }} + + - name: Miniconda setup + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.py }} + channels: bblanchon,pypdfium2-team + channel-priority: strict + + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: conda_package + path: conda_dist/ + + - name: Prepare + run: | + python -VV + ls -l conda_dist/ + conda install -y pypdfium2_${{ inputs.package }} -c bblanchon -c pypdfium2-team -c ./conda_dist/ + + - name: Test raw package + if: inputs.package == 'raw' + run: python conda/raw/minitest.py + + - name: Test helpers package + if: inputs.package == 'helpers' + run: | + conda install -y pytest pillow numpy + ./run test + + publish: + + if: ${{ inputs.publish }} + needs: [build, test] + runs-on: ubuntu-latest + + steps: + + - name: Miniconda setup + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.py }} + channels: bblanchon,pypdfium2-team + channel-priority: strict + + - name: Install deps + run: conda install -y anaconda-client + + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: conda_package + path: conda_dist/ + + # anaconda upload $ARTIFACT_PATH + - name: Upload to Anaconda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + run: | + ARTIFACT_PATH=conda_dist/noarch/pypdfium2_${{ inputs.package }}-*.tar.bz2 + file $ARTIFACT_PATH diff --git a/.github/workflows/gh_pages.yaml b/.github/workflows/gh_pages.yaml index a052bd21b..d4f775b88 100644 --- a/.github/workflows/gh_pages.yaml +++ b/.github/workflows/gh_pages.yaml @@ -25,10 +25,6 @@ concurrency: group: "pages" cancel-in-progress: true -defaults: - run: - shell: bash - jobs: build: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/main.yaml similarity index 91% rename from .github/workflows/build_packages.yaml rename to .github/workflows/main.yaml index c4f6e13df..aed0b61d8 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/main.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2023 geisserml # SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause -name: Build Packages +name: Main packaging on: workflow_dispatch: inputs: @@ -14,6 +14,9 @@ on: publish: default: false type: boolean + conda: + default: false + type: boolean py_version: default: '3.10' type: string @@ -232,10 +235,27 @@ jobs: with: workflow: gh_pages.yaml # takes no inputs + conda_trigger: + needs: [build, test, publish] + if: ${{ inputs.conda && !cancelled() && !contains(needs.*.result, 'failure') }} + runs-on: ${{ inputs.runner }} + + steps: + - name: Trigger conda pypdfium2_helpers build + uses: benc-uk/workflow-dispatch@v1 + with: + workflow: conda.yaml + inputs: | + { + "package": "helpers", + "test": "true", + "publish": "${{ inputs.publish }}", + "py_version": "3.11" + } cleanup: - needs: [build, test, publish] - if: always() + needs: [build, test, publish, conda_trigger] + if: ${{ !cancelled() }} runs-on: ${{ inputs.runner }} steps: diff --git a/.github/workflows/test_release.yaml b/.github/workflows/test_release.yaml index f95d961d7..ea19259a3 100644 --- a/.github/workflows/test_release.yaml +++ b/.github/workflows/test_release.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2023 geisserml # SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause -name: Test Release +name: Test PyPI Release on: workflow_dispatch: inputs: diff --git a/.github/workflows/trigger_conda_raw.yaml b/.github/workflows/trigger_conda_raw.yaml new file mode 100644 index 000000000..fb1ccceed --- /dev/null +++ b/.github/workflows/trigger_conda_raw.yaml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2023 geisserml +# SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause + +name: Trigger conda_raw release +on: + schedule: + # 1 day after pdfium-binaries + - cron: '0 4 * * 2' + workflow_dispatch: + +jobs: + + trigger: + runs-on: ubuntu-latest + + steps: + - name: Trigger + uses: benc-uk/workflow-dispatch@v1 + with: + workflow: conda.yaml + inputs: | + { + "package": "raw", + "test": "true", + "publish": "true", + "py_version": "3.11" + } diff --git a/.github/workflows/trigger_release.yaml b/.github/workflows/trigger_main.yaml similarity index 91% rename from .github/workflows/trigger_release.yaml rename to .github/workflows/trigger_main.yaml index ff67b18b9..e5eaf4293 100644 --- a/.github/workflows/trigger_release.yaml +++ b/.github/workflows/trigger_main.yaml @@ -3,7 +3,7 @@ # Separate trigger workflow because we can't configure inputs for scheduled workflow runs (and don't want publish enabled by default in the main workflow) -name: Trigger Release +name: Trigger main release on: # https://github.com/bblanchon/pdfium-binaries/blob/master/.github/workflows/trigger.yml # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule @@ -13,17 +13,13 @@ on: # - cron: '0 4 * * 2' # old weekly schedule workflow_dispatch: -defaults: - run: - shell: bash - jobs: - release: + trigger: runs-on: ubuntu-latest steps: - - name: Trigger Release + - name: Trigger uses: benc-uk/workflow-dispatch@v1 with: workflow: build_packages.yaml @@ -32,6 +28,7 @@ jobs: "pre_test": "false", "test": "true", "publish": "true", + "conda": "true", "py_version": "3.10", "runner": "ubuntu-latest" } diff --git a/README.md b/README.md index 4642c9310..538d61ce8 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,43 @@ pypdfium2 includes helpers to simplify common use cases, while the raw PDFium/ct ## Installation + + * From PyPI (recommended) ```bash python3 -m pip install -U pypdfium2 ``` This will use a pre-built wheel package, the easiest way of installing pypdfium2. + +* From Conda + + **Beware:** There have been some third-party attempts to conda package pypdfium2 and pdfium-binaries. **Any recipes/packages that might be provided by other distributors, including `anaconda/main` or `conda-forge`, are unofficial!** See below for more info. + + To install the official package: + ```bash + conda config --add channels bblanchon + conda config --add channels pypdfium2-team + conda config --set channel_priority strict + conda install pypdfium2-team::pypdfium2_helpers -c bblanchon -c pypdfium2-team + ``` + You may add `--env` to the `conda config` calls for environment-local config. + + To check if the channels are correct: + ```bash + conda list --show-channel-urls "pypdfium2|pdfium-binaries" + conda config --show-sources + ``` + + To depend on pypdfium2 in a recipe: + ```yaml + requirements: + run: + - pypdfium2-team::pypdfium2_helpers + ``` + You'll want to have downstream callers add the channels as shown above. + + * From source * Dependencies: @@ -63,12 +94,6 @@ pypdfium2 includes helpers to simplify common use cases, while the raw PDFium/ct [^pdfium_buildsystem]: This means pdfium may not compile on arbitrary hosts. The script is limited to build hosts supported by Google's toolchain. Ideally, we'd need an alternative build system that runs with system packages instead. -* Conda - - pypdfium2 will soon provide official conda packages in a custom channel. - The main packaging code is merged, only CI integration / docs are not done yet. - - **Beware:** There have been some third-party attempts to conda package pypdfium2/pdfium-binaries. **Any recipes/packages that might be provided by other distributors, including `anaconda/main` or `conda-forge`, are unofficial!** * Unofficial packages diff --git a/conda/bundle/recipe/meta.yaml b/conda/bundle/recipe/meta.yaml index 9afeca9e9..92e2f37a2 100644 --- a/conda/bundle/recipe/meta.yaml +++ b/conda/bundle/recipe/meta.yaml @@ -8,7 +8,6 @@ # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {% set helpers_ver = environ["M_HELPERS_VER"] %} -{% set git_depth = environ["M_GIT_DEPTH"] %} {% set pl_spec = environ["IN_PDFIUM_PLATFORM"] %} # {% set setup_cfg = load_file_data("setup.cfg") %} @@ -18,7 +17,6 @@ package: source: git_url: ../../.. - git_depth: {{ git_depth }} build: number: 0 diff --git a/conda/helpers/recipe/meta.yaml b/conda/helpers/recipe/meta.yaml index 18e1fda29..48ead560b 100644 --- a/conda/helpers/recipe/meta.yaml +++ b/conda/helpers/recipe/meta.yaml @@ -3,15 +3,13 @@ {% set pdfium_max = environ["PDFIUM_MAX"] %} {% set helpers_ver = environ["M_HELPERS_VER"] %} -{% set git_depth = environ["M_GIT_DEPTH"] %} package: - name: pypdfium2 + name: pypdfium2_helpers version: {{ helpers_ver }} source: git_url: ../../.. - git_depth: {{ git_depth }} build: number: 0 diff --git a/conda/raw/recipe/meta.yaml b/conda/raw/recipe/meta.yaml index 4b0ecd833..e50860b3e 100644 --- a/conda/raw/recipe/meta.yaml +++ b/conda/raw/recipe/meta.yaml @@ -3,7 +3,6 @@ {% set pdfium_short = environ["PDFIUM_SHORT"] %} {% set pdfium_full = environ["PDFIUM_FULL"] %} -{% set git_depth = environ["M_GIT_DEPTH"] %} package: name: pypdfium2_raw @@ -11,7 +10,6 @@ package: source: git_url: ../../.. - git_depth: {{ git_depth }} build: number: 0 diff --git a/setupsrc/pypdfium2_setup/craft_packages.py b/setupsrc/pypdfium2_setup/craft_packages.py index 6c9f8088c..980e50689 100644 --- a/setupsrc/pypdfium2_setup/craft_packages.py +++ b/setupsrc/pypdfium2_setup/craft_packages.py @@ -20,6 +20,7 @@ P_CONDA_RAW = "conda_raw" P_CONDA_HELPERS = "conda_helpers" +CondaChannels = ("-c", "bblanchon", "-c", "pypdfium2-team") CondaDir = ProjectDir / "conda" @@ -109,7 +110,7 @@ def __exit__(self, *_): def run_conda_build(recipe_dir, out_dir, args=[]): with TmpCommitCtx(): - run_cmd(["conda", "build", recipe_dir, "--output-folder", out_dir, *args], cwd=ProjectDir, env=os.environ) + run_cmd(["conda", "build", recipe_dir, "--output-folder", out_dir, *args, *CondaChannels], cwd=ProjectDir, env=os.environ) CondaNames = { @@ -174,6 +175,7 @@ def main_conda_raw(args): def main_conda_helpers(args): + # Set the current pdfium version as upper boundary, for inherent API safety. # Unfortunately, pdfium does not do semantic versioning, so it is hard to achieve safe upward flexibility. # See also https://groups.google.com/g/pdfium/c/kCmgW_gTFYE/m/BPoJgbwOCQAJ @@ -185,6 +187,9 @@ def main_conda_helpers(args): # Assuming a month has 30 days, this would result in # 2 * 30 * (6-2) = 240 os.environ["PDFIUM_MAX"] = str(args.pdfium_ver) + + # NOTE To build with a local pypdfium2_raw, add the args below for the source dir, and remove the pypdfium2-team prefix from the helpers recipe's run requirements + # args=["-c", CondaDir/"raw"/"out"] run_conda_build(CondaDir/"helpers", CondaDir/"helpers"/"out") @@ -245,7 +250,6 @@ def main(): main_pypi(args) elif args.parser.startswith("conda"): helpers_info = parse_git_tag() - os.environ["M_GIT_DEPTH"] = str(helpers_info["n_commits"] + 2) os.environ["M_HELPERS_VER"] = merge_tag(helpers_info, "py") if args.parser == P_CONDA_BUNDLE: main_conda_bundle(args)