conda_raw: handle rebuilds (#280)

.github/workflows/conda.yaml

@@ -11,6 +11,9 @@ on:
         options:
           - raw
           - helpers
+      pdfium_ver:
+        default: 'latest'
+        type: string
       test:
         default: true
         type: boolean
@@ -27,7 +30,7 @@ defaults:
   run:
     shell: bash -el {0}
 
-# TODO it would be nice to also support building both packages in one run and testing the helpers package with the newly-built raw package, but this may be failry complicated
+# TODO it would be nice to also support building both packages in one run and testing the helpers package with the newly-built raw package, but this may be fairly complicated
 
 jobs:
 
@@ -58,8 +61,15 @@ jobs:
           git config --global user.name "geisserml"
           python -m pip install -U -r req/setup.txt
       
-      - name: Build package
-        run: ./run craft conda_${{ inputs.package }}
+      - name: Build raw package
+        if: inputs.package == 'raw'
+        run: |
+          python setupsrc/pypdfium2_setup/autorelease_conda_raw.py --pdfium-ver ${{ inputs.pdfium_ver }}
+          ./run craft --pdfium-ver ${{ inputs.pdfium_ver }} conda_raw
+      
+      - name: Build helpers package
+        if: inputs.package == 'helpers'
+        run: ./run craft --pdfium-ver ${{ inputs.pdfium_ver }} conda_helpers
 
       - name: Upload artifact
         uses: actions/upload-artifact@v3

.github/workflows/main.yaml

@@ -248,6 +248,7 @@ jobs:
           inputs: |
             {
               "package": "helpers",
+              "pdfium_ver": "latest",
               "test": "true",
               "publish": "${{ inputs.publish }}",
               "py_version": "3.11"

.github/workflows/trigger_conda_raw.yaml

@@ -21,6 +21,7 @@ jobs:
           inputs: |
             {
               "package": "raw",
+              "pdfium_ver": "latest",
               "test": "true",
               "publish": "true",
               "py_version": "3.11"

conda/raw/recipe/meta.yaml

@@ -3,6 +3,7 @@
 
 {% set pdfium_short = environ["PDFIUM_SHORT"] %}
 {% set pdfium_full = environ["PDFIUM_FULL"] %}
+{% set build_num = environ["BUILD_NUM"] %}
 
 package:
   name: pypdfium2_raw
@@ -12,7 +13,7 @@ source:
   git_url: ../../..
 
 build:
-  number: 0
+  number: {{ build_num }}
   noarch: python
   script_env:
     - PYPDFIUM_MODULES=raw

setupsrc/pypdfium2_setup/autorelease.py

@@ -105,7 +105,7 @@ def log_changes(summary, prev_pdfium, new_pdfium, new_tag, beta):
 def register_changes(new_tag):
     run_local(["git", "checkout", "-B", "autorelease_tmp"])
     run_local(["git", "add", *PlacesToRegister])
-    run_local(["git", "commit", "-m", "[autorelease] update changelog and version file"])
+    run_local(["git", "commit", "-m", f"[autorelease main] update {new_tag}"])
     run_local(["git", "tag", "-a", new_tag, "-m", "Autorelease"])
 
 

setupsrc/pypdfium2_setup/autorelease_conda_raw.py

@@ -0,0 +1,43 @@
+#! /usr/bin/env python3
+# SPDX-FileCopyrightText: 2023 geisserml <[email protected]>
+# SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
+
+import sys
+import json
+import argparse
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parents[1]))
+from pypdfium2_setup.packaging_base import *
+
+
+def main():
+
+    # FIXME we currently traverse the whole list with max() - any chance of (elegantly) avoiding this while maintaining inherent correctness?
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--pdfium-ver", default=None)
+    args = parser.parse_args()
+    is_literal_latest = args.pdfium_ver == "latest"
+    if not args.pdfium_ver or is_literal_latest:
+        args.pdfium_ver = PdfiumVer.get_latest()
+    else:
+        args.pdfium_ver = int(args.pdfium_ver)
+
+    # parse existing releases to automatically handle arbitrary version builds
+    search = run_cmd(["conda", "search", "--json", "pypdfium2_raw", "--override-channels", "-c", "pypdfium2-team"], cwd=None, capture=True)
+    search = reversed(json.loads(search)["pypdfium2_raw"])
+
+    if is_literal_latest:
+        assert args.pdfium_ver > max([int(d["version"]) for d in search]), "Literal latest must resolve to a new version. This is done to avoid rebuilds without new version in scheduled releases. If you want to rebuild, omit --pdfium-ver or pass the resolved value."
+
+    # determine build number
+    build = max([d["build_number"] for d in search if int(d["version"]) == args.pdfium_ver], default=None)
+    build = 0 if build is None else build+1
+    print(build, file=sys.stderr)
+
+    # store build number in a file for use in a subsequent craft_packages call
+    CondaRaw_BuildNumF.write_text(str(build))
+
+
+if __name__ == "__main__":
+    main()

setupsrc/pypdfium2_setup/craft_packages.py

@@ -20,8 +20,6 @@
 P_CONDA_RAW = "conda_raw"
 P_CONDA_HELPERS = "conda_helpers"
 
-CondaDir = ProjectDir / "conda"
-
 
 def parse_args():
 
@@ -30,7 +28,6 @@ def parse_args():
     )
     root_parser.add_argument(
         "--pdfium-ver",
-        type = int,
         default = None,
     )
     subparsers = root_parser.add_subparsers(dest="parser")
@@ -55,8 +52,10 @@ def parse_args():
         )
 
     args = root_parser.parse_args()
-    if not args.pdfium_ver:
+    if not args.pdfium_ver or args.pdfium_ver == "latest":
         args.pdfium_ver = PdfiumVer.get_latest()
+    else:
+        args.pdfium_ver = int(args.pdfium_ver)
     if args.parser == P_CONDA_BUNDLE:
         if args.platforms and args.platforms[0] == "all":
             args.platforms = list(CondaNames.keys())
@@ -168,6 +167,9 @@ def main_conda_bundle(args):
 def main_conda_raw(args):
     os.environ["PDFIUM_SHORT"] = str(args.pdfium_ver)
     os.environ["PDFIUM_FULL"] = ".".join([str(v) for v in PdfiumVer.to_full(args.pdfium_ver)])
+    assert CondaRaw_BuildNumF.exists(), "build number must be given explicitly through conda/raw/build_num.txt - run autorelease_conda_raw.py to create"
+    build_num = int(CondaRaw_BuildNumF.read_text().strip())
+    os.environ["BUILD_NUM"] = str(build_num)
     emplace_func = partial(prepare_setup, ExtPlats.system, args.pdfium_ver, use_v8=None)
     with CondaExtPlatfiles(emplace_func):
         run_conda_build(CondaDir/"raw", CondaDir/"raw"/"out", args=["--override-channels", "-c", "bblanchon"])

setupsrc/pypdfium2_setup/packaging_base.py

@@ -48,10 +48,13 @@
 HAVE_GIT_REPO     = (ProjectDir / ".git").exists()
 
 AutoreleaseDir  = ProjectDir / "autorelease"
-AR_RecordFile   = AutoreleaseDir / "record.json"  # TODO verify contents on before merge
+AR_RecordFile   = AutoreleaseDir / "record.json"
 AR_ConfigFile   = AutoreleaseDir / "config.json"
 RefBindingsFile = AutoreleaseDir / BindingsFN
 
+CondaDir = ProjectDir / "conda"
+CondaRaw_BuildNumF = CondaDir / "raw" / "build_num.txt"
+
 RepositoryURL  = "https://github.com/pypdfium2-team/pypdfium2"
 PdfiumURL      = "https://pdfium.googlesource.com/pdfium"
 DepotToolsURL  = "https://chromium.googlesource.com/chromium/tools/depot_tools.git"