Conda packaging code (#268)

No CI integration yet, to be done in a following PR.

.github/workflows/build_packages.yaml

@@ -82,8 +82,8 @@ jobs:
       - name: Run test suite
         run: ./run test
 
-      - name: Run packaging script
-        run: ./run packaging
+      - name: Run PyPI packaging script
+        run: ./run packaging_pypi
 
       - name: Upload release notes
         uses: actions/upload-artifact@v3

.gitignore

@@ -3,6 +3,7 @@
 
 build/
 dist/
+conda/*/out/
 tests/output/
 tests_old/output/
 

conda/bundle/recipe/meta.yaml

@@ -0,0 +1,75 @@
+# SPDX-FileCopyrightText: 2023 geisserml <[email protected]>
+# SPDX-License-Identifier: CC-BY-4.0
+
+# NOTE
+# This is an attempt to bundle the pdfium binaries with conda packages, like we do for pypi wheels.
+# However, conda does not support CPU specific but Python version independent packages, meaning we'd have to build for each Python separately, so the better solution is probably to unbundle pdfium.
+# For now, let's retain this passively as a second option just in case a situation arises where bundling would be desired anyway.
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% set helpers_ver = environ["M_HELPERS_VER"] %}
+{% set git_depth = environ["M_GIT_DEPTH"] %}
+{% set pl_spec = environ["IN_PDFIUM_PLATFORM"] %}
+# {% set setup_cfg = load_file_data("setup.cfg") %}
+
+package:
+  name: pypdfium2_bundle
+  version: {{ helpers_ver }}
+
+source:
+  git_url: ../../..
+  git_depth: {{ git_depth }}
+
+build:
+  number: 0
+  entry_points:
+    - pypdfium2 = pypdfium2.__main__:cli_main
+  script_env:
+    - PDFIUM_PLATFORM=prepared!{{ pl_spec }}
+  script:
+    - {{ PYTHON }} conda/prepare_script.py
+    - {{ PYTHON }} -m pip install . -v --no-deps --no-build-isolation
+
+requirements:
+  # NOTE conda theoretically offers an additional host section, but for our purposes this is effectively the same as the build section
+  build:
+    - git
+    - python
+    - pip
+    - setuptools
+    - wheel !=0.38.0,!=0.38.1
+  run:
+    - python
+
+# pass --no-test if cross-building for non-host target
+# NOTE not embedding the whole helpers test suite to avoid blowing upload size
+test:
+  requires:
+    - pip
+    - python
+  imports:
+    - pypdfium2
+    - pypdfium2_raw  # bundled
+  source_files:
+    - conda/raw/minitest.py
+  commands:
+    - pip check
+    - pypdfium2 --help
+    - python conda/raw/minitest.py
+
+about:
+  summary: Python bindings to PDFium (bundled helpers/binary/bindings)
+  license: (Apache-2.0 OR BSD-3-Clause) AND LicenseRef-PdfiumThirdParty
+  license_file:
+    - LICENSES/Apache-2.0.txt
+    - LICENSES/BSD-3-Clause.txt
+    - LICENSES/CC-BY-4.0.txt
+    - LICENSES/LicenseRef-PdfiumThirdParty.txt
+    - .reuse/dep5-wheel
+  dev_url: https://github.com/pypdfium2-team/pypdfium2
+  doc_url: https://pypdfium2.readthedocs.io
+
+extra:
+  recipe-maintainers:
+    - pypdfium2-team
+    - mara004

conda/helpers/recipe/meta.yaml

@@ -0,0 +1,73 @@
+# SPDX-FileCopyrightText: 2023 geisserml <[email protected]>
+# SPDX-License-Identifier: CC-BY-4.0
+
+{% set pdfium_max = environ["PDFIUM_MAX"] %}
+{% set helpers_ver = environ["M_HELPERS_VER"] %}
+{% set git_depth = environ["M_GIT_DEPTH"] %}
+
+package:
+  name: pypdfium2
+  version: {{ helpers_ver }}
+
+source:
+  git_url: ../../..
+  git_depth: {{ git_depth }}
+
+build:
+  number: 0
+  noarch: python
+  entry_points:
+    - pypdfium2 = pypdfium2.__main__:cli_main
+  script_env:
+    - PYPDFIUM_MODULES=helpers
+  script:
+    - {{ PYTHON }} conda/prepare_script.py
+    - {{ PYTHON }} -m pip install . -v --no-deps --no-build-isolation
+
+requirements:
+  build:
+    - git
+    - python
+    - pip
+    - setuptools
+    - wheel !=0.38.0,!=0.38.1
+  run:
+    # Set an upper boundary for pypdfium2_raw as defined in craft_packages.py
+    # NOTE There currently is no significant minimum pdfium requirement, but we could add one should the necessity arise.
+    - python
+    - pypdfium2_team::pypdfium2_raw <={{ pdfium_max }}
+
+# FIXME this will embed the whole test suite in the package - not sure if that's a good idea
+test:
+  requires:
+    - pip
+    - pytest
+    - pillow
+    - numpy
+  imports:
+    - pypdfium2
+  source_files:
+    - tests/
+    - tests_old/
+  commands:
+    - pip check
+    - pypdfium2 --help
+    - pytest tests/ tests_old/
+
+about:
+  summary: Python bindings to PDFium (helpers, external bindings)
+  description: |
+    This package provides python helpers around pdfium.
+    Dependants are suggested to pin to a major version, but any tighter pinning is discouraged since it increases the risk for conflicts, and would lock you out from future fixes.
+  license: Apache-2.0 OR BSD-3-Clause
+  license_file:
+    - LICENSES/Apache-2.0.txt
+    - LICENSES/BSD-3-Clause.txt
+    - LICENSES/CC-BY-4.0.txt
+  dev_url: https://github.com/pypdfium2-team/pypdfium2
+  doc_url: https://pypdfium2.readthedocs.io
+
+extra:
+  recipe-maintainers:
+    - pypdfium2-team
+    - mara004

conda/prepare_script.py

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: 2023 geisserml <[email protected]>
+# SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parents[1] / "setupsrc"))
+from pypdfium2_setup.craft_packages import TmpCommitCtx
+
+def main():
+    if TmpCommitCtx.FILE.exists():
+        TmpCommitCtx.undo()
+
+if __name__ == "__main__":
+    main()

conda/raw/minitest.py

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: 2023 geisserml <[email protected]>
+# SPDX-License-Identifier: CC-BY-4.0 OR Apache-2.0 OR BSD-3-Clause
+
+# minimal test confirming we can call the library
+
+import pypdfium2_raw as pdfium
+
+# see pypdfium2::_library_scope.py
+init_config = pdfium.FPDF_LIBRARY_CONFIG(
+    version = 2,
+    m_pUserFontPaths = None,
+    m_pIsolate = None,
+    m_v8EmbedderSlot = 0,
+)
+pdfium.FPDF_InitLibraryWithConfig(init_config)
+
+doc = pdfium.FPDF_CreateNewDocument()
+page = pdfium.FPDFPage_New(doc, 0, 595, 842)
+pdfium.FPDF_ClosePage(page)
+pdfium.FPDF_CloseDocument(doc)
+
+pdfium.FPDF_DestroyLibrary()

conda/raw/recipe/meta.yaml

@@ -0,0 +1,67 @@
+# SPDX-FileCopyrightText: 2023 geisserml <[email protected]>
+# SPDX-License-Identifier: CC-BY-4.0
+
+{% set pdfium_short = environ["PDFIUM_SHORT"] %}
+{% set pdfium_full = environ["PDFIUM_FULL"] %}
+{% set git_depth = environ["M_GIT_DEPTH"] %}
+
+package:
+  name: pypdfium2_raw
+  version: {{ pdfium_short }}
+
+source:
+  git_url: ../../..
+  git_depth: {{ git_depth }}
+
+build:
+  number: 0
+  noarch: python
+  script_env:
+    - PYPDFIUM_MODULES=raw
+    - PDFIUM_PLATFORM=prepared!system:{{ pdfium_short }}
+  script:
+    - {{ PYTHON }} conda/prepare_script.py
+    - {{ PYTHON }} -m pip install . -v --no-deps --no-build-isolation
+
+requirements:
+  build:
+    - git
+    - python
+    - pip
+    - setuptools
+    - wheel !=0.38.0,!=0.38.1
+  run:
+    # Pin pdfium-binaries to an exact version to ensure bindings/binary ABI match.
+    # As long as we rebuild pypdfium2_raw continuously in sync with pdfium-binaries, this should not become a flexibility problem.
+    - python
+    - bblanchon::pdfium-binaries =={{ pdfium_full }}
+
+test:
+  requires:
+    - pip
+    - python
+  imports:
+    - pypdfium2_raw
+  source_files:
+    - conda/raw/minitest.py
+  commands:
+    - pip check
+    - python conda/raw/minitest.py
+
+about:
+  summary: Python bindings to PDFium (raw, external binary)
+  description: |
+    This package provides raw ctypes bindings to pdfium.
+    Important: DO NOT PIN to an exact version, as pypdfium2_raw itself pins pdfium-binaries to achieve ABI safety.
+  license: Apache-2.0 OR BSD-3-Clause
+  license_file:
+    - LICENSES/Apache-2.0.txt
+    - LICENSES/BSD-3-Clause.txt
+    - LICENSES/CC-BY-4.0.txt
+  dev_url: https://github.com/pypdfium2-team/pypdfium2
+  doc_url: https://pypdfium2.readthedocs.io
+
+extra:
+  recipe-maintainers:
+    - pypdfium2-team
+    - mara004

docs/devel/changelog_staging.md

@@ -5,4 +5,11 @@
 
 # Changelog for next release
 - Fixed faulty version repr (avoid trailing `+` if desc is empty).
-- Added `PDFIUM_BINDINGS=reference` to use pre-built bindings when installing from source.
+- Merged conda packaging code. The packages build, but there is no CI integration yet.
+- Updated setup code, mainly to support conda.
+  * Independent bindings cache. Download headers from pdfium. Extract archive members explicitly.
+  * Cleaned up version integration of sourcebuild.
+  * Changed `system` platform to generate files according to given version, instead of expecting given files.
+  * Added `provided!` prefix to platform spec, allowing to install with given files.
+  * Added `PDFIUM_BINDINGS=reference` to use pre-built bindings when installing from source.
+- Updated Readme.

run

@@ -14,16 +14,16 @@ function check() {
 }
 
 function clean() {
-    rm -rf pypdfium2*.egg-info/ src/pypdfium2*.egg-info/ build/ dist/ data/* tests/output/* tests_old/output/*
+    rm -rf pypdfium2*.egg-info/ src/pypdfium2*.egg-info/ build/ dist/ data/* tests/output/* tests_old/output/* conda/bundle/out/ conda/helpers/out/ conda/raw/out/
 }
 
-function packaging() {
+function packaging_pypi() {
     clean
     check
 
     # calling update_pdfium is not strictly necessary, but may improve performance because downloads are done in parallel, rather than linear with each package
     python3 setupsrc/pypdfium2_setup/update_pdfium.py
-    python3 setupsrc/pypdfium2_setup/craft_packages.py
+    python3 setupsrc/pypdfium2_setup/craft_packages.py pypi
 
     twine check dist/*
     # ignore W002: erroneous detection of __init__.py files as duplicates
@@ -53,8 +53,14 @@ check)
 clean)
     clean;;
 
-packaging)
-    packaging;;
+packaging_pypi)
+    packaging_pypi;;
+
+update)
+    python3 setupsrc/pypdfium2_setup/update_pdfium.py $args;;
+
+craft)
+    python3 setupsrc/pypdfium2_setup/craft_packages.py $args;;
 
 build)
     python3 setupsrc/pypdfium2_setup/build_pdfium.py $args;;

setup.cfg

@@ -27,10 +27,6 @@ classifiers =
     Topic :: Multimedia :: Graphics
     Topic :: Software Development :: Libraries
 
-[options.entry_points]
-console_scripts =
-    pypdfium2 = pypdfium2.__main__:cli_main
-
 # NOTE We use requirements files instead of [options.extras_require]. Rationale:
 # - BUG(177): PyPI refuses upload if custom deps ($DEP @ git+$URL) are specified.
 # - Installation is independent of pypdfium2 (allows to install setup deps beforehand and then use --no-build-isolation).