From d35cac55709d26f908d243a5e38fb44cb819c300 Mon Sep 17 00:00:00 2001 From: Guillaume Fraux Date: Mon, 25 Nov 2024 17:23:57 +0100 Subject: [PATCH] Improve sdist generation for Python packages They no longer require torch/featomic/metatensor dependencies since these are only required to build the wheels. They also automatically package the Rust/C++ sources in a tarball --- MANIFEST.in | 27 ------ python/featomic-torch/.gitignore | 1 - python/featomic-torch/MANIFEST.in | 2 +- .../featomic-torch/build-backend/backend.py | 12 +-- python/featomic-torch/setup.py | 89 +++++++++++++------ python/featomic/AUTHORS | 1 + python/featomic/LICENSE | 1 + python/featomic/MANIFEST.in | 9 ++ python/featomic/README.rst | 1 + .../featomic/pyproject.toml | 18 +--- setup.py => python/featomic/setup.py | 76 ++++++++++++++-- ruff.toml | 11 +++ scripts/clean-python.sh | 8 +- tox.ini | 26 ++---- 14 files changed, 172 insertions(+), 110 deletions(-) delete mode 100644 MANIFEST.in delete mode 100644 python/featomic-torch/.gitignore create mode 120000 python/featomic/AUTHORS create mode 120000 python/featomic/LICENSE create mode 100644 python/featomic/MANIFEST.in create mode 120000 python/featomic/README.rst rename pyproject.toml => python/featomic/pyproject.toml (82%) rename setup.py => python/featomic/setup.py (80%) create mode 100644 ruff.toml diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index c62e5dcf0..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,27 +0,0 @@ -global-exclude *.pyc -global-exclude .DS_Store - -prune docs - -recursive-include featomic * -recursive-include docs/featomic-json-schema * - -# include the minimal crates from the Cargo workspace -include python/Cargo.toml -include python/lib.rs -include featomic-torch/Cargo.toml -include featomic-torch/lib.rs - -include Cargo.* -include pyproject.toml -include AUTHORS -include LICENSE - -prune python/tests -prune python/*.egg-info - -prune featomic/tests -prune featomic/benches/data -prune featomic/examples/data - -exclude tox.ini diff --git a/python/featomic-torch/.gitignore b/python/featomic-torch/.gitignore deleted file mode 100644 index b3bfb1f9c..000000000 --- a/python/featomic-torch/.gitignore +++ /dev/null @@ -1 +0,0 @@ -featomic-torch-*.tar.gz diff --git a/python/featomic-torch/MANIFEST.in b/python/featomic-torch/MANIFEST.in index 64e2d4049..cc10c55c0 100644 --- a/python/featomic-torch/MANIFEST.in +++ b/python/featomic-torch/MANIFEST.in @@ -5,6 +5,6 @@ include pyproject.toml include AUTHORS include LICENSE -include featomic-torch.tar.gz +include featomic-torch-*.tar.gz include build-backend/backend.py diff --git a/python/featomic-torch/build-backend/backend.py b/python/featomic-torch/build-backend/backend.py index 42a8267ae..30f6b7b9b 100644 --- a/python/featomic-torch/build-backend/backend.py +++ b/python/featomic-torch/build-backend/backend.py @@ -7,9 +7,9 @@ from setuptools import build_meta -ROOT = os.path.realpath(os.path.dirname(__file__)) -FEATOMIC_SRC = os.path.realpath(os.path.join(ROOT, "..", "..", "..")) -if os.path.exists(os.path.join(FEATOMIC_SRC, "featomic")): +ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__), "..")) +FEATOMIC_SRC = os.path.realpath(os.path.join(ROOT, "..", "featomic")) +if os.path.exists(FEATOMIC_SRC): # we are building from a git checkout # add a random uuid to the file url to prevent pip from using a cached @@ -21,6 +21,7 @@ FEATOMIC_DEP = "featomic >=0.1.0.dev0,<0.2.0" +get_requires_for_build_sdist = build_meta.get_requires_for_build_sdist prepare_metadata_for_build_wheel = build_meta.prepare_metadata_for_build_wheel build_wheel = build_meta.build_wheel build_sdist = build_meta.build_sdist @@ -33,8 +34,3 @@ def get_requires_for_build_wheel(config_settings=None): "metatensor-torch >=0.6.0,<0.7.0", FEATOMIC_DEP, ] - - -def get_requires_for_build_sdist(config_settings=None): - defaults = build_meta.get_requires_for_build_sdist(config_settings) - return defaults + [FEATOMIC_DEP] diff --git a/python/featomic-torch/setup.py b/python/featomic-torch/setup.py index 1c5ea2c40..434b47617 100644 --- a/python/featomic-torch/setup.py +++ b/python/featomic-torch/setup.py @@ -12,28 +12,8 @@ ROOT = os.path.realpath(os.path.dirname(__file__)) -FEATOMIC_SRC = os.path.join(ROOT, "..", "..", "featomic") - +FEATOMIC_PYTHON_SRC = os.path.join(ROOT, "..", "featomic") FEATOMIC_TORCH_SRC = os.path.join(ROOT, "..", "..", "featomic-torch") -if not os.path.exists(FEATOMIC_TORCH_SRC): - # we are building from a sdist, which should include featomic-torch - # sources as a tarball - tarballs = glob.glob(os.path.join(ROOT, "featomic-torch-cxx-*.tar.gz")) - - if not len(tarballs) == 1: - raise RuntimeError( - "expected a single 'featomic-torch-cxx-*.tar.gz' file containing " - "featomic-torch C++ sources" - ) - - FEATOMIC_TORCH_SRC = os.path.realpath(tarballs[0]) - subprocess.run( - ["cmake", "-E", "tar", "xf", FEATOMIC_TORCH_SRC], - cwd=ROOT, - check=True, - ) - - FEATOMIC_TORCH_SRC = ".".join(FEATOMIC_TORCH_SRC.split(".")[:-2]) class cmake_ext(build_ext): @@ -149,20 +129,54 @@ def run(self): ) -class sdist_git_version(sdist): +class sdist_generate_data(sdist): """ - Create a sdist with an additional generated file containing the extra - version from git. + Create a sdist with an additional generated files: + - `git_extra_version` + - `featomic-torch-cxx-*.tar.gz` """ def run(self): with open("git_extra_version", "w") as fd: fd.write(git_extra_version()) + generate_cxx_tar() + # run original sdist super().run() os.unlink("git_extra_version") + for path in glob.glob("featomic-torch-cxx-*.tar.gz"): + os.unlink(path) + + +def generate_cxx_tar(): + script = os.path.join(ROOT, "..", "..", "scripts", "package-featomic-torch.sh") + assert os.path.exists(script) + + try: + output = subprocess.run( + ["bash", "--version"], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + encoding="utf8", + ) + except Exception as e: + raise RuntimeError("could not run `bash`, is it installed?") from e + + output = subprocess.run( + ["bash", script, os.getcwd()], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + encoding="utf8", + ) + if output.returncode != 0: + stderr = output.stderr + stdout = output.stdout + raise RuntimeError( + "failed to collect C++ sources for Python sdist\n" + f"stdout:\n {stdout}\n\nstderr:\n {stderr}" + ) def git_extra_version(): @@ -223,6 +237,26 @@ def git_extra_version(): if __name__ == "__main__": + if not os.path.exists(FEATOMIC_TORCH_SRC): + # we are building from a sdist, which should include featomic-torch + # sources as a tarball + tarballs = glob.glob(os.path.join(ROOT, "featomic-torch-cxx-*.tar.gz")) + + if not len(tarballs) == 1: + raise RuntimeError( + "expected a single 'featomic-torch-cxx-*.tar.gz' file containing " + "featomic-torch C++ sources" + ) + + FEATOMIC_TORCH_SRC = os.path.realpath(tarballs[0]) + subprocess.run( + ["cmake", "-E", "tar", "xf", FEATOMIC_TORCH_SRC], + cwd=ROOT, + check=True, + ) + + FEATOMIC_TORCH_SRC = ".".join(FEATOMIC_TORCH_SRC.split(".")[:-2]) + if os.path.exists("git_extra_version"): # we are building from a sdist, without git available, but the git # version was recorded in a git_extra_version file @@ -247,14 +281,13 @@ def git_extra_version(): "torch >= 1.12", "metatensor-torch >=0.6.0,<0.7.0", ] - if os.path.exists(FEATOMIC_SRC): + if os.path.exists(FEATOMIC_PYTHON_SRC): # we are building from a git checkout - featomic_path = os.path.realpath(os.path.join(ROOT, "..", "..")) # add a random uuid to the file url to prevent pip from using a cached # wheel for featomic, and force it to re-build from scratch uuid = uuid.uuid4() - install_requires.append(f"featomic @ file://{featomic_path}?{uuid}") + install_requires.append(f"featomic @ file://{FEATOMIC_PYTHON_SRC}?{uuid}") else: # we are building from a sdist/installing from a wheel install_requires.append("featomic >=0.1.0.dev0,<0.2.0") @@ -269,7 +302,7 @@ def git_extra_version(): cmdclass={ "build_ext": cmake_ext, "bdist_egg": bdist_egg if "bdist_egg" in sys.argv else bdist_egg_disabled, - "sdist": sdist_git_version, + "sdist": sdist_generate_data, }, package_data={ "featomic-torch": [ diff --git a/python/featomic/AUTHORS b/python/featomic/AUTHORS new file mode 120000 index 000000000..f04b7e8a2 --- /dev/null +++ b/python/featomic/AUTHORS @@ -0,0 +1 @@ +../../AUTHORS \ No newline at end of file diff --git a/python/featomic/LICENSE b/python/featomic/LICENSE new file mode 120000 index 000000000..30cff7403 --- /dev/null +++ b/python/featomic/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/featomic/MANIFEST.in b/python/featomic/MANIFEST.in new file mode 100644 index 000000000..23e3f0dac --- /dev/null +++ b/python/featomic/MANIFEST.in @@ -0,0 +1,9 @@ +include featomic-cxx-*.tar.gz +include git_extra_version + +include pyproject.toml +include CMakeLists.txt +include AUTHORS +include LICENSE + +prune tests diff --git a/python/featomic/README.rst b/python/featomic/README.rst new file mode 120000 index 000000000..c768ff7d9 --- /dev/null +++ b/python/featomic/README.rst @@ -0,0 +1 @@ +../../README.rst \ No newline at end of file diff --git a/pyproject.toml b/python/featomic/pyproject.toml similarity index 82% rename from pyproject.toml rename to python/featomic/pyproject.toml index 9061e4bb9..59a0c25db 100644 --- a/pyproject.toml +++ b/python/featomic/pyproject.toml @@ -51,27 +51,11 @@ build-backend = "setuptools.build_meta" zip-safe = true [tool.setuptools.packages.find] -where = ["python/featomic"] include = ["featomic*"] namespaces = false ### ======================================================================== ### - -[tool.ruff.lint] -select = ["E", "F", "B", "I"] -ignore = ["B018", "B904"] - -[tool.ruff.lint.isort] -lines-after-imports = 2 -known-first-party = ["featomic"] -known-third-party = ["torch"] - -[tool.ruff.format] -docstring-code-format = true - -### ======================================================================== ### - [tool.pytest.ini_options] python_files = ["*.py"] -testpaths = ["python/featomic/tests"] +testpaths = ["tests"] diff --git a/setup.py b/python/featomic/setup.py similarity index 80% rename from setup.py rename to python/featomic/setup.py index e99b08903..97da2c5cf 100644 --- a/setup.py +++ b/python/featomic/setup.py @@ -13,7 +13,7 @@ ROOT = os.path.realpath(os.path.dirname(__file__)) -FEATOMIC_TORCH = os.path.join(ROOT, "python", "featomic-torch") +FEATOMIC_SRC = os.path.realpath(os.path.join(ROOT, "..", "..", "featomic")) FEATOMIC_BUILD_TYPE = os.environ.get("FEATOMIC_BUILD_TYPE", "release") if FEATOMIC_BUILD_TYPE not in ["debug", "release"]: @@ -23,6 +23,9 @@ ) +FEATOMIC_TORCH_SRC = os.path.join(ROOT, "..", "featomic-torch") + + class universal_wheel(bdist_wheel): """Helper class for override wheel tag. @@ -46,7 +49,7 @@ class cmake_ext(build_ext): def run(self): """Run cmake build and install the resulting library.""" - source_dir = os.path.join(ROOT, "featomic") + source_dir = FEATOMIC_SRC build_dir = os.path.join(ROOT, "build", "cmake-build") install_dir = os.path.join(os.path.realpath(self.build_lib), "featomic") @@ -144,25 +147,59 @@ def run(self): ) -class sdist_git_version(sdist): +class sdist_generate_data(sdist): """ - Create a sdist with an additional generated file containing the extra - version from git. + Create a sdist with an additional generated files: + - `git_extra_version` + - `featomic-cxx-*.tar.gz` """ def run(self): with open("git_extra_version", "w") as fd: fd.write(git_extra_version()) + generate_cxx_tar() + # run original sdist super().run() os.unlink("git_extra_version") + for path in glob.glob("featomic-cxx-*.tar.gz"): + os.unlink(path) + + +def generate_cxx_tar(): + script = os.path.join(ROOT, "..", "..", "scripts", "package-featomic.sh") + assert os.path.exists(script) + + try: + output = subprocess.run( + ["bash", "--version"], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + encoding="utf8", + ) + except Exception as e: + raise RuntimeError("could not run `bash`, is it installed?") from e + + output = subprocess.run( + ["bash", script, os.getcwd()], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + encoding="utf8", + ) + if output.returncode != 0: + stderr = output.stderr + stdout = output.stdout + raise RuntimeError( + "failed to collect C++ sources for Python sdist\n" + f"stdout:\n {stdout}\n\nstderr:\n {stderr}" + ) def get_rust_version(): # read version from Cargo.toml - with open(os.path.join(ROOT, "featomic", "Cargo.toml")) as fd: + with open(os.path.join(FEATOMIC_SRC, "Cargo.toml")) as fd: for line in fd: if line.startswith("version"): _, version = line.split(" = ") @@ -233,6 +270,27 @@ def git_extra_version(): if __name__ == "__main__": + if not os.path.exists(FEATOMIC_SRC): + # we are building from a sdist, which should include featomic Rust + # sources as a tarball + tarballs = glob.glob(os.path.join(ROOT, "featomic-*.tar.gz")) + + if not len(tarballs) == 1: + raise RuntimeError( + "expected a single 'featomic-*.tar.gz' file containing " + "featomic Rust sources. remove all files and re-run " + "scripts/package-featomic.sh" + ) + + FEATOMIC_SRC = os.path.realpath(tarballs[0]) + subprocess.run( + ["cmake", "-E", "tar", "xf", FEATOMIC_SRC], + cwd=ROOT, + check=True, + ) + + FEATOMIC_SRC = ".".join(FEATOMIC_SRC.split(".")[:-2]) + if os.path.exists("git_extra_version"): # we are building from a sdist, without git available, but the git # version was recorded in a git_extra_version file @@ -247,13 +305,13 @@ def git_extra_version(): authors = fd.read().splitlines() extras_require = {} - if os.path.exists(FEATOMIC_TORCH): + if os.path.exists(FEATOMIC_TORCH_SRC): # we are building from a git checkout # add a random uuid to the file url to prevent pip from using a cached # wheel for featomic-torch, and force it to re-build from scratch uuid = uuid.uuid4() - extras_require["torch"] = f"featomic-torch @ file://{FEATOMIC_TORCH}?{uuid}" + extras_require["torch"] = f"featomic-torch @ file://{FEATOMIC_TORCH_SRC}?{uuid}" else: # we are building from a sdist/installing from a wheel extras_require["torch"] = "featomic-torch >=0.1.0.dev0,<0.2.0" @@ -271,7 +329,7 @@ def git_extra_version(): "build_ext": cmake_ext, "bdist_egg": bdist_egg if "bdist_egg" in sys.argv else bdist_egg_disabled, "bdist_wheel": universal_wheel, - "sdist": sdist_git_version, + "sdist": sdist_generate_data, }, package_data={ "featomic": [ diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 000000000..0c20eca99 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,11 @@ +[lint] +select = ["E", "F", "B", "I"] +ignore = ["B018", "B904"] + +[lint.isort] +lines-after-imports = 2 +known-first-party = ["featomic"] +known-third-party = ["torch"] + +[format] +docstring-code-format = true diff --git a/scripts/clean-python.sh b/scripts/clean-python.sh index 8e456c1a8..b23cd475a 100755 --- a/scripts/clean-python.sh +++ b/scripts/clean-python.sh @@ -10,12 +10,18 @@ cd "$ROOT_DIR" rm -rf dist rm -rf build -rm -rf .coverage + rm -rf docs/build rm -rf docs/src/examples +rm -rf python/featomic/dist +rm -rf python/featomic/build +rm -rf python/featomic/featomic-cxx-*.tar.gz + rm -rf python/featomic-torch/dist rm -rf python/featomic-torch/build +rm -rf python/featomic-torch/featomic-torch-cxx-*.tar.gz find . -name "*.egg-info" -exec rm -rf "{}" + find . -name "__pycache__" -exec rm -rf "{}" + +find . -name ".coverage" -exec rm -rf "{}" + diff --git a/tox.ini b/tox.ini index ff1d27797..b2682d061 100644 --- a/tox.ini +++ b/tox.ini @@ -160,36 +160,26 @@ commands = ruff check --fix-only {[testenv]lint-folders} -[testenv:build-python] +[testenv:build-tests] +description = Asserts Pythons package build integrity so one can build sdist and wheels package = skip -# Make sure we can build sdist and a wheel for python deps = - twine build + twine # a tool to check sdist and wheels metadata + pip2pi # tool to create PyPI-like package indexes -allowlist_externals = - bash - +allowlist_externals = bash commands = python --version # print the version of python used in this test - bash ./scripts/package-featomic-torch.sh python/featomic-torch/ - - bash -c "rm -rf {envtmpdir}/dist" - - # check building sdist from a checkout, and wheel from the sdist - python -m build . --outdir {envtmpdir}/dist - - # for featomic-torch, we can not build from a sdist until featomic - # is available on PyPI, so we build both sdist and wheel from a checkout - python -m build python/featomic-torch --sdist --outdir {envtmpdir}/dist - python -m build python/featomic-torch --wheel --outdir {envtmpdir}/dist + bash ./scripts/build-all-wheels.sh {envtmpdir} twine check {envtmpdir}/dist/*.tar.gz twine check {envtmpdir}/dist/*.whl # check building wheels directly from the a checkout - python -m build . --wheel --outdir {envtmpdir}/dist + python -m build python/featomic --wheel --outdir {envtmpdir}/dist + python -m build python/featomic-torch --wheel --outdir {envtmpdir}/dist [flake8]