diff --git a/.github/workflows/setup_python.yml b/.github/workflows/setup_python.yml new file mode 100644 index 0000000..910ea46 --- /dev/null +++ b/.github/workflows/setup_python.yml @@ -0,0 +1,38 @@ +name: setup-python + +on: + push: + branches: + - main + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.11", "3.10", "3.9"] + steps: + - uses: actions/checkout@v4 + + - name: 'Set up Python ${{ matrix.python-version }}' + uses: actions/setup-python@v5 + # https://github.com/marketplace/actions/setup-python + with: + python-version: '${{ matrix.python-version }}' + + - name: 'Just call --help with Python v${{ matrix.python-version }}' + run: | + python3 manageprojects/setup_python.py --help + + - name: 'Call setup python script with Python v${{ matrix.python-version }}' + env: + PYTHONUNBUFFERED: 1 + PYTHONWARNINGS: always + run: | + sudo python3 manageprojects/setup_python.py -vv + + - name: 'Test the installed interpreter' + run: | + $(python3 manageprojects/setup_python.py) -VV diff --git a/README.md b/README.md index 84c585f..360540a 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,8 @@ Besides this, `manageprojects` also includes other generic helper for Python pac * `publish_package()` - Build and upload a new release to PyPi, but with many pre-checks. * `format-file` - Format/Check a Python source file with Darker & Co., useful as IDE action. - * `install_python.py` - Install Python interpreter, if needed, from official Python FTP server, verified. + * `install_python.py` - [Install Python interpreter, if needed, from official Python FTP server, verified.](https://github.com/jedie/manageprojects/blob/main/docs/install_python.md) + * `setup_python.py` - [Download and setup redistributable Python Interpreter, if needed.](https://github.com/jedie/manageprojects/blob/main/docs/setup_python.md) Read below the `Helper` section. @@ -346,7 +347,8 @@ See also git tags: https://github.com/jedie/manageprojects/tags [comment]: <> (✂✂✂ auto generated history start ✂✂✂) -* [**dev**](https://github.com/jedie/manageprojects/compare/v0.18.0...main) +* [v0.19.0](https://github.com/jedie/manageprojects/compare/v0.18.0...v0.19.0) + * 2024-09-15 - NEW: setup_python.py * 2024-09-15 - Update requirements * [v0.18.0](https://github.com/jedie/manageprojects/compare/v0.17.1...v0.18.0) * 2024-08-29 - Fix wrong "module" in publish call :( diff --git a/docs/setup_python.md b/docs/setup_python.md new file mode 100644 index 0000000..8bfc8cc --- /dev/null +++ b/docs/setup_python.md @@ -0,0 +1,135 @@ +# Boot Redistributable Python + +This is a standalone script (no dependencies) to download and setup +https://github.com/indygreg/python-build-standalone/ redistributable Python interpreter. +But only if it's needed! + +Minimal version to used this script is Python v3.9. + +The downloaded archive will be verified with the hash checksum. + +The download will be only done, if the system Python is not the same major version as requested +and if the local Python is not up-to-date. + +## CLI + +The CLI interface looks like e.g.: + +```shell +$ python3 setup_python.py --help + +usage: setup_python.py [-h] [-v] [--skip-temp-deletion] [--force-update] [major_version] + +Download and setup redistributable Python Interpreter from https://github.com/indygreg/python-build-standalone/ if +needed ;) + +positional arguments: + major_version Specify the Python version like: 3.10, 3.11, 3.12, ... (default: 3.12) + +options: + -h, --help show this help message and exit + -v, --verbose Increase verbosity level (can be used multiple times, e.g.: -vv) (default: 0) + --skip-temp-deletion Skip deletion of temporary files (default: False) + --force-update Update local Python interpreter, even if it is up-to-date (default: False) + +``` + +## Include in own projects + +There is a unittest base class to include `setup_python.py` script in your project. +If will check if the file is up2date and if not, it will update it. + +Just include `manageprojects` as a dev dependency in your project. +And add a test like this: + +```python +class IncludeSetupPythonTestCase(IncludeSetupPythonBaseTestCase): + + # Set the path where the `setup_python.py` should be copied to: + DESTINATION_PATH = Path(your_package.__file__).parent) / 'setup_python.py' + + # Just call the method in a test, it will pass, if the file is up2date: + def test_setup_python_is_up2date(self): + self.auto_update_setup_python() +``` + +Feel free to do it in a completely different way, this is just a suggestion ;) + +## Workflow - 1. Check system Python + +If the system Python is the same major version as the required Python, we skip the download. + +The script just returns the path to the system Python interpreter. + +A local installed interpreter (e.g. in "~/.local") will be auto updated. + +## Workflow - 2. Collect latest release data + +We fetch the latest release data from the GitHub API: +https://raw.githubusercontent.com/indygreg/python-build-standalone/latest-release/latest-release.json + +## Workflow - 3. Obtaining optimized Python distribution + +See: https://gregoryszorc.com/docs/python-build-standalone/main/running.html + +We choose the optimized variant based on the priority list: + +1. `pgo+lto` +2. `pgo` +3. `lto` + +For `x86-64` Linux we check the CPU flags from `/proc/cpuinfo` to determine the best variant. + +The "debug" build are ignored. + +## Workflow - 4. Check existing Python + +If the latest Python version is already installed, we skip the download. + +## Workflow - 4. Download and verify Archive + +All downloads will be done with a secure connection (SSL) and server authentication. + +If the latest Python version is already installed, we skip the download. + +Download will be done in a temporary directory. + +We download the archive file and the hash file for verification: + +* Archive extension: `.tar.zst` +* Hash extension: `.tar.zst.sha256` + +We check the file hash after downloading the archive. + +## Workflow - 5. Add info JSON + +We add the file `info.json` with all relevant information. + +## Workflow - 6. Setup Python + +We add a shell script to `~/.local/bin/pythonX.XX` to start the Python interpreter. + +We display version information from Python and pip on `stderr`. + +The extracted Python will be moved to the final destination in `~/.local/pythonX.XX/`. + +The script set's the correct `PYTHONHOME` environment variable. + +## Workflow - 7. print the path + +If no errors occurred, the path to the Python interpreter will be printed to `stdout`. +So it's usable in shell scripts, like: + +```shell +#!/usr/bin/env sh + +set -e + +PY_313_BIN=$(python3 setup_python.py -v 3.13) +echo "Python 3.13 used from: '${PY_313_BIN}'" + +set -x + +${PY_313_BIN} -VV + +``` \ No newline at end of file diff --git a/manageprojects/__init__.py b/manageprojects/__init__.py index 313bea9..e70d607 100644 --- a/manageprojects/__init__.py +++ b/manageprojects/__init__.py @@ -3,5 +3,5 @@ Manage Python / Django projects """ -__version__ = '0.18.0' +__version__ = '0.19.0' __author__ = 'Jens Diemer ' diff --git a/manageprojects/install_python.py b/manageprojects/install_python.py index 8367a6d..71945be 100644 --- a/manageprojects/install_python.py +++ b/manageprojects/install_python.py @@ -38,12 +38,12 @@ """DocWrite: install_python.md # Install Python Interpreter Download Python source code from official Python FTP server: -DocWriteMacro: manageprojects.tests.docwrite_macros.ftp_url""" +DocWriteMacro: manageprojects.tests.docwrite_macros_install_python.ftp_url""" PY_FTP_INDEX_URL = 'https://www.python.org/ftp/python/' """DocWrite: install_python.md ## Supported Python Versions The following major Python versions are supported and verified with GPG keys: -DocWriteMacro: manageprojects.tests.docwrite_macros.supported_python_versions +DocWriteMacro: manageprojects.tests.docwrite_macros_install_python.supported_python_versions The GPG keys taken from the official Python download page: https://www.python.org/downloads/""" GPG_KEY_IDS = { # Thomas Wouters (3.12.x and 3.13.x source files and tags): @@ -58,7 +58,7 @@ """DocWrite: install_python.md ## Workflow - 3. Check local installed Python We assume that the `make altinstall` will install local Python interpreter into: -DocWriteMacro: manageprojects.tests.docwrite_macros.default_install_prefix +DocWriteMacro: manageprojects.tests.docwrite_macros_install_python.default_install_prefix See: https://docs.python.org/3/using/configure.html#cmdoption-prefix""" DEFAULT_INSTALL_PREFIX = '/usr/local' @@ -198,7 +198,7 @@ def install_python( """DocWrite: install_python.md ## Workflow - 2. Get latest Python release We fetch the latest Python release from the Python FTP server, from: - DocWriteMacro: manageprojects.tests.docwrite_macros.ftp_url""" + DocWriteMacro: manageprojects.tests.docwrite_macros_install_python.ftp_url""" # Get latest full version number of Python from Python FTP: py_required_version = get_latest_versions( html=get_html_page(PY_FTP_INDEX_URL), @@ -225,7 +225,7 @@ def install_python( """DocWrite: install_python.md ## Workflow - 4. Download Python sources The download will be done in a temporary directory. The directory will be deleted after the installation. This can be skipped via CLI argument. The directory will be prefixed with: - DocWriteMacro: manageprojects.tests.docwrite_macros.temp_prefix""" + DocWriteMacro: manageprojects.tests.docwrite_macros_install_python.temp_prefix""" with TemporaryDirectory(prefix=TEMP_PREFIX, delete=delete_temp) as temp_path: base_url = f'{PY_FTP_INDEX_URL}{py_required_version}' @@ -291,7 +291,7 @@ def get_parser() -> argparse.ArgumentParser: ```shell $ python3 install_python.py --help - DocWriteMacro: manageprojects.tests.docwrite_macros.help + DocWriteMacro: manageprojects.tests.docwrite_macros_install_python.help ``` """ parser = argparse.ArgumentParser( @@ -349,6 +349,6 @@ def main() -> Path: """DocWrite: install_python.md ## Workflow - 7. print the path If no errors occurred, the path to the Python interpreter will be printed to `stdout`. So it's usable in shell scripts, like: - DocWriteMacro: manageprojects.tests.docwrite_macros.example_shell_script + DocWriteMacro: manageprojects.tests.docwrite_macros_install_python.example_shell_script """ print(python_path) diff --git a/manageprojects/setup_python.py b/manageprojects/setup_python.py new file mode 100644 index 0000000..87a2e55 --- /dev/null +++ b/manageprojects/setup_python.py @@ -0,0 +1,520 @@ +#!/usr/bin/env python3 + +""" + DocWrite: setup_python.md # Boot Redistributable Python + + This is a standalone script (no dependencies) to download and setup + https://github.com/indygreg/python-build-standalone/ redistributable Python interpreter. + But only if it's needed! +""" +from __future__ import annotations + +import argparse +import dataclasses +import datetime +import hashlib +import json +import logging +import platform +import re +import shlex +import shutil +import ssl +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from urllib import request + + +"""DocWrite: setup_python.md # Boot Redistributable Python +Minimal version to used this script is Python v3.9.""" +assert sys.version_info >= (3, 9), f'Python version {sys.version_info} is too old!' + + +DEFAULT_MAJOR_VERSION = '3.12' +GUTHUB_PROJECT = 'indygreg/python-build-standalone' +LASTEST_RELEASE_URL = f'https://raw.githubusercontent.com/{GUTHUB_PROJECT}/latest-release/latest-release.json' +HASH_NAME = 'sha256' +ARCHIVE_EXTENSION = '.tar.zst' +ARCHIVE_HASH_EXTENSION = f'.tar.zst.{HASH_NAME}' + +OPTIMIZATION_PRIORITY = ['pgo+lto', 'pgo', 'lto'] +TEMP_PREFIX = 'redist_python_' +DOWNLOAD_CHUNK_SIZE = 512 * 1024 # 512 KiB + +logger = logging.getLogger(__name__) + + +def assert_is_dir(path): + if not isinstance(path, Path): + path = Path(path) + + if not path.is_dir(): + raise NotADirectoryError(f'Directory does not exists: "{path}"') + + +def assert_is_file(path): + if not isinstance(path, Path): + path = Path(path) + + assert_is_dir(path.parent) + + if not path.is_file(): + raise FileNotFoundError(f'File does not exists: "{path}"') + + +class TemporaryDirectory: + """tempfile.TemporaryDirectory in Python 3.9 has no "delete", yet.""" + + def __init__(self, prefix, delete: bool): + self.prefix = prefix + self.delete = delete + + def __enter__(self) -> Path: + self.temp_path = Path(tempfile.mkdtemp(prefix=self.prefix)) + return self.temp_path + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.delete: + shutil.rmtree(self.temp_path, ignore_errors=True) + if exc_type: + return False + + +def urlopen(url: str): + print(f'Fetching {url}', file=sys.stderr) + """DocWrite: setup_python.md ## Workflow - 4. Download and verify Archive + All downloads will be done with a secure connection (SSL) and server authentication.""" + context = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH) + return request.urlopen(url=url, context=context) + + +def fetch(url: str) -> bytes: + return urlopen(url).read() + + +def fetch_json(url: str) -> dict: + return json.loads(fetch(url)) + + +def download(*, url: str, dst_path: Path, total_size: int, hash_name: str, hash_value: str) -> Path: + """DocWrite: setup_python.md # Boot Redistributable Python + The downloaded archive will be verified with the hash checksum. + """ + filename = Path(url).name + file_path = dst_path / filename + logger.debug('Download %s into %s...', url, file_path) + + file_hash = hashlib.new(hash_name) + response = urlopen(url) + next_update = time.monotonic() + 1 + with file_path.open('wb') as f: + while True: + chunk = response.read(DOWNLOAD_CHUNK_SIZE) + if not chunk: + break + f.write(chunk) + file_hash.update(chunk) + if time.monotonic() >= next_update: + f.flush() + percent = (file_path.stat().st_size / total_size) * 100 + print( + f'\rDownloaded {file_path.stat().st_size} Bytes ({percent:.1f}%)...', + file=sys.stderr, + end='', + flush=True, + ) + next_update += 1 + + file_size = file_path.stat().st_size + print(f'\rDownloaded {file_size} Bytes (100%)', file=sys.stderr, flush=True) + assert file_size == total_size, f'Downloaded {file_size=} Bytes is not expected {total_size=} Bytes!' + + file_hash = file_hash.hexdigest() + logger.debug('Check %s hash...', file_hash) + assert file_hash == hash_value, f'{file_hash=} != {hash_value=}' + print(f'{hash_name} checksum verified: {file_hash!r}, ok.', file=sys.stderr) + + return file_path + + +def removesuffix(text: str, suffix: str) -> str: + assert text.endswith(suffix), f'{text=} does not end with {suffix=}' + return text[: -len(suffix)] + + +def run(args, **kwargs): + logger.debug('Running: %s (%s)', shlex.join(str(arg) for arg in args), kwargs) + return subprocess.run(args, **kwargs) + + +def verbose_check_output(args) -> str: + completed_process = run(args, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + return completed_process.stdout.strip() + + +def get_platform_parts(): + """DocWrite: setup_python.md ## Workflow - 3. Obtaining optimized Python distribution + See: https://gregoryszorc.com/docs/python-build-standalone/main/running.html + """ + parts = [sys.platform] + abi = 'gnu' if any(platform.libc_ver()) else 'musl' + logger.debug('Use %r ABI', abi) + parts.append(abi) + + arch = platform.machine().lower() + if sys.platform == 'linux' and arch == 'x86_64': + """DocWrite: setup_python.md ## Workflow - 3. Obtaining optimized Python distribution + For `x86-64` Linux we check the CPU flags from `/proc/cpuinfo` to determine the best variant.""" + try: + contents = Path('/proc/cpuinfo').read_text() + except OSError: + pass + else: + # Based on https://github.com/pypa/hatch/blob/master/src/hatch/python/resolve.py + # See https://clang.llvm.org/docs/UsersManual.html#x86 for the + # instructions for each architecture variant and + # https://github.com/torvalds/linux/blob/master/arch/x86/include/asm/cpufeatures.h + # for the corresponding Linux flags + v2_flags = {'cx16', 'lahf_lm', 'popcnt', 'pni', 'sse4_1', 'sse4_2', 'ssse3'} + v3_flags = {'avx', 'avx2', 'bmi1', 'bmi2', 'f16c', 'fma', 'movbe', 'xsave'} | v2_flags + v4_flags = {'avx512f', 'avx512bw', 'avx512cd', 'avx512dq', 'avx512vl'} | v3_flags + + cpu_flags = set() + for line in contents.splitlines(): + key, _, value = line.partition(':') + if key.strip() == 'flags': + cpu_flags |= set(value.strip().split()) + + logger.debug('CPU flags: %s', ', '.join(sorted(cpu_flags))) + + missing_v4_flags = v4_flags - cpu_flags + if not missing_v4_flags: + arch = 'x86_64_v4' + else: + logger.debug('Missing v4 flags: %s', ', '.join(sorted(missing_v4_flags))) + missing_v3_flags = v3_flags - cpu_flags + if not missing_v3_flags: + arch = 'x86_64_v3' + else: + logger.debug('Missing v3 flags: %s', ', '.join(sorted(missing_v3_flags))) + missing_v2_flags = v2_flags - cpu_flags + if not missing_v2_flags: + arch = 'x86_64_v2' + else: + logger.debug('Missing v2 flags: %s', ', '.join(sorted(missing_v2_flags))) + + logger.info('Use arch: %r', arch) + parts.append(arch) + return parts + + +def get_best_variant(names): + """DocWrite: setup_python.md ## Workflow - 3. Obtaining optimized Python distribution + We choose the optimized variant based on the priority list: + DocWriteMacro: manageprojects.tests.docwrite_macros_setup_python.optimization_priority + """ + for optimization in OPTIMIZATION_PRIORITY: + for name in names: + if optimization in name: + return name + logger.warning('No optimization found in names: %r', names) + return names[0] + + +def get_python_version(python_bin: str | Path) -> str | None: + logger.debug('Check %s version', python_bin) + if output := run([python_bin, '-V'], capture_output=True, text=True).stdout.split(): + full_version = output[-1] + logger.info('Version of "%s" is: %r', python_bin, full_version) + return full_version + + +@dataclasses.dataclass +class DownloadInfo: + url: str + size: int + + +def setup_python( + *, + major_version: str, + delete_temp: bool = True, + force_update: bool = False, +): + """DocWrite: setup_python.md # Boot Redistributable Python + The download will be only done, if the system Python is not the same major version as requested + and if the local Python is not up-to-date. + """ + + logger.info('Requested major Python version: %s', major_version) + + existing_version = None + existing_python_bin = None + """DocWrite: setup_python.md ## Workflow - 1. Check system Python + If the system Python is the same major version as the required Python, we skip the download.""" + for try_version in (major_version, '3'): + filename = f'python{try_version}' + logger.debug('Checking %s...', filename) + if python3bin := shutil.which(filename): + if (full_version := get_python_version(python3bin)) and full_version.startswith(major_version): + existing_version = full_version + existing_python_bin = python3bin + """DocWrite: setup_python.md ## Workflow - 1. Check system Python + The script just returns the path to the system Python interpreter.""" + + if 'local' in python3bin: + """DocWrite: setup_python.md ## Workflow - 1. Check system Python + A local installed interpreter (e.g. in "~/.local") will be auto updated.""" + continue + + logger.info('System Python v%s already installed: Return path %r of it.', existing_version, python3bin) + return Path(python3bin) + else: + logger.debug('%s not found, ok.', filename) + + logger.debug('Existing Python version: %s', existing_version) + + filters = [ARCHIVE_EXTENSION, *get_platform_parts()] + logger.debug('Use filters: %s', filters) + + """DocWrite: setup_python.md ## Workflow - 2. Collect latest release data + We fetch the latest release data from the GitHub API: + DocWriteMacro: manageprojects.tests.docwrite_macros_setup_python.lastest_release_url""" + data = fetch_json(LASTEST_RELEASE_URL) + logger.debug('Latest release data: %r', data) + tag = data['tag'] + release_url = f'https://api.github.com/repos/{GUTHUB_PROJECT}/releases/tags/{tag}' + release_data = fetch_json(release_url) + assets = release_data['assets'] + + archive_infos = {} + hash_urls = {} + + for asset in assets: + full_name = asset['name'] + if '-debug-' in full_name: + """DocWrite: setup_python.md ## Workflow - 3. Obtaining optimized Python distribution + The "debug" build are ignored.""" + continue + + if not full_name.startswith(f'cpython-{major_version}.'): + # Ignore all other major versions + continue + + if not all(f in full_name for f in filters): + # Ignore incompatible assets + continue + + """DocWrite: setup_python.md ## Workflow - 4. Download and verify Archive + We download the archive file and the hash file for verification: + DocWriteMacro: manageprojects.tests.docwrite_macros_setup_python.extension_info + """ + if full_name.endswith(ARCHIVE_EXTENSION): + name = removesuffix(full_name, ARCHIVE_EXTENSION) + archive_infos[name] = DownloadInfo(url=asset['browser_download_url'], size=asset['size']) + elif full_name.endswith(ARCHIVE_HASH_EXTENSION): + name = removesuffix(full_name, ARCHIVE_HASH_EXTENSION) + hash_urls[name] = asset['browser_download_url'] + + assert archive_infos, f'No "{ARCHIVE_EXTENSION}" found in {assets=}' + assert hash_urls, f'No "{ARCHIVE_HASH_EXTENSION}" found in {assets=}' + + assert archive_infos.keys() == hash_urls.keys(), f'{archive_infos.keys()=} != {hash_urls.keys()=}' + + best_variant = get_best_variant(archive_infos.keys()) + logger.debug('Use best variant: %r', best_variant) + + """DocWrite: setup_python.md ## Workflow - 4. Check existing Python + If the latest Python version is already installed, we skip the download.""" + if existing_python_bin and existing_version: + # full_name e.g.: cpython-3.13.0rc2+20240909-x86_64_v3-unknown-linux-gnu-pgo-full.tar.zst + # get full version: 3.13.0rc2 + if match := re.search(r'cpython-(.+?)\+', best_variant): + full_version = match.group(1) + logger.debug('Available Python version: %s', full_version) + if full_version == existing_version: + logger.info( + 'Local Python v%s is up-to-date: Return path %r of it.', existing_version, existing_python_bin + ) + if force_update: + logger.info('Force update requested: Continue with download ...') + else: + return Path(existing_python_bin) + else: + logger.warning('No version found in %r', best_variant) + + local_path = Path.home() / '.local' + logger.debug('Check "%s" directory', local_path) + local_path.mkdir(parents=False, exist_ok=True) + + """DocWrite: setup_python.md ## Workflow - 4. Download and verify Archive + If the latest Python version is already installed, we skip the download.""" + archive_info: DownloadInfo = archive_infos[best_variant] + logger.debug('Archive info: %s', archive_info) + + hash_url: str = hash_urls[best_variant] + logger.debug('Hash URL: %s', hash_url) + + # Download checksum file: + hash_value = fetch(hash_url).decode().strip() + logger.debug('%s hash value: %s', HASH_NAME, hash_value) + + """DocWrite: setup_python.md ## Workflow - 4. Download and verify Archive + Download will be done in a temporary directory.""" + with TemporaryDirectory(prefix=TEMP_PREFIX, delete=delete_temp) as temp_path: + """DocWrite: setup_python.md ## Workflow - 4. Download and verify Archive + We check the file hash after downloading the archive.""" + archive_temp_path = download( + url=archive_info.url, + dst_path=temp_path, + total_size=archive_info.size, + hash_name=HASH_NAME, + hash_value=hash_value, + ) + + # Extract .tar.zstd archive file into temporary directory: + logger.debug('Extract %s into %s ...', archive_temp_path, temp_path) + run( + ['tar', '--use-compress-program=zstd', '--extract', '--file', archive_temp_path, '--directory', temp_path], + check=True, + ) + + src_path = temp_path / 'python' + assert_is_dir(src_path) + + temp_python_path = src_path / 'install' / 'bin' / 'python3' + assert_is_file(temp_python_path) + + python_version_info = verbose_check_output([str(temp_python_path), '-VV']).strip() + pip_version_into = verbose_check_output([str(temp_python_path), '-m', 'pip', '-VV']).strip() + + """DocWrite: setup_python.md ## Workflow - 5. Add info JSON + We add the file `info.json` with all relevant information.""" + info_file_path = src_path / 'info.json' + info = dict( + download_by=__file__, + download_dt=datetime.datetime.now().isoformat(), + download_filters=filters, + major_version=major_version, + tag=tag, + archive_url=archive_info.url, + hash_url=hash_url, + archive_hash_name=HASH_NAME, + archive_hash_value=hash_value, + python_version_info=python_version_info, + pip_version_info=pip_version_into, + ) + info_file_path.write_text(json.dumps(info, indent=4, ensure_ascii=False)) + + """DocWrite: setup_python.md ## Workflow - 6. Setup Python + The extracted Python will be moved to the final destination in `~/.local/pythonX.XX/`.""" + dest_path = Path.home() / '.local' / f'python{major_version}' + logger.debug('Move %s to %s ...', src_path, dest_path) + if dest_path.exists(): + logger.info('Remove existing %r ...', dest_path) + shutil.rmtree(dest_path) + shutil.move(src_path, dest_path) + + python_home_path = dest_path / 'install' + + """DocWrite: setup_python.md ## Workflow - 6. Setup Python + We add a shell script to `~/.local/bin/pythonX.XX` to start the Python interpreter.""" + bin_path = python_home_path / 'bin' / f'python{major_version}' + assert_is_file(bin_path) + + local_bin_path = Path.home() / '.local' / 'bin' / f'python{major_version}' + logger.debug('Create %s ...', local_bin_path) + local_bin_path.parent.mkdir(parents=True, exist_ok=True) + with local_bin_path.open('w') as f: + """DocWrite: setup_python.md ## Workflow - 6. Setup Python + The script set's the correct `PYTHONHOME` environment variable.""" + f.writelines( + [ + '#!/bin/sh\n', + f'export PYTHONHOME="{python_home_path}"\n', + f'exec "{bin_path}" "$@"\n', + ] + ) + local_bin_path.chmod(0o777) + + """DocWrite: setup_python.md ## Workflow - 6. Setup Python + We display version information from Python and pip on `stderr`.""" + print('Installed Python:', verbose_check_output([str(local_bin_path), '-VV']), file=sys.stderr) + print('Pip info:', verbose_check_output([str(local_bin_path), '-m', 'pip', '-VV']), file=sys.stderr) + + logger.info('Python v%s installed: Return path %r of it.', major_version, local_bin_path) + return local_bin_path + + +def get_parser() -> argparse.ArgumentParser: + """ + DocWrite: setup_python.md ## CLI + The CLI interface looks like e.g.: + + DocWriteMacro: manageprojects.tests.docwrite_macros_setup_python.help + """ + parser = argparse.ArgumentParser( + description=( + 'Download and setup redistributable Python Interpreter' + f' from https://github.com/{GUTHUB_PROJECT}/ if needed ;)' + ), + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + 'major_version', + nargs=argparse.OPTIONAL, + default=DEFAULT_MAJOR_VERSION, + help='Specify the Python version like: 3.10, 3.11, 3.12, ...', + ) + parser.add_argument( + '-v', + '--verbose', + action='count', + default=0, + help='Increase verbosity level (can be used multiple times, e.g.: -vv)', + ) + parser.add_argument( + '--skip-temp-deletion', + action='store_true', + help='Skip deletion of temporary files', + ) + parser.add_argument( + '--force-update', + action='store_true', + help='Update local Python interpreter, even if it is up-to-date', + ) + return parser + + +def main(args=None): + parser = get_parser() + args = parser.parse_args(args=args) + verbose2level = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} + logging.basicConfig( + level=verbose2level.get(args.verbose, logging.DEBUG), + format='%(levelname)9s %(message)s', + stream=sys.stderr, + ) + logger.debug('Arguments: %s', args) + + return setup_python( + major_version=args.major_version, + delete_temp=not args.skip_temp_deletion, + force_update=args.force_update, + ) + + +if __name__ == '__main__': + python_path = main() + + """DocWrite: setup_python.md ## Workflow - 7. print the path + If no errors occurred, the path to the Python interpreter will be printed to `stdout`. + So it's usable in shell scripts, like: + + DocWriteMacro: manageprojects.tests.docwrite_macros_setup_python.example_shell_script + """ + print(python_path) diff --git a/manageprojects/setup_python_example.sh b/manageprojects/setup_python_example.sh new file mode 100644 index 0000000..5633e86 --- /dev/null +++ b/manageprojects/setup_python_example.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh + +set -e + +PY_313_BIN=$(python3 setup_python.py -v 3.13) +echo "Python 3.13 used from: '${PY_313_BIN}'" + +set -x + +${PY_313_BIN} -VV diff --git a/manageprojects/tests/docwrite_macros.py b/manageprojects/tests/docwrite_macros_install_python.py similarity index 100% rename from manageprojects/tests/docwrite_macros.py rename to manageprojects/tests/docwrite_macros_install_python.py diff --git a/manageprojects/tests/docwrite_macros_setup_python.py b/manageprojects/tests/docwrite_macros_setup_python.py new file mode 100644 index 0000000..3b2a1eb --- /dev/null +++ b/manageprojects/tests/docwrite_macros_setup_python.py @@ -0,0 +1,55 @@ +import argparse +from argparse import ArgumentParser +from pathlib import Path +from unittest.mock import patch + +from bx_py_utils.doc_write.data_structures import MacroContext +from bx_py_utils.path import assert_is_file + +from manageprojects import setup_python +from manageprojects.setup_python import ARCHIVE_EXTENSION, ARCHIVE_HASH_EXTENSION, LASTEST_RELEASE_URL + + +PROG = Path(setup_python.__file__).name +EXAMPLE_SCRIPT_PATH = Path(setup_python.__file__).parent / 'setup_python_example.sh' + + +class HelpFormatterMock(argparse.ArgumentDefaultsHelpFormatter): + + def __init__(self, *args, **kwargs): + kwargs['width'] = 120 + kwargs['prog'] = PROG # Force: 'setup_python.py' + super().__init__(*args, **kwargs) + + +def help(macro_context: MacroContext): + yield '```shell' + yield f'$ python3 {PROG} --help\n' + with patch.object(argparse, 'ArgumentDefaultsHelpFormatter', HelpFormatterMock): + parser: ArgumentParser = setup_python.get_parser() + yield parser.format_help() + yield '```' + + +def example_shell_script(macro_context: MacroContext): + assert_is_file(EXAMPLE_SCRIPT_PATH) + + yield '```shell' + yield EXAMPLE_SCRIPT_PATH.read_text() + yield '```' + + +def lastest_release_url(macro_context: MacroContext): + yield LASTEST_RELEASE_URL + + +def optimization_priority(macro_context: MacroContext): + yield '' + for number, optimization in enumerate(setup_python.OPTIMIZATION_PRIORITY, 1): + yield f'{number}. `{optimization}`' + + +def extension_info(macro_context: MacroContext): + yield '' + yield f'* Archive extension: `{ARCHIVE_EXTENSION}`' + yield f'* Hash extension: `{ARCHIVE_HASH_EXTENSION}`' diff --git a/manageprojects/tests/test_install_python.py b/manageprojects/tests/test_install_python.py index e674c97..ab762e3 100644 --- a/manageprojects/tests/test_install_python.py +++ b/manageprojects/tests/test_install_python.py @@ -11,7 +11,7 @@ from rich.rule import Rule from manageprojects.install_python import extract_versions, get_latest_versions -from manageprojects.tests.docwrite_macros import EXAMPLE_SCRIPT_PATH +from manageprojects.tests.docwrite_macros_install_python import EXAMPLE_SCRIPT_PATH from manageprojects.utilities.include_install_python import SOURCE_PATH, IncludeInstallPythonBaseTestCase diff --git a/manageprojects/tests/test_setup_python.py b/manageprojects/tests/test_setup_python.py new file mode 100644 index 0000000..9f8a440 --- /dev/null +++ b/manageprojects/tests/test_setup_python.py @@ -0,0 +1,54 @@ +import filecmp +import subprocess +import sys +import tempfile +from pathlib import Path +from unittest import TestCase + +from bx_py_utils.path import assert_is_file + +from manageprojects.setup_python import main +from manageprojects.utilities.include_setup_python import SOURCE_PATH, IncludeSetupPythonBaseTestCase + + +class SetupPythonTestCase(TestCase): + def test_use_system_python(self): + major_version = f'{sys.version_info.major}.{sys.version_info.minor}' + + python_path = main(args=(major_version,)) + self.assertIsInstance(python_path, Path) + assert_is_file(python_path) + + process = subprocess.run([str(python_path), '-V'], capture_output=True, text=True) + self.assertEqual(process.returncode, 0) + output = process.stdout.strip() + assert output.startswith(f'Python {major_version}.'), f'{output=}' + + +class IncludeSetupPythonTestCase(IncludeSetupPythonBaseTestCase): + maxDiff = None + + def test_auto_update_setup_python(self): + self.assertIsNone(self.DESTINATION_PATH) + self.assertEqual(SOURCE_PATH.name, 'setup_python.py') + + with tempfile.TemporaryDirectory() as temp_dir: + self.DESTINATION_PATH = Path(temp_dir) / 'test.py' + + with self.assertRaises(AssertionError) as cm: + self.auto_update_setup_python() + + self.assertIn('File does not exists', str(cm.exception)) + + self.DESTINATION_PATH.write_text('OLD') + self.assertFalse(filecmp.cmp(SOURCE_PATH, self.DESTINATION_PATH)) + + with self.assertRaises(AssertionError) as cm: + self.auto_update_setup_python() + self.assertIn(' updated, please commit ', str(cm.exception)) + + self.assertTrue(filecmp.cmp(SOURCE_PATH, self.DESTINATION_PATH)) + + result = self.auto_update_setup_python() + self.assertIsNone(result) + self.assertTrue(filecmp.cmp(SOURCE_PATH, self.DESTINATION_PATH)) diff --git a/manageprojects/utilities/include_setup_python.py b/manageprojects/utilities/include_setup_python.py new file mode 100644 index 0000000..3053aec --- /dev/null +++ b/manageprojects/utilities/include_setup_python.py @@ -0,0 +1,58 @@ +import filecmp +from pathlib import Path +from unittest import TestCase + +from bx_py_utils.path import assert_is_dir, assert_is_file + +import manageprojects + + +SOURCE_PATH = Path(manageprojects.__file__).parent / 'setup_python.py' + + +class IncludeSetupPythonBaseTestCase(TestCase): + """DocWrite: setup_python.md ## Include in own projects + There is a unittest base class to include `setup_python.py` script in your project. + If will check if the file is up2date and if not, it will update it. + + Just include `manageprojects` as a dev dependency in your project. + And add a test like this: + + ```python + class IncludeSetupPythonTestCase(IncludeSetupPythonBaseTestCase): + + # Set the path where the `setup_python.py` should be copied to: + DESTINATION_PATH = Path(your_package.__file__).parent) / 'setup_python.py' + + # Just call the method in a test, it will pass, if the file is up2date: + def test_setup_python_is_up2date(self): + self.auto_update_setup_python() + ``` + + Feel free to do it in a completely different way, this is just a suggestion ;) + """ + + DESTINATION_PATH: Path = None # must be set in the subclass + + def auto_update_setup_python(self): + assert_is_file(SOURCE_PATH) + + self.assertIsInstance( + self.DESTINATION_PATH, + Path, + 'Please set the "DESTINATION_PATH" in the subclass', + ) + self.assertEqual(self.DESTINATION_PATH.suffix, '.py') + assert_is_dir(self.DESTINATION_PATH.parent) + + self.assertTrue( + self.DESTINATION_PATH.is_file(), + f'File does not exists: "{self.DESTINATION_PATH}"\n(Please add at least a empty file there!)', + ) + + if filecmp.cmp(SOURCE_PATH, self.DESTINATION_PATH, shallow=False): + # Files are equal -> nothing to do + return + + self.DESTINATION_PATH.write_text(SOURCE_PATH.read_text()) + self.fail(f'File "{self.DESTINATION_PATH}" was updated, please commit the changes') diff --git a/pyproject.toml b/pyproject.toml index e308d80..298802e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -156,7 +156,7 @@ exclude_lines = [ legacy_tox_ini = """ [tox] isolated_build = True -envlist = py{312,311} +envlist = py{313,312,311} skip_missing_interpreters = True [testenv]