From 15253e3c683e16b5580c5af49d8ff46e4741abff Mon Sep 17 00:00:00 2001 From: Jackson Lee <86482098+jacksonlee-civis@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:16:00 -0500 Subject: [PATCH] [CIVIS-9806] ENH update civis-python to v2.4.0; add uv (#99) * ENH update base image to python 3.12.7 * ENH update core dependency versions * FIX include awscli v2 in requirements files * TST expected shell commands should be available * ENH add uv to image * MAINT update changelog * FIX version env vars * MAINT update readme * MAINT always build images for dev branches * TST add an apt-get test * FIX set env var for uv to use system python * STY typos * TST uv --- .circleci/test_image.py | 29 ++++++++++++++++ CHANGELOG.md | 13 ++++++++ Dockerfile | 23 ++++++------- README.md | 6 ++-- buildspec/push.yaml | 6 ++-- docker-compose.yml | 2 +- generate-requirements-full.sh | 2 +- requirements-core.txt | 16 +++++---- requirements-full.txt | 63 +++++++++++++++++++++++++---------- 9 files changed, 116 insertions(+), 44 deletions(-) diff --git a/.circleci/test_image.py b/.circleci/test_image.py index 99b6239..04e2df1 100755 --- a/.circleci/test_image.py +++ b/.circleci/test_image.py @@ -2,9 +2,12 @@ import os import re +import shutil +import subprocess import unittest +# Just use the stdlib's `unittest` rather than needing to install `pytest`. class TestImage(unittest.TestCase): def test_version(self): @@ -42,6 +45,32 @@ def test_civis_can_import(self): import civis.ml # noqa: F401 import civis.utils # noqa: F401 + def test_shell_commands_available(self): + """Ensure the main shell commands are available.""" + # A non-exhaustive list of commands -- we just test those we'd likely use. + expected_cmds = "aws civis curl git pip python unzip uv wget".split() + for cmd in expected_cmds: + self.assertIsNotNone(shutil.which(cmd), f"{cmd} not found in PATH") + + def _test_shell_command(self, cmd: str): + """Check if the shell command runs successfully in the image.""" + try: + subprocess.check_call(cmd, shell=True) + except subprocess.CalledProcessError as e: + self.fail( + f"apt-get test failed with return code {e.returncode}\n" + f"stdout: {e.stdout}\n" + f"stderr: {e.stderr}" + ) + + def test_apt_get(self): + """Ensure that apt-get works in the image.""" + self._test_shell_command("apt-get update -y && apt-get install -y htop") + + def test_uv(self): + """Ensure that uv works in the image.""" + self._test_shell_command("uv pip install python-iso639") + if __name__ == "__main__": unittest.main() diff --git a/CHANGELOG.md b/CHANGELOG.md index 91766d8..f792bec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,19 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [8.1.0] + +- Python version updated to v3.12.7 +- Core dependencies updated to latest versions: + * awscli 2.17.37 -> 2.19.5 + * boto3 1.34.127 -> 1.35.58 + * civis 2.3.0 -> 2.4.0 + * numpy 2.0.0 -> 2.1.3 + * pandas 2.2.2 -> 2.2.3 + * scikit-learn 1.5.0 -> 1.5.2 + * scipy 1.13.1 -> 1.14.1 +- uv added to the image + ## [8.0.1] - Python version updated to v3.12.6 diff --git a/Dockerfile b/Dockerfile index 54deebe..040ee80 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG PLATFORM=linux/x86_64 -ARG BASE_IMAGE=python:3.12.6-slim +ARG BASE_IMAGE=python:3.12.7-slim # This is the primary build target used for the production image FROM --platform=$PLATFORM $BASE_IMAGE AS production @@ -37,25 +37,24 @@ COPY requirements-full.txt . RUN pip install --progress-bar off --no-cache-dir -r requirements-full.txt && \ rm requirements-full.txt +# Install uv. +ADD https://astral.sh/uv/0.5.1/install.sh /uv-installer.sh +RUN sh /uv-installer.sh && rm /uv-installer.sh +ENV PATH="/root/.local/bin/:$PATH" \ + UV_SYSTEM_PYTHON=1 + # Instruct joblib to use disk for temporary files. Joblib defaults to # /shm when that directory is present. In the Docker container, /shm is # present but defaults to 64 MB. # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=8.0.1 \ +ENV VERSION=8.1.0 \ VERSION_MAJOR=8 \ - VERSION_MINOR=0 \ - VERSION_MICRO=1 - -# Install the AWSCLI for moving match targets in the QC workflow. -# See https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html#cliv2-linux-install -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ - unzip awscliv2.zip && \ - ./aws/install && \ - rm -rf aws awscliv2.zip + VERSION_MINOR=1 \ + VERSION_MICRO=0 -# This build target is for testing in Circle CI. +# This build target is for testing in CircleCI. FROM --platform=$PLATFORM production AS test COPY .circleci/test_image.py . COPY CHANGELOG.md . diff --git a/README.md b/README.md index ae170de..ea97734 100644 --- a/README.md +++ b/README.md @@ -66,8 +66,8 @@ default location for staging temporary files to the /tmp directory. The normal default is /shm. /shm is a RAM disk which defaults to a 64 MB size in Docker containers, too small for typical scientific computing. -# Updating existing PyPi Package Version -1. Update version of existing package in `requirements-core.txt` +# Updating Existing Package Versions +1. Update versions of existing packages in `requirements-core.txt` 2. Run script `generate-requirements-full.sh` # Creating Equivalent Local Environments @@ -88,7 +88,7 @@ and describe any changes in the [change log](CHANGELOG.md). ## For Maintainers This repo has autobuild enabled. Any PR that is merged to master will -be built as the `latest` tag on Dockerhub. +be built as the `latest` tag on DockerHub. Once you are ready to create a new version, go to the "releases" tab of the repository and click "Draft a new release". GitHub will prompt you to create a new tag, release title, and release description. The tag should use semantic versioning in the form "vX.X.X"; "major.minor.micro". diff --git a/buildspec/push.yaml b/buildspec/push.yaml index 671bfa1..0f10bc4 100644 --- a/buildspec/push.yaml +++ b/buildspec/push.yaml @@ -10,9 +10,9 @@ phases: - echo $COMMIT_HASH_SHORT - echo $BRANCH_NAME - docker build --tag ${FIPS_REPOSITORY_URI}:${COMMIT_HASH_SHORT} --tag ${FIPS_REPOSITORY_URI}:${BRANCH_NAME} . - # This config tests the codebuild login and the build but does not push dev images. - # The following lines can be temporarily uncommented to test a dev image. - # - docker image push --all-tags ${FIPS_REPOSITORY_URI} + # We have a life cycle policy in place to expire and delete images from dev branches, + # so there are no issues with pushing as many of these images as there may be. + - docker image push --all-tags ${FIPS_REPOSITORY_URI} post_build: commands: - echo Build completed! diff --git a/docker-compose.yml b/docker-compose.yml index 54e6f0a..dedb24b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,4 +8,4 @@ services: - .:/app stdin_open: true tty: true - working_dir: /app \ No newline at end of file + working_dir: /app diff --git a/generate-requirements-full.sh b/generate-requirements-full.sh index b77b33e..460b358 100755 --- a/generate-requirements-full.sh +++ b/generate-requirements-full.sh @@ -2,4 +2,4 @@ # Run this script to update requirements-core.txt. # It uses Docker to ensure that the environment matches what will be used in the production image. set -e -docker compose run --rm pip-tools /bin/sh -c "pip install --upgrade pip-tools && pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras --upgrade requirements-core.txt" +docker compose run --rm pip-tools /bin/sh -c "apt-get update -y && apt-get install -y git && pip install --upgrade pip-tools && pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras --upgrade requirements-core.txt" diff --git a/requirements-core.txt b/requirements-core.txt index 2dad232..7f4e1eb 100644 --- a/requirements-core.txt +++ b/requirements-core.txt @@ -1,7 +1,11 @@ -boto3==1.34.127 -civis==2.3.0 -numpy==2.0.0 -pandas==2.2.2 +# awscli v2 is not officially available on PyPI (https://github.com/aws/aws-cli/issues/4947). +# Specifying awscli in requirements-core.txt here ensures that it (and its transitive dependencies) +# are taken into account when generating the final requirements-full.txt file. +awscli @ git+https://github.com/aws/aws-cli@2.19.5 +boto3==1.35.58 +civis==2.4.0 +numpy==2.1.3 +pandas==2.2.3 requests==2.32.3 -scikit-learn==1.5.0 -scipy==1.13.1 +scikit-learn==1.5.2 +scipy==1.14.1 diff --git a/requirements-full.txt b/requirements-full.txt index d60c45e..0476d0a 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -8,26 +8,41 @@ attrs==24.2.0 # via # jsonschema # referencing -boto3==1.34.127 +awscli @ git+https://github.com/aws/aws-cli@2.19.5 # via -r requirements-core.txt -botocore==1.34.162 +awscrt==0.22.0 + # via awscli +boto3==1.35.58 + # via -r requirements-core.txt +botocore==1.35.58 # via # boto3 # s3transfer -certifi==2024.7.4 +certifi==2024.8.30 # via requests -charset-normalizer==3.3.2 +cffi==1.17.1 + # via cryptography +charset-normalizer==3.4.0 # via requests -civis==2.3.0 +civis==2.4.0 # via -r requirements-core.txt click==8.1.7 # via civis -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via civis -idna==3.8 +colorama==0.4.6 + # via awscli +cryptography==43.0.1 + # via awscli +distro==1.8.0 + # via awscli +docutils==0.19 + # via awscli +idna==3.10 # via requests jmespath==1.0.1 # via + # awscli # boto3 # botocore joblib==1.4.2 @@ -38,21 +53,26 @@ jsonref==1.1.0 # via civis jsonschema==4.23.0 # via civis -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via jsonschema -numpy==2.0.0 +numpy==2.1.3 # via # -r requirements-core.txt # pandas # scikit-learn # scipy -pandas==2.2.2 +pandas==2.2.3 # via -r requirements-core.txt -python-dateutil==2.9.0.post0 +prompt-toolkit==3.0.38 + # via awscli +pycparser==2.22 + # via cffi +python-dateutil==2.9.0 # via + # awscli # botocore # pandas -pytz==2024.1 +pytz==2024.2 # via pandas pyyaml==6.0.2 # via civis @@ -64,15 +84,19 @@ requests==2.32.3 # via # -r requirements-core.txt # civis -rpds-py==0.20.0 +rpds-py==0.21.0 # via # jsonschema # referencing -s3transfer==0.10.2 +ruamel-yaml==0.17.21 + # via awscli +ruamel-yaml-clib==0.2.8 + # via awscli +s3transfer==0.10.3 # via boto3 -scikit-learn==1.5.0 +scikit-learn==1.5.2 # via -r requirements-core.txt -scipy==1.13.1 +scipy==1.14.1 # via # -r requirements-core.txt # scikit-learn @@ -82,9 +106,12 @@ tenacity==9.0.0 # via civis threadpoolctl==3.5.0 # via scikit-learn -tzdata==2024.1 +tzdata==2024.2 # via pandas -urllib3==2.2.2 +urllib3==1.26.20 # via + # awscli # botocore # requests +wcwidth==0.2.13 + # via prompt-toolkit