diff --git a/.circleci/config.yml b/.circleci/config.yml index e6c439a..8b16708 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: - checkout - setup_remote_docker - run: - name: Build container + name: Build image command: docker build --target test -t ds-python . - run: name: Verify build completed diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ee9065..fd35c1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [8.0.0] +- Core dependencies updated to latest versions: + * awscli 1.33.9 -> 2.17.37 +- Python version updated to v3.12.5 +- Fixes apt-get for debian package installations + ## [7.3.0] - Core dependencies updated to latest versions: * awscli 1.32.112 -> 1.33.9 diff --git a/Dockerfile b/Dockerfile index 17c831c..be39bc5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,13 @@ -FROM python:3.12.4-slim AS production +ARG PLATFORM=linux/x86_64 +ARG BASE_IMAGE=python:3.12.5-slim -LABEL maintainer = support@civisanalytics.com +# This is the primary build target used for the production image +FROM --platform=$PLATFORM $BASE_IMAGE AS production + +# Disable pip warnings https://stackoverflow.com/a/72551258 +ENV PIP_ROOT_USER_ACTION=ignore + +LABEL maintainer=support@civisanalytics.com RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && \ apt-get install -y --no-install-recommends locales && \ @@ -19,7 +26,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && libxrender1 \ wget \ ca-certificates \ - curl && \ + curl \ + mandoc \ + unzip && \ apt-get clean -y && \ rm -rf /var/lib/apt/lists/* @@ -34,14 +43,28 @@ RUN pip install --progress-bar off --no-cache-dir -r requirements-full.txt && \ # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=7.3.0 \ - VERSION_MAJOR=7 \ - VERSION_MINOR=3 \ +ENV VERSION=8.0.0 \ + VERSION_MAJOR=8 \ + VERSION_MINOR=0 \ VERSION_MICRO=0 -FROM production AS test +# Install the AWSCLI for moving match targets in the QC workflow. +# See https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html#cliv2-linux-install +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip && \ + ./aws/install && \ + rm -rf aws awscliv2.zip + +# This build target is for testing in Circle CI. +FROM --platform=$PLATFORM production AS test COPY .circleci/test_image.py . COPY CHANGELOG.md . -# Defaults to production as the final stage +# This build target is for updating dependencies. +# See generate-requirements.full.sh. +FROM --platform=$PLATFORM $BASE_IMAGE AS pip-tools +RUN pip install -U --no-cache-dir pip pip-tools --progress-bar off +CMD ["/bin/bash"] + +# Default to the production build target. FROM production diff --git a/buildspec/merge_master.yaml b/buildspec/merge_master.yaml index 7cfb899..27fcd9d 100644 --- a/buildspec/merge_master.yaml +++ b/buildspec/merge_master.yaml @@ -3,12 +3,12 @@ phases: pre_build: commands: - echo Logging in to Amazon ECR... - - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${REPOSITORY_URI} + - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${FIPS_REPOSITORY_URI} build: commands: - echo Building the Docker image... - - docker build -t ${REPOSITORY_URI}:latest . - - docker image push --all-tags ${REPOSITORY_URI} + - docker build -t ${FIPS_REPOSITORY_URI}:latest . + - docker image push --all-tags ${FIPS_REPOSITORY_URI} post_build: commands: - echo Build completed! diff --git a/buildspec/push.yaml b/buildspec/push.yaml new file mode 100644 index 0000000..671bfa1 --- /dev/null +++ b/buildspec/push.yaml @@ -0,0 +1,19 @@ +version: 0.2 +phases: + build: + commands: + - echo Logging in to Amazon ECR... + - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${FIPS_REPOSITORY_URI} + - export COMMIT_HASH_SHORT="$(echo $COMMIT_HASH | cut -c 1-7)" + - echo Building the Docker image... + - echo $FIPS_REPOSITORY_URI + - echo $COMMIT_HASH_SHORT + - echo $BRANCH_NAME + - docker build --tag ${FIPS_REPOSITORY_URI}:${COMMIT_HASH_SHORT} --tag ${FIPS_REPOSITORY_URI}:${BRANCH_NAME} . + # This config tests the codebuild login and the build but does not push dev images. + # The following lines can be temporarily uncommented to test a dev image. + # - docker image push --all-tags ${FIPS_REPOSITORY_URI} + post_build: + commands: + - echo Build completed! + - printf '{"tag":"%s"}' $COMMIT_HASH_SHORT > build.json diff --git a/buildspec/release.yaml b/buildspec/release.yaml index d31ecd2..add6384 100644 --- a/buildspec/release.yaml +++ b/buildspec/release.yaml @@ -4,13 +4,13 @@ phases: build: commands: - echo Logging in to Amazon ECR... - - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${REPOSITORY_URI} + - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${FIPS_REPOSITORY_URI} - echo Building the Docker image... - PATCH_TAG=${TAG_NAME#"v"} # major.minor.patch - MINOR_TAG=${PATCH_TAG%.*} # major.minor - MAJOR_TAG=${MINOR_TAG%.*} # major - - docker build -t ${REPOSITORY_URI}:${PATCH_TAG} -t ${REPOSITORY_URI}:${MINOR_TAG} -t ${REPOSITORY_URI}:${MAJOR_TAG} . - - docker image push --all-tags ${REPOSITORY_URI} + - docker build -t ${FIPS_REPOSITORY_URI}:${PATCH_TAG} -t ${FIPS_REPOSITORY_URI}:${MINOR_TAG} -t ${FIPS_REPOSITORY_URI}:${MAJOR_TAG} . + - docker image push --all-tags ${FIPS_REPOSITORY_URI} post_build: commands: - echo Build completed! diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..54e6f0a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ +services: + pip-tools: + build: + context: . + dockerfile: ./Dockerfile + target: pip-tools + volumes: + - .:/app + stdin_open: true + tty: true + working_dir: /app \ No newline at end of file diff --git a/generate-requirements-full.sh b/generate-requirements-full.sh index 84753d5..b77b33e 100755 --- a/generate-requirements-full.sh +++ b/generate-requirements-full.sh @@ -1,4 +1,5 @@ #!/bin/bash -python --version -pip install --upgrade pip-tools -pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras --upgrade requirements-core.txt +# Run this script to update requirements-core.txt. +# It uses Docker to ensure that the environment matches what will be used in the production image. +set -e +docker compose run --rm pip-tools /bin/sh -c "pip install --upgrade pip-tools && pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras --upgrade requirements-core.txt" diff --git a/requirements-core.txt b/requirements-core.txt index 2552408..2dad232 100644 --- a/requirements-core.txt +++ b/requirements-core.txt @@ -1,4 +1,3 @@ -awscli==1.33.9 boto3==1.34.127 civis==2.3.0 numpy==2.0.0 diff --git a/requirements-full.txt b/requirements-full.txt index fd068ad..d60c45e 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -4,20 +4,17 @@ # # pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras requirements-core.txt # -attrs==23.2.0 +attrs==24.2.0 # via # jsonschema # referencing -awscli==1.33.9 - # via -r requirements-core.txt boto3==1.34.127 # via -r requirements-core.txt -botocore==1.34.127 +botocore==1.34.162 # via - # awscli # boto3 # s3transfer -certifi==2024.6.2 +certifi==2024.7.4 # via requests charset-normalizer==3.3.2 # via requests @@ -27,11 +24,7 @@ click==8.1.7 # via civis cloudpickle==3.0.0 # via civis -colorama==0.4.6 - # via awscli -docutils==0.16 - # via awscli -idna==3.7 +idna==3.8 # via requests jmespath==1.0.1 # via @@ -43,7 +36,7 @@ joblib==1.4.2 # scikit-learn jsonref==1.1.0 # via civis -jsonschema==4.22.0 +jsonschema==4.23.0 # via civis jsonschema-specifications==2023.12.1 # via jsonschema @@ -55,18 +48,14 @@ numpy==2.0.0 # scipy pandas==2.2.2 # via -r requirements-core.txt -pyasn1==0.6.0 - # via rsa python-dateutil==2.9.0.post0 # via # botocore # pandas pytz==2024.1 # via pandas -pyyaml==6.0.1 - # via - # awscli - # civis +pyyaml==6.0.2 + # via civis referencing==0.35.1 # via # jsonschema @@ -75,16 +64,12 @@ requests==2.32.3 # via # -r requirements-core.txt # civis -rpds-py==0.18.1 +rpds-py==0.20.0 # via # jsonschema # referencing -rsa==4.7.2 - # via awscli -s3transfer==0.10.1 - # via - # awscli - # boto3 +s3transfer==0.10.2 + # via boto3 scikit-learn==1.5.0 # via -r requirements-core.txt scipy==1.13.1 @@ -93,7 +78,7 @@ scipy==1.13.1 # scikit-learn six==1.16.0 # via python-dateutil -tenacity==8.4.1 +tenacity==9.0.0 # via civis threadpoolctl==3.5.0 # via scikit-learn