[CIVIS-9315] Fix apt-get and update AWS CLI (#97)

mheilman · web-flow · commit c6a15dde3941 · 2024-08-26T13:46:45.000-05:00
This fixes `apt-get` by updating the base image. Also: * Updates the AWS CLI to v2. * Makes `generate-requirements-full.sh` run in a docker container to ensure the environment matches. * Updates the code build config to use a FIPS URL (see, e.g., civisanalytics/datascience-r#52) and to run builds for branches. * Updates the major version to 8 since the AWS CLI major version changed. * Addresses https://github.com/civisanalytics/datascience-python/security/dependabot/10
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -7,7 +7,7 @@ jobs:
       - checkout
       - setup_remote_docker
       - run:
-          name: Build container
+          name: Build image
           command: docker build --target test -t ds-python .
       - run:
           name: Verify build completed
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,12 @@ Version number changes (major.minor.micro) in this package denote the following:
 
 ## Unreleased
 
+## [8.0.0]
+- Core dependencies updated to latest versions:
+  * awscli 1.33.9 -> 2.17.37
+- Python version updated to v3.12.5
+- Fixes apt-get for debian package installations
+
 ## [7.3.0]
 - Core dependencies updated to latest versions:
   * awscli 1.32.112 -> 1.33.9
diff --git a/Dockerfile b/Dockerfile
@@ -1,6 +1,13 @@
-FROM python:3.12.4-slim AS production
+ARG PLATFORM=linux/x86_64
+ARG BASE_IMAGE=python:3.12.5-slim
 
-LABEL maintainer = support@civisanalytics.com
+# This is the primary build target used for the production image
+FROM --platform=$PLATFORM $BASE_IMAGE AS production
+
+# Disable pip warnings https://stackoverflow.com/a/72551258
+ENV PIP_ROOT_USER_ACTION=ignore
+
+LABEL maintainer=support@civisanalytics.com
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && \
   apt-get install -y --no-install-recommends locales && \
@@ -19,7 +26,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends &&
   libxrender1 \
   wget \
   ca-certificates \
-  curl && \
+  curl \
+  mandoc \
+  unzip && \
   apt-get clean -y && \
   rm -rf /var/lib/apt/lists/*
 
@@ -34,14 +43,28 @@ RUN pip install --progress-bar off --no-cache-dir -r requirements-full.txt && \
 # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342
 ENV JOBLIB_TEMP_FOLDER=/tmp
 
-ENV VERSION=7.3.0 \
-  VERSION_MAJOR=7 \
-  VERSION_MINOR=3 \
+ENV VERSION=8.0.0 \
+  VERSION_MAJOR=8 \
+  VERSION_MINOR=0 \
   VERSION_MICRO=0
 
-FROM production AS test
+# Install the AWSCLI for moving match targets in the QC workflow.
+# See https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html#cliv2-linux-install
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+    unzip awscliv2.zip && \
+    ./aws/install && \
+    rm -rf aws awscliv2.zip
+
+# This build target is for testing in Circle CI.
+FROM --platform=$PLATFORM production AS test
 COPY .circleci/test_image.py .
 COPY CHANGELOG.md .
 
-# Defaults to production as the final stage
+# This build target is for updating dependencies.
+# See generate-requirements.full.sh.
+FROM --platform=$PLATFORM $BASE_IMAGE AS pip-tools
+RUN pip install -U --no-cache-dir pip pip-tools --progress-bar off
+CMD ["/bin/bash"]
+
+# Default to the production build target.
 FROM production
diff --git a/buildspec/merge_master.yaml b/buildspec/merge_master.yaml
@@ -3,12 +3,12 @@ phases:
   pre_build:
     commands:
       - echo Logging in to Amazon ECR...
-      - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${REPOSITORY_URI}
+      - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${FIPS_REPOSITORY_URI}
   build:
     commands:
       - echo Building the Docker image...
-      - docker build -t ${REPOSITORY_URI}:latest .
-      - docker image push --all-tags  ${REPOSITORY_URI}
+      - docker build -t ${FIPS_REPOSITORY_URI}:latest .
+      - docker image push --all-tags  ${FIPS_REPOSITORY_URI}
   post_build:
     commands:
       - echo Build completed!
diff --git a/buildspec/push.yaml b/buildspec/push.yaml
@@ -0,0 +1,19 @@
+version: 0.2
+phases:
+  build:
+    commands:
+      - echo Logging in to Amazon ECR...
+      - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${FIPS_REPOSITORY_URI}
+      - export COMMIT_HASH_SHORT="$(echo $COMMIT_HASH | cut -c 1-7)"
+      - echo Building the Docker image...
+      - echo $FIPS_REPOSITORY_URI
+      - echo $COMMIT_HASH_SHORT
+      - echo $BRANCH_NAME
+      - docker build --tag ${FIPS_REPOSITORY_URI}:${COMMIT_HASH_SHORT} --tag ${FIPS_REPOSITORY_URI}:${BRANCH_NAME} .
+      # This config tests the codebuild login and the build but does not push dev images.
+      # The following lines can be temporarily uncommented to test a dev image.
+      # - docker image push --all-tags ${FIPS_REPOSITORY_URI}
+  post_build:
+    commands:
+      - echo Build completed!
+      - printf '{"tag":"%s"}' $COMMIT_HASH_SHORT > build.json
diff --git a/buildspec/release.yaml b/buildspec/release.yaml
@@ -4,13 +4,13 @@ phases:
   build:
     commands:
       - echo Logging in to Amazon ECR...
-      - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${REPOSITORY_URI}
+      - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${FIPS_REPOSITORY_URI}
       - echo Building the Docker image...
       - PATCH_TAG=${TAG_NAME#"v"} # major.minor.patch
       - MINOR_TAG=${PATCH_TAG%.*} # major.minor
       - MAJOR_TAG=${MINOR_TAG%.*} # major
-      - docker build -t ${REPOSITORY_URI}:${PATCH_TAG} -t ${REPOSITORY_URI}:${MINOR_TAG} -t ${REPOSITORY_URI}:${MAJOR_TAG} .
-      - docker image push --all-tags  ${REPOSITORY_URI}
+      - docker build -t ${FIPS_REPOSITORY_URI}:${PATCH_TAG} -t ${FIPS_REPOSITORY_URI}:${MINOR_TAG} -t ${FIPS_REPOSITORY_URI}:${MAJOR_TAG} .
+      - docker image push --all-tags  ${FIPS_REPOSITORY_URI}
   post_build:
     commands:
       - echo Build completed!
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -0,0 +1,11 @@
+services:
+  pip-tools:
+    build:
+      context: .
+      dockerfile: ./Dockerfile
+      target: pip-tools
+    volumes:
+      - .:/app
+    stdin_open: true
+    tty: true
+    working_dir: /app
diff --git a/generate-requirements-full.sh b/generate-requirements-full.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
-python --version
-pip install --upgrade pip-tools
-pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras --upgrade requirements-core.txt
+# Run this script to update requirements-core.txt.
+# It uses Docker to ensure that the environment matches what will be used in the production image.
+set -e
+docker compose run --rm pip-tools /bin/sh -c "pip install --upgrade pip-tools && pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras --upgrade requirements-core.txt"
diff --git a/requirements-core.txt b/requirements-core.txt
@@ -1,4 +1,3 @@
-awscli==1.33.9
 boto3==1.34.127
 civis==2.3.0
 numpy==2.0.0
diff --git a/requirements-full.txt b/requirements-full.txt
@@ -4,20 +4,17 @@
 #
 #    pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras requirements-core.txt
 #
-attrs==23.2.0
+attrs==24.2.0
     # via
     #   jsonschema
     #   referencing
-awscli==1.33.9
-    # via -r requirements-core.txt
 boto3==1.34.127
     # via -r requirements-core.txt
-botocore==1.34.127
+botocore==1.34.162
     # via
-    #   awscli
     #   boto3
     #   s3transfer
-certifi==2024.6.2
+certifi==2024.7.4
     # via requests
 charset-normalizer==3.3.2
     # via requests
@@ -27,11 +24,7 @@ click==8.1.7
     # via civis
 cloudpickle==3.0.0
     # via civis
-colorama==0.4.6
-    # via awscli
-docutils==0.16
-    # via awscli
-idna==3.7
+idna==3.8
     # via requests
 jmespath==1.0.1
     # via
@@ -43,7 +36,7 @@ joblib==1.4.2
     #   scikit-learn
 jsonref==1.1.0
     # via civis
-jsonschema==4.22.0
+jsonschema==4.23.0
     # via civis
 jsonschema-specifications==2023.12.1
     # via jsonschema
@@ -55,18 +48,14 @@ numpy==2.0.0
     #   scipy
 pandas==2.2.2
     # via -r requirements-core.txt
-pyasn1==0.6.0
-    # via rsa
 python-dateutil==2.9.0.post0
     # via
     #   botocore
     #   pandas
 pytz==2024.1
     # via pandas
-pyyaml==6.0.1
-    # via
-    #   awscli
-    #   civis
+pyyaml==6.0.2
+    # via civis
 referencing==0.35.1
     # via
     #   jsonschema
@@ -75,16 +64,12 @@ requests==2.32.3
     # via
     #   -r requirements-core.txt
     #   civis
-rpds-py==0.18.1
+rpds-py==0.20.0
     # via
     #   jsonschema
     #   referencing
-rsa==4.7.2
-    # via awscli
-s3transfer==0.10.1
-    # via
-    #   awscli
-    #   boto3
+s3transfer==0.10.2
+    # via boto3
 scikit-learn==1.5.0
     # via -r requirements-core.txt
 scipy==1.13.1
@@ -93,7 +78,7 @@ scipy==1.13.1
     #   scikit-learn
 six==1.16.0
     # via python-dateutil
-tenacity==8.4.1
+tenacity==9.0.0
     # via civis
 threadpoolctl==3.5.0
     # via scikit-learn

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,3 @@`
`1`		`-awscli==1.33.9`
`2`	`1`	`boto3==1.34.127`
`3`	`2`	`civis==2.3.0`
`4`	`3`	`numpy==2.0.0`