-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into feat/brhbuilder
- Loading branch information
Showing
38 changed files
with
1,331 additions
and
883 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| name: Push new nextflow images to ECR | ||
| on: | ||
| workflow_dispatch: | ||
| push: | ||
| branches: master | ||
| paths: | ||
| - 'nextflow-base-images/**' #Runs every time this folder gets updated | ||
| - '.github/workflows/build_and_push_nf_base_images.yml' | ||
| schedule: | ||
| - cron: '0 0 * * 6' # Run at midnight UTC every Saturday | ||
|
|
||
| jobs: | ||
| build_and_push_nf_images: | ||
| runs-on: ubuntu-latest | ||
|
|
||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v3 | ||
| with: | ||
| path: containers | ||
|
|
||
|
|
||
| - name: Checkout other private repository | ||
| uses: actions/checkout@v3 | ||
| with: | ||
| repository: uc-cdis/base-images | ||
| token: ${{ secrets.PLANXCYBORG_TOKEN }} | ||
| path: base-images | ||
|
|
||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v2 | ||
|
|
||
| - name: Set up AWS CLI | ||
| uses: aws-actions/configure-aws-credentials@v2 | ||
| with: | ||
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_ACCT_654654631253_ECR }} | ||
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_ACCT_654654631253_ECR }} | ||
| aws-region: us-east-1 | ||
|
|
||
| - name: Login to Amazon ECR | ||
| run: | | ||
| aws ecr-public get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin public.ecr.aws/u5x5h6w3 | ||
| - name: Build and push Docker images | ||
| run: | | ||
| dir=base-images/amazonlinux-base/ | ||
| echo "Building an image present in $dir" | ||
| image_name=nextflow-approved/public | ||
| tag_name=$(basename "$dir") | ||
| docker build -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir | ||
| docker push public.ecr.aws/u5x5h6w3/$image_name:$tag_name | ||
| echo "Built an image with name --> $image_name:$tag_name" | ||
| for dir in containers/nextflow-base-images/*/;do | ||
| echo "Building an image present in $dir" | ||
| image_name=nextflow-approved/public | ||
| tag_name=$(basename "$dir") | ||
| docker build -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir | ||
| docker push public.ecr.aws/u5x5h6w3/$image_name:$tag_name | ||
| echo "Built an image with name --> $image_name:$tag_name" | ||
| done | ||
| - name: Trigger `get_layer_info_for_nf_imgs.yml` workflow | ||
| env: | ||
| GH_TOKEN: ${{ secrets.PLANXCYBORG_TOKEN }} | ||
| run: | | ||
| gh api \ | ||
| --method POST \ | ||
| -H "Accept: application/vnd.github+json" \ | ||
| -H "X-GitHub-Api-Version: 2022-11-28" \ | ||
| /repos/uc-cdis/containers/actions/workflows/get_layer_info_for_nf_imgs.yml/dispatches \ | ||
| -f "ref=master" | ||
| - name: Delete untagged docker images from public ECR | ||
| run: | | ||
| REPO_NAME=nextflow-approved/public | ||
| IMAGE_IDS=$(aws ecr-public describe-images --repository-name $REPO_NAME --query 'imageDetails[?imageTags==null].imageDigest' --output text) | ||
| if [ -n "$IMAGE_IDS" ]; then | ||
| for IMAGE_ID in $IMAGE_IDS; do | ||
| echo "Deleting image with SHA hash - $IMAGE_ID" | ||
| aws ecr-public batch-delete-image --repository-name $REPO_NAME --image-ids imageDigest=$IMAGE_ID | ||
| done | ||
| else | ||
| echo "No untagged images to delete." | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| name: Build and Push jupyter-covid19 | ||
|
|
||
| on: | ||
| push: | ||
| paths: | ||
| - jupyter-covid19/** | ||
| - .github/workflows/build_jupyter_covid19_image.yml | ||
|
|
||
| jobs: | ||
| jupyter-covid19: | ||
| name: Build and Push jupyter-covid19 | ||
| uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master | ||
| with: | ||
| DOCKERFILE_LOCATION: "./jupyter-covid19/Dockerfile" | ||
| DOCKERFILE_BUILD_CONTEXT: "./jupyter-covid19" | ||
| OVERRIDE_REPO_NAME: "jupyter-notebook" | ||
| OVERRIDE_TAG_NAME: "covid19-$(echo ${GITHUB_REF#refs/*/} | tr / _)" | ||
| secrets: | ||
| ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} | ||
| ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} | ||
| QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} | ||
| QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| name: Update Nextflow-approved image's layers | ||
| on: | ||
| # Primarily this workflow is only expected to be triggered by `build_and_push_nf_base_images.yml`, these extra triggers are just added for convenience. | ||
| workflow_dispatch: | ||
| push: | ||
| branches: master | ||
| paths: | ||
| - 'nextflow-base-images/**' | ||
| - '.github/workflows/get_layer_info_for_nf_imgs.yml' | ||
| jobs: | ||
| get_image_layers: | ||
| runs-on: ubuntu-latest | ||
|
|
||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| ref: store/branch_to_track_nf_img_layers | ||
|
|
||
| - name: Pull layers information and updating nextflow-base-images/approved_img_layers.json | ||
| run: | | ||
| echo "Line $LINENO: starting the script" | ||
| url_list=$(curl -s "https://raw.githubusercontent.com/uc-cdis/containers/master/nextflow-base-images/allowed_base_images.txt") | ||
| max_version_count=2 # Configurable value which indicates maximum of how many versions of past image hashes do we support | ||
| TOKEN=$(curl -s https://public.ecr.aws/token/ | jq -r .token) | ||
| output_file_name="nextflow-base-images/approved_img_layers.json" | ||
| if [ -f $output_file_name ]; then | ||
| layer_json=$(cat $output_file_name 2>/dev/null) | ||
| fi | ||
| if [ -z "$layer_json" ]; then | ||
| layer_json="{}" | ||
| fi | ||
| echo "Line: $LINENO layer_json=$layer_json" | ||
| # Function to fetch the manifest with retry logic for rate limiting | ||
| fetch_manifest() { | ||
| local url=$1 | ||
| local retries=5 | ||
| local wait_time=10 | ||
| local response | ||
| for ((i=0; i<retries; i++)); do | ||
| response=$(curl -s -H "Authorization: Bearer $TOKEN" "$url") | ||
| error_code=$(echo "$response" | jq -r '.errors[0].code // empty') | ||
| if [ "$error_code" == "TOOMANYREQUESTS" ]; then | ||
| echo "Line $LINENO: Rate limit exceeded, waiting for $wait_time seconds before retrying..." >&2 | ||
| sleep $wait_time | ||
| wait_time=$((wait_time * 2)) # Exponential backoff | ||
| else | ||
| echo "$response" | ||
| return | ||
| fi | ||
| done | ||
| echo "Line $LINENO: Failed to fetch manifest after $retries attempts due to rate limiting." >&2 | ||
| return 1 | ||
| } | ||
| while IFS= read -r image_url; do | ||
| # Ignore lines that start with # | ||
| if [[ "$image_url" == \#* ]]; then | ||
| echo "Line_$LINENO: Skipping this line -- $image_url" | ||
| continue | ||
| fi | ||
| # Strip the first * if a line starts with * | ||
| if [[ "$image_url" == \** ]]; then | ||
| echo "Line_$LINENO: Stripping * from this line -- $image_url" | ||
| image_url="${image_url:1}" | ||
| fi | ||
| echo "Line $LINENO: started reading lines from script" | ||
| manifest_url=$(echo "${image_url}" | sed 's|public\.ecr\.aws/\(.*\):\(.*\)|https://public.ecr.aws/v2/\1/manifests/\2|') | ||
| tag_name=$(echo "${image_url}" | sed 's|\(.*\):\(.*\)|\2|') | ||
| echo "$LINENO: manifest_url = $manifest_url, tag_name = $tag_name" | ||
| # Fetch the manifest with retry using exponential backoff | ||
| response=$(fetch_manifest "$manifest_url" | jq "[.layers[].digest]|.[-1]") | ||
| if [ $? -ne 0 ] || [ -z "$response" ]; then | ||
| echo "Line $LINENO: Failed to retrieve valid response from manifest_url -- $manifest_url" >&2 | ||
| continue | ||
| fi | ||
| # Extracting current value of `image_url` if the key exists, otherwise return `{}` | ||
| current_value=$(echo "$layer_json" | jq -e ".[\"$image_url\"] // empty" || echo "{}") | ||
| # Eliminating older values (sorted using string comparison) if there are more than `max_version_count` and return all the remaining values. | ||
| current_value=$(echo "$current_value" | jq --argjson k "$max_version_count" 'to_entries | sort_by(.key) | reverse | if length > $k then .[:$k] else . end | from_entries') | ||
| # Adding a new entry to the current_value json with key as current time stamp and value as the last sha256 of the base image and returning them in descending order of time. | ||
| value=$(echo "$current_value" | jq --arg key "$(date "+%Y_%m_%d_%H:%M:%S")" --argjson response "$response" '. + { ($key) : $response }' | jq 'to_entries | sort_by(.key) | reverse | from_entries') | ||
| # Adding/Updating the updated value along with the image_url as the key. | ||
| layer_json=$(echo "$layer_json" | jq --arg key "$image_url" --argjson value "$value" '. + { ($key): $value }') | ||
| done <<< "$url_list" | ||
| if [ -n "$layer_json" ]; then | ||
| echo "$layer_json" > $output_file_name | ||
| fi | ||
| - name: Commit and push changes | ||
| uses: stefanzweifel/git-auto-commit-action@v5 | ||
| with: | ||
| commit_message: "Update approved_img_layers.json" | ||
| branch: store/branch_to_track_nf_img_layers | ||
| create_branch: true | ||
| commit_options: '-a' | ||
| env: | ||
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
Oops, something went wrong.