fail the step if image is not present

jenkins-jobs-scripts/step10/check_ecr_image.py

@@ -0,0 +1,64 @@
+import os
+import boto3
+
+release = os.environ.get("RELEASE_VERSION")
+failed_list = []
+
+ecr = boto3.client('ecr')
+
+def get_ecr_image(services):
+    # command to run gen3 ecr check
+    try:
+        response = ecr.describe_images(
+            repositoryName=f'gen3/{services}',
+            imageIds=[{'imageTag': release}]
+        )
+        image_info = response['imageDetails'][0]
+        print(f"ECR image with tag '{release}' exists in repository 'gen3/{services}")
+        print(f"Image Tag: '{image_info['imageTags']}") 
+    except ecr.exceptions.ImageNotFoundException:
+        print(f"ECR image with tag '{release}' does not exist in repository 'gen3/{services}")
+        failed_list.append(services)
+
+# here
+# key : github repo name
+# value : quay image build name
+repo_dict = {
+    "pelican": "pelican-export",
+    "docker-nginx": "nginx",
+    "gen3-fuse": "gen3fuse-sidecar",
+    "cloud-automation": "awshelper",
+    "ACCESS-backend": "access-backend",
+    "cdis-data-client": "gen3-client",
+}
+
+print("Check if the Quay Images are ready")
+with open("../../repo_list.txt") as repoList:
+    for repo in repoList:
+        repo = repo.strip()
+        if repo in repo_dict:
+            services = repo_dict[repo]
+            get_ecr_image(services)
+            continue
+        elif repo == "sower-jobs":
+            print("Iterating through the list of images for sower-jobs")
+            sower_jobs = [
+                "metadata-manifest-ingestion",
+                "get-dbgap-metadata",
+                "manifest-indexing",
+                "download-indexd-manifest",
+                "batch-export",
+            ]
+            for sower_job in sower_jobs:
+                services = sower_job.strip()
+                get_ecr_image(services)
+                continue
+        else:
+            services = repo
+            get_ecr_image(services)
+
+print(f"List of repos that failed the check : {failed_list}")
+# if the failed_list contains any repo name
+# then the job should fail and print the list
+if failed_list:
+    raise Exception(f"The following services do not have the ECR image for {release}: {failed_list}")

jenkins-jobs-scripts/step10/check_ecr_image.sh

@@ -1,63 +1,3 @@
-#!/bin/bash +x
+#!/bin/bash
 
-set -e
-
-export KUBECTL_NAMESPACE="default"
-
-git clone https://github.com/uc-cdis/cloud-automation.git
-
-# setup gen3 CLI
-source ./cloud-automation/gen3/gen3setup.sh
-
-check_image () {
-    gen3 ecr describe-image $ECR_REPO $RELEASE_VERSION
-    RC=$?
-    if [ $RC -ne 0 ]; then
-        echo "## Release image $RELEASE_VERSION does not exit in repo gen3/$ECR_REPO."
-        FAILED="true"
-    else
-        echo "## Release Image $RELEASE_VERSION exists in repo gen3/$ECR_REPO."
-    fi
-}
-
-repo_list="../../repo_list.txt"
-while IFS= read -r repo; do
-    echo "-------------"
-    echo "## Looking for Image .. "
-    ECR_REPO="$repo"
-    FAILED="false"
-    if [ "$repo" == "pelican" ]; then
-      ECR_REPO="pelican-export"
-    elif [ "$repo" == "docker-nginx" ]; then
-      ECR_REPO="nginx"
-    elif [ "$repo" == "cdis-data-client" ]; then
-      echo "Found a repo called cdis-data-client"
-      echo "there is no docker img for this repo. Ignore..."
-      continue
-    elif [ "$repo" == "gen3-fuse" ]; then
-      ECR_REPO="gen3fuse-sidecar"
-    elif [ "$repo" == "cloud-automation" ]; then
-      ECR_REPO="awshelper"
-    elif [ "$repo" == "sower-jobs" ]; then
-      echo "## iterating through the list ['metadata-manifest-ingestion', 'get-dbgap-metadata', 'manifest-indexing', 'download-indexd-manifest', 'batch-export']"
-      sower_job=(metadata-manifest-ingestion get-dbgap-metadata manifest-indexing download-indexd-manifest batch-export)
-      for sowerjob in "${sower_job[@]}"; do
-        ECR_REPO="$sowerjob"
-        set +e
-        check_image
-        set -e
-      done
-      continue
-
-    elif [ "$repo" == "ACCESS-backend" ]; then
-      ECR_REPO="access-backend"
-    fi
-
-    set +e
-    check_image
-    set -e
-done < "$repo_list"
-
-if [ $FAILED == "true" ]; then
-    exit 1
-fi
+/env/bin/python /src/jenkins-jobs-scripts/step10/check_ecr_image.py

jenkins-jobs-scripts/step4/check_ecr_image.py

@@ -0,0 +1,64 @@
+import os
+import boto3
+
+release = os.environ.get("INTEGRATION_BRANCH")
+failed_list = []
+
+ecr = boto3.client('ecr')
+
+def get_ecr_image(services):
+    # command to run gen3 ecr check
+    try:
+        response = ecr.describe_images(
+            repositoryName=f'gen3/{services}',
+            imageIds=[{'imageTag': release}]
+        )
+        image_info = response['imageDetails'][0]
+        print(f"ECR image with tag '{release}' exists in repository 'gen3/{services}")
+        print(f"Image Tag: '{image_info['imageTags']}") 
+    except ecr.exceptions.ImageNotFoundException:
+        print(f"ECR image with tag '{release}' does not exist in repository 'gen3/{services}")
+        failed_list.append(services)
+
+# here
+# key : github repo name
+# value : quay image build name
+repo_dict = {
+    "pelican": "pelican-export",
+    "docker-nginx": "nginx",
+    "gen3-fuse": "gen3fuse-sidecar",
+    "cloud-automation": "awshelper",
+    "ACCESS-backend": "access-backend",
+    "cdis-data-client": "gen3-client",
+}
+
+print("Check if the Quay Images are ready")
+with open("../../repo_list.txt") as repoList:
+    for repo in repoList:
+        repo = repo.strip()
+        if repo in repo_dict:
+            services = repo_dict[repo]
+            get_ecr_image(services)
+            continue
+        elif repo == "sower-jobs":
+            print("Iterating through the list of images for sower-jobs")
+            sower_jobs = [
+                "metadata-manifest-ingestion",
+                "get-dbgap-metadata",
+                "manifest-indexing",
+                "download-indexd-manifest",
+                "batch-export",
+            ]
+            for sower_job in sower_jobs:
+                services = sower_job.strip()
+                get_ecr_image(services)
+                continue
+        else:
+            services = repo
+            get_ecr_image(services)
+
+print(f"List of repos that failed the check : {failed_list}")
+# if the failed_list contains any repo name
+# then the job should fail and print the list
+if failed_list:
+    raise Exception(f"The following services do not have the ECR image for {release}: {failed_list}")

jenkins-jobs-scripts/step4/check_ecr_image.sh

@@ -1,63 +1,3 @@
-#!/bin/bash +x
+#!/bin/bash
 
-set -e
-
-export KUBECTL_NAMESPACE="default"
-
-git clone https://github.com/uc-cdis/cloud-automation.git
-
-# setup gen3 CLI
-source ./cloud-automation/gen3/gen3setup.sh
-
-check_image () {
-    gen3 ecr describe-image $ECR_REPO $INTEGRATION_BRANCH
-    RC=$?
-    if [ $RC -ne 0 ]; then
-        echo "## Release image $INTEGRATION_BRANCH does not exit in repo gen3/$ECR_REPO."
-        FAILED="true"
-    else
-        echo "## Release Image $INTEGRATION_BRANCH exists in repo gen3/$ECR_REPO."
-    fi
-}
-
-repo_list="../../repo_list.txt"
-while IFS= read -r repo; do
-    echo "-------------"
-    echo "## Looking for Image .. "
-    ECR_REPO="$repo"
-    FAILED="false"
-    if [ "$repo" == "pelican" ]; then
-      ECR_REPO="pelican-export"
-    elif [ "$repo" == "docker-nginx" ]; then
-      ECR_REPO="nginx"
-    elif [ "$repo" == "cdis-data-client" ]; then
-      echo "Found a repo called cdis-data-client"
-      echo "there is no docker img for this repo. Ignore..."
-      continue
-    elif [ "$repo" == "gen3-fuse" ]; then
-      ECR_REPO="gen3fuse-sidecar"
-    elif [ "$repo" == "cloud-automation" ]; then
-      ECR_REPO="awshelper"
-    elif [ "$repo" == "sower-jobs" ]; then
-      echo "## iterating through the list ['metadata-manifest-ingestion', 'get-dbgap-metadata', 'manifest-indexing', 'download-indexd-manifest', 'batch-export']"
-      sower_job=(metadata-manifest-ingestion get-dbgap-metadata manifest-indexing download-indexd-manifest batch-export)
-      for sowerjob in "${sower_job[@]}"; do
-        ECR_REPO="$sowerjob"
-        set +e
-        check_image
-        set -e
-      done
-      continue
-
-    elif [ "$repo" == "ACCESS-backend" ]; then
-      ECR_REPO="access-backend"
-    fi
-
-    set +e
-    check_image
-    set -e
-done < "$repo_list"
-
-if [ $FAILED == "true" ]; then
-    exit 1
-fi
+/env/bin/python /src/jenkins-jobs-scripts/step4/check_ecr_image.py