add github action to build all images

uc-cdis · May 30, 2024 · 0be0b2b · 0be0b2b
1 parent 3f6eb37
commit 0be0b2b
Showing 1 changed file with 126 additions and 1 deletion.
diff --git a/.github/workflows/image_build_push.yaml b/.github/workflows/image_build_push.yaml
@@ -3,7 +3,7 @@ name: Build Image and Push to Quay
 on: push
 
 jobs:
-  ci:
+  build-hadoop-base:
     name: Build Hadoop base image
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
@@ -16,3 +16,128 @@ jobs:
       ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
       QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
       QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+  build-spark-base:
+    name: Build Spark base image
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    with:
+      OVERRIDE_REPO_NAME: spark-base
+      OVERRIDE_TAG_NAME: 3.3.0-hadoop3.3
+      DOCKERFILE_LOCATION: "./spark/base/Dockerfile"
+      USE_QUAY_ONLY: true
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+  build-namenode:
+    name: Build namenode image
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    needs: [build-hadoop-base]
+    with:
+      OVERRIDE_REPO_NAME: namenode
+      OVERRIDE_TAG_NAME: 3.3.0
+      DOCKERFILE_LOCATION: "./hadoop/namenode/Dockerfile"
+      USE_QUAY_ONLY: true
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+  build-datanode:
+    name: Build datanode image
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    needs: [build-hadoop-base]
+    with:
+      OVERRIDE_REPO_NAME: namenode
+      OVERRIDE_TAG_NAME: 3.3.0
+      DOCKERFILE_LOCATION: "./hadoop/datanode/Dockerfile"
+      USE_QUAY_ONLY: true
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+  build-nodemanager:
+    name: Build nodemanager image
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    needs: [build-hadoop-base]
+    with:
+      OVERRIDE_REPO_NAME: nodemanager
+      OVERRIDE_TAG_NAME: 3.3.0
+      DOCKERFILE_LOCATION: "./hadoop/nodemanager/Dockerfile"
+      USE_QUAY_ONLY: true
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+  build-resourcemanager:
+    name: Build resourcemanager image
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    needs: [build-hadoop-base]
+    with:
+      OVERRIDE_REPO_NAME: resourcemanager
+      OVERRIDE_TAG_NAME: 3.3.0
+      DOCKERFILE_LOCATION: "./hadoop/resourcemanager/Dockerfile"
+      USE_QUAY_ONLY: true
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+  build-historyserver:
+    name: Build historyserver image
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    needs: [build-hadoop-base]
+    with:
+      OVERRIDE_REPO_NAME: historyserver
+      OVERRIDE_TAG_NAME: 3.3.0-hadoop3.3
+      DOCKERFILE_LOCATION: "./hadoop/historyserver/Dockerfile"
+      USE_QUAY_ONLY: true
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+  build-master:
+    name: Build Spark master image
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    needs: [build-spark-base]
+    with:
+      OVERRIDE_REPO_NAME: spark-master
+      OVERRIDE_TAG_NAME: 3.3.0-hadoop3.3
+      DOCKERFILE_LOCATION: "./spark/master/Dockerfile"
+      USE_QUAY_ONLY: true
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+  build-worker:
+    name: Build Spark worker image
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    needs: [build-spark-base]
+    with:
+      OVERRIDE_REPO_NAME: spark-worker
+      OVERRIDE_TAG_NAME: 3.3.0-hadoop3.3
+      DOCKERFILE_LOCATION: "./spark/worker/Dockerfile"
+      USE_QUAY_ONLY: true
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+  build-submit:
+    name: Build Spark submit image
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    needs: [build-spark-base]
+    with:
+      OVERRIDE_REPO_NAME: spark-submit
+      OVERRIDE_TAG_NAME: 3.3.0-hadoop3.3
+      DOCKERFILE_LOCATION: "./spark/submit/Dockerfile"
+      USE_QUAY_ONLY: true
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}