Improvements to GH Actions in our CI (#2218)

ROCm · Sep 24, 2023 · 800d19c · 800d19c
1 parent 8c05642
commit 800d19c
Showing 1 changed file with 45 additions and 37 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -11,6 +11,9 @@ on:
 env:
   DOCKER_USER: ${{secrets.DOCKERHUB_USERID}}
   DOCKER_TOKEN: ${{secrets.DOCKERHUB_TOKEN}}
+  DOCKER_IMAGE_UBUNTU: "rocm/migraphx-ci-ubuntu"
+  DOCKER_IMAGE_SLES: "rocm/migraphx-ci-sles"
+
 
 jobs:
   cancel:
@@ -37,26 +40,24 @@ jobs:
       - name: Create Image Tag
         id: image_hash
         run: |
-          echo "imagetag=rocm/migraphx-private:hip-clang-${{hashFiles('**/hip-clang.docker', '**/*requirements.txt', '**/install_prereqs.sh', '**/rbuild.ini')}}" >> $GITHUB_OUTPUT
-          echo "imagetag_sles=rocm/migraphx-sles-private:hip-clang-${{hashFiles('**/tools/docker/sles.docker', '**/*requirements.txt', '**/install_prereqs.sh', '**/rbuild.ini')}}" >> $GITHUB_OUTPUT
+          echo "imagetag=hip-clang-${{hashFiles('**/hip-clang.docker', '**/*requirements.txt', '**/install_prereqs.sh', '**/rbuild.ini')}}" >> $GITHUB_OUTPUT
+          echo "imagetag_sles=hip-clang-${{hashFiles('**/tools/docker/sles.docker', '**/*requirements.txt', '**/install_prereqs.sh', '**/rbuild.ini')}}" >> $GITHUB_OUTPUT
 
       - name: Check if image is built already
         id: check_image
         env:
-          DOCKERIMAGE: ${{ steps.image_hash.outputs.imagetag }}
-          DOCKERIMAGE_SLES: ${{ steps.image_hash.outputs.imagetag_sles }}
+          DOCKER_TAG_UBUNTU: ${{ steps.image_hash.outputs.imagetag }}
+          DOCKER_TAG_SLES: ${{ steps.image_hash.outputs.imagetag_sles }}
         run: |
-          echo $DOCKER_TOKEN | docker login -u $DOCKER_USER --password-stdin
-
-          if [[ "$(docker manifest inspect $DOCKERIMAGE 2> /dev/null)" != "" ]]; then
+          if [[ "$(docker manifest inspect $DOCKER_IMAGE_UBUNTU:$DOCKER_TAG_UBUNTU 2> /dev/null)" != "" ]]; then
             echo "imageexists=true" >> $GITHUB_OUTPUT
             echo "Image already exists, skip building available"
           else
             echo "imageexists=false" >> $GITHUB_OUTPUT
             echo "Tag does not exist, build and publishing required"
           fi
 
-          if [[ "$(docker manifest inspect $DOCKERIMAGE_SLES 2> /dev/null)" != "" ]]; then
+          if [[ "$(docker manifest inspect $DOCKER_IMAGE_SLES:$DOCKER_TAG_SLES 2> /dev/null)" != "" ]]; then
             echo "imageexists_sles=true" >> $GITHUB_OUTPUT
             echo "SLES Image already exists, skip building available"
           else
@@ -75,11 +76,17 @@ jobs:
 
     - name: Build and publish 
       env:            
-        DOCKERIMAGE: ${{ needs.check_image.outputs.imagetag }}
+        DOCKER_TAG_UBUNTU: ${{ needs.check_image.outputs.imagetag }}
       run: |
+        # The TOKEN and USERID are github secrets, Action failures at this step
+        # can come from a PR from a fork changing a file which forces a rebuild
+        # Resolve by making an internal PR of the Forked PR
         echo $DOCKER_TOKEN | docker login -u $DOCKER_USER --password-stdin
-        docker build . --file hip-clang.docker --tag $DOCKERIMAGE;
-        docker push $DOCKERIMAGE;
+
+        docker pull $DOCKER_IMAGE_UBUNTU:latest || true
+        docker build . --file hip-clang.docker --cache-from $DOCKER_IMAGE_UBUNTU:latest --tag $DOCKER_IMAGE_UBUNTU:$DOCKER_TAG_UBUNTU --tag $DOCKER_IMAGE_UBUNTU:latest;
+        docker push $DOCKER_IMAGE_UBUNTU:$DOCKER_TAG_UBUNTU;
+        docker push $DOCKER_IMAGE_UBUNTU:latest;
 
   build_SLES_image:
     name: Build SLES image
@@ -90,18 +97,24 @@ jobs:
     - uses: actions/checkout@v3
     - name: Build and publish SLES
       env:            
-        DOCKERIMAGE_SLES: ${{ needs.check_image.outputs.imagetag_sles }}
+        DOCKER_TAG_SLES: ${{ needs.check_image.outputs.imagetag_sles }}
       run: |
+        # The TOKEN and USERID are github secrets, Action failures at this step
+        # can come from a PR from a fork changing a file wichi forces a rebuild
+        # Resolve by making an internal PR of the Forked PR
         echo $DOCKER_TOKEN | docker login -u $DOCKER_USER --password-stdin
-        docker build . --file tools/docker/sles.docker --tag $DOCKERIMAGE_SLES;
-        docker push $DOCKERIMAGE_SLES;
+
+        docker pull $DOCKER_IMAGE_SLES:latest || true
+        docker build . --file ./tools/docker/sles.docker --cache-from $DOCKER_IMAGE_SLES:latest --tag $DOCKER_IMAGE_SLES:$DOCKER_TAG_SLES --tag $DOCKER_IMAGE_SLES:latest;
+        docker push $DOCKER_IMAGE_SLES:$DOCKER_TAG_SLES;
+        docker push $DOCKER_IMAGE_SLES:latest;
 
 
   tidy:
     runs-on: ROCM-Ubuntu
     needs: [ build_image, check_image ]
     env:
-      DOCKERIMAGE: ${{ needs.check_image.outputs.imagetag }}
+      DOCKER_TAG_UBUNTU: ${{ needs.check_image.outputs.imagetag }}
 
     if: ${{ !cancelled() && (needs.build_image.result == 'success' || needs.build_image.result == 'skipped') }}
     steps: 
@@ -115,12 +128,8 @@ jobs:
         key: tidy-cache-${{ github.ref }}
         restore-keys: tidy-cache-
 
-    - name: Docker Login
-      run: |
-        echo $DOCKER_TOKEN | docker login -u $DOCKER_USER --password-stdin
-
     - name: Clang Tidy
-      shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data $DOCKERIMAGE bash < {0}"
+      shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data $DOCKER_IMAGE_UBUNTU:$DOCKER_TAG_UBUNTU bash < {0}"
       run: |
         mkdir build
         cd build
@@ -159,7 +168,7 @@ jobs:
     runs-on: ROCM-Ubuntu
     needs: [ build_image, check_image ]
     env:
-      DOCKERIMAGE: ${{ needs.check_image.outputs.imagetag }}
+      DOCKER_TAG_UBUNTU: ${{ needs.check_image.outputs.imagetag }}
 
     if: ${{ !cancelled() && (needs.build_image.result == 'success' || needs.build_image.result == 'skipped') }}
     steps:
@@ -173,12 +182,8 @@ jobs:
         key: cppcheck-cache-${{ hashFiles('cppcheck.rules', 'CMakeLists.txt') }}-${{ github.ref }}
         restore-keys: cppcheck-cache-${{ hashFiles('cppcheck.rules', 'CMakeLists.txt') }}-
 
-    - name: Docker Login
-      run: |
-        echo $DOCKER_TOKEN | docker login -u $DOCKER_USER --password-stdin
-
     - name: Cppcheck
-      shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data $DOCKERIMAGE bash < {0}"
+      shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data $DOCKER_IMAGE_UBUNTU:$DOCKER_TAG_UBUNTU bash < {0}"
       run: |
         mkdir build
         cd build
@@ -212,20 +217,27 @@ jobs:
     runs-on: ubuntu-latest
     needs: [ build_image, check_image ]
     env:
-      DOCKERIMAGE: ${{ needs.check_image.outputs.imagetag }}
+      DOCKER_TAG_UBUNTU: ${{ needs.check_image.outputs.imagetag }}
 
     if: ${{ !cancelled() && (needs.build_image.result == 'success' || needs.build_image.result == 'skipped') }}
     steps:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
 
-    - name: Docker Login
-      run: |
-        echo $DOCKER_TOKEN | docker login -u $DOCKER_USER --password-stdin
+    - name: Free space
+      uses: jlumbroso/free-disk-space@main
+      with:
+        tool-cache: true
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: true
+        swap-storage: true
+        docker-images: true
 
     - name: Check formatting
-      shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data $DOCKERIMAGE bash < {0}"
+      shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data $DOCKER_IMAGE_UBUNTU:$DOCKER_TAG_UBUNTU bash < {0}"
       run: |
         set -e
         git config --global --add safe.directory /data
@@ -235,7 +247,7 @@ jobs:
     runs-on: ROCM-Ubuntu
     needs: [ build_SLES_image, check_image ]
     env:
-      DOCKERIMAGE_SLES: ${{ needs.check_image.outputs.imagetag_sles }}    
+      DOCKER_TAG_SLES: ${{ needs.check_image.outputs.imagetag_sles }}
 
     if: ${{ !cancelled() && (needs.build_SLES_image.result == 'success' || needs.build_SLES_image.result == 'skipped') }}      
     steps:
@@ -251,12 +263,8 @@ jobs:
         key: ccache-sles-${{ github.ref }}
         restore-keys: ccache-sles-
 
-    - name: Docker Login
-      run: |
-        echo $DOCKER_TOKEN | docker login -u $DOCKER_USER --password-stdin
-
     - name: Build migraphx
-      shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data $DOCKERIMAGE_SLES bash < {0}"
+      shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data $DOCKER_IMAGE_SLES:$DOCKER_TAG_SLES bash < {0}"
       run: |
         set -e
         export CCACHE_COMPRESSLEVEL=10