From b687525a76549d116987c2f4c3b7044666fe1643 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Sat, 11 Nov 2023 16:25:31 +0530 Subject: [PATCH] merge docker manifests build for different platforms --- .github/workflows/cd-syft.yml | 133 ++++++++++++++++++++++------------ 1 file changed, 87 insertions(+), 46 deletions(-) diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml index 0c9f2991b1a..758c990e3db 100644 --- a/.github/workflows/cd-syft.yml +++ b/.github/workflows/cd-syft.yml @@ -43,6 +43,11 @@ jobs: matrix: runner: [sh-arc-linux-x64, sh-arc-linux-arm64] runs-on: ${{ matrix.runner }} + + outputs: + release_tag: ${{ steps.release_metadata.outputs.release_tag }} + grid_version: ${{ steps.release_metadata.outputs.grid_version }} + steps: - uses: actions/checkout@v3 @@ -98,7 +103,7 @@ jobs: else echo "release_platform=linux/arm64" >> $GITHUB_OUTPUT fi - echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT + echo "grid_version=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT - name: Set up Docker Buildx id: buildx @@ -110,79 +115,120 @@ jobs: username: ${{ secrets.DOCKER_LOGIN }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Docker meta for Grid-Backend - id: meta_grid_backend - uses: docker/metadata-action@v5 - with: - images: openmined/grid-backend - tags: | - type=raw,value=${{ steps.release_metadata.outputs.GRID_VERSION }} - type=raw,value=${{ steps.release_metadata.outputs.release_tag }} - - name: Build and push `grid-backend` image to DockerHub + id: grid-backend-build uses: docker/build-push-action@v5 with: context: ./packages file: ./packages/grid/backend/backend.dockerfile - push: true platforms: ${{ steps.release_metadata.outputs.release_platform }} - tags: ${{ steps.meta_grid_backend.outputs.tags }} target: backend + outputs: type=image,name=openmined/grid-backend,push-by-digest=true,name-canonical=true,push=true cache-from: type=gha cache-to: type=gha,mode=max - - name: Docker meta for Grid-Frontend - id: meta_grid_frontend - uses: docker/metadata-action@v5 - with: - images: openmined/grid-frontend - tags: | - type=raw,value=${{ steps.release_metadata.outputs.GRID_VERSION }} - type=raw,value=${{ steps.release_metadata.outputs.release_tag }} + - name: Export digest for grid-backend + run: | + mkdir -p /tmp/digests/grid-backend + digest="${{ steps.grid-backend-build.outputs.digest }}" + touch "/tmp/digests/grid-backend/${digest#sha256:}" - name: Build and push `grid-frontend` image to DockerHub + id: grid-frontend-build uses: docker/build-push-action@v5 with: context: ./packages/grid/frontend file: ./packages/grid/frontend/frontend.dockerfile - push: true platforms: ${{ steps.release_metadata.outputs.release_platform }} - tags: ${{ steps.meta_grid_frontend.outputs.tags }} + outputs: type=image,name=openmined/grid-frontend,push-by-digest=true,name-canonical=true,push=true target: grid-ui-development cache-from: type=gha cache-to: type=gha,mode=max - - name: Docker meta for Grid-Enclave - id: meta_grid_enclave - uses: docker/metadata-action@v5 - with: - images: openmined/grid-enclave - tags: | - type=raw,value=${{ steps.release_metadata.outputs.GRID_VERSION }} - type=raw,value=${{ steps.release_metadata.outputs.release_tag }} + - name: Export digest for grid-frontend + run: | + mkdir -p /tmp/digests/grid-frontend + digest="${{ steps.grid-frontend-build.outputs.digest }}" + touch "/tmp/digests/grid-frontend/${digest#sha256:}" - name: Build and push `grid-enclave` image to DockerHub + id: grid-enclave-build uses: docker/build-push-action@v5 with: context: ./packages file: ./packages/grid/worker/worker.dockerfile - push: true platforms: ${{ steps.release_metadata.outputs.release_platform }} - tags: ${{ steps.meta_grid_enclave.outputs.tags }} + outputs: type=image,name=openmined/grid-enclave,push-by-digest=true,name-canonical=true,push=true target: worker cache-from: type=gha cache-to: type=gha,mode=max + - name: Export digest for grid-enclave + run: | + mkdir -p /tmp/digests/grid-enclave + digest="${{ steps.grid-enclave-build.outputs.digest }}" + touch "/tmp/digests/grid-enclave/${digest#sha256:}" + + - name: Upload digest for grid-backend, grid-frontend and grid-enclave + uses: actions/upload-artifact@v3 + with: + name: digests-${{ steps.release_metadata.outputs.grid_version }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + #Used to merge x64 and arm64 into one docker image + merge-docker-images: + needs: [build-and-push-docker-images] + if: always() && (needs.build-and-push-docker-images.result == 'success') + + runs-on: sh-arc-linux-x64 + + steps: + - name: Download digests + uses: actions/download-artifact@v3 + with: + name: digests-${{ needs.build-and-push-docker-images.outputs.grid_version }} + path: /tmp/digests + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_LOGIN }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Create manifest list and push for grid-backend + working-directory: /tmp/digests/grid-backend + run: | + docker buildx imagetools create \ + -t openmined/grid-backend:${{ needs.build-and-push-docker-images.outputs.grid_version }} \ + -t openmined/grid-backend:${{ needs.build-and-push-docker-images.outputs.release_tag }} \ + $(printf 'opemined/grid-backend@sha256:%s ' *) + + - name: Create manifest list and push for grid-frontend + working-directory: /tmp/digests/grid-frontend + run: | + docker buildx imagetools create \ + -t openmined/grid-frontend:${{ needs.build-and-push-docker-images.outputs.grid_version }} \ + -t openmined/grid-frontend:${{ needs.build-and-push-docker-images.outputs.release_tag }} \ + $(printf 'opemined/grid-frontend@sha256:%s ' *) + + - name: Create manifest list and push for grid-enclave + working-directory: /tmp/digests/grid-enclave + run: | + docker buildx imagetools create \ + -t openmined/grid-enclave:${{ needs.build-and-push-docker-images.outputs.grid_version }} \ + -t openmined/grid-enclave:${{ needs.build-and-push-docker-images.outputs.release_tag }} \ + $(printf 'opemined/grid-enclave@sha256:%s ' *) + deploy-syft: - needs: - [ - call-pr-tests-linting, - call-pr-tests-syft, - call-pr-tests-stack, - build-and-push-docker-images, - ] - if: always() && (needs.call-pr-tests-linting.result == 'success' && needs.call-pr-tests-syft.result == 'success' && needs.call-pr-tests-stack.result == 'success' || github.event.inputs.skip_tests == 'true') && needs.build-and-push-docker-images.result == 'success' - # runs-on: [self-hosted, Linux] + needs: [merge-docker-images] + if: always() && needs.merge-docker-images.result == 'success' + runs-on: ubuntu-latest permissions: contents: write # For tag and release notes. @@ -305,11 +351,6 @@ jobs: twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/* fi - - name: Set Grid package version - id: grid-version - shell: bash - run: echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT - # Checkout Infra repo (nested) - name: Checkout Infra Repo uses: actions/checkout@v3