aws: fix TOOMANYREQUESTS error, log into ecr public

Without this patch kaniko is failing with the following TOOMANYREQUESTS error: INFO[0000] Retrieving image public.ecr.aws/docker/library/golang:1.21-bullseye from registry public.ecr.aws error building image: unable to complete operation after 0 attempts, last error: GET https://public.ecr.aws/v2/docker/library/golang/manifests/sha256:301b0f36ff74f5b3b0fcae9a158b6338fd6b6d1ed8231b0fff6460a065cebeb3: TOOMANYREQUESTS: Rate exceeded This change refactors the jobs. A new ecr job executes every hour and sores the docker login credentials into a repository secret. These credentials are then read by each job without needing to log into ECR because the credentials stored in the secret should always be valid and up to date.
holos-run · Sep 3, 2024 · 52bdca1 · 52bdca1
1 parent ff23e83
commit 52bdca1
Show file tree

Hide file tree

Showing 4 changed files with 95 additions and 111 deletions.
diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml
@@ -3,60 +3,40 @@ name: Debian
 on:
   workflow_dispatch: {}
   schedule:
-    - cron: "30 1 * * *" # 1:30AM UTC, 6:30PM PST
+    - cron: "40 1 * * *" # 1 AM UTC, 6 PM PST
 
 jobs:
   git:
-    runs-on: [ubuntu-22.04]
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-  aws:
-    runs-on: [ubuntu-22.04]
-    permissions:
-      id-token: write # Necessary to get aws creds via oidc token exchange
-      contents: read
-    steps:
-      - name: AWS Credentials
-        id: login-aws
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          # Defined at https://github.com/holos-run/holos-infra/blob/main/terraform/projects/nonprod-holos/shared_services/aws/github_oidc/main.tf#L90-L106
-          role-to-assume: arn:aws:iam::271053619184:role/gha-app-role
-          aws-region: us-east-2
-          output-credentials: true
-      - name: AWS ECR Credentials
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
-      - name: Docker Login
-        id: docker-login
-        run: |
-          echo -n ${{ steps.login-ecr.outputs.docker_password_271053619184_dkr_ecr_us_east_2_amazonaws_com }} | docker login --password-stdin --username ${{ steps.login-ecr.outputs.docker_username_271053619184_dkr_ecr_us_east_2_amazonaws_com }} ${{ steps.login-ecr.outputs.registry }}
-          echo "docker-config=$(cat ~/.docker/config.json | base64 -w 0)" >> $GITHUB_OUTPUT
+      - name: SHA
+        id: sha
+        run: echo "sha=$(/usr/bin/git log -1 --format='%H')" >> $GITHUB_OUTPUT
     outputs:
-      registry: ${{ steps.login-ecr.outputs.registry }}
-      docker-config: ${{ steps.docker-login.outputs.docker-config }}
+      sha: ${{ steps.sha.outputs.sha }}
   kaniko:
-    needs: [git, aws]
-    runs-on: [ubuntu-22.04]
+    runs-on: ubuntu-latest
+    needs: [git]
     container:
       image: gcr.io/kaniko-project/executor:v1.23.2-debug
     permissions:
       contents: read # read the repository
     steps:
-      - name: Build and push container image
+      - name: Build and Push Container
         run: |
-          # Kaniko
-          echo -n ${{ needs.aws.outputs.docker-config }} | base64 -d > /kaniko/.docker/config.json
+          # Docker config provided by ecr.yaml workflow.
+          echo -n ${{ secrets.DOCKER_CONFIG_BASE64 }} | base64 -d > /kaniko/.docker/config.json
 
           # Configure git credentials to access github private repositories.
-          export GIT_USERNAME='holos-server-go'
+          export GIT_USERNAME='GITHUB_TOKEN'
           export GIT_PASSWORD='${{ secrets.GITHUB_TOKEN }}'
 
           # Build and push
           /kaniko/executor --dockerfile=debian/Dockerfile \
             --context='${{ github.repositoryUrl }}#${{ needs.git.outputs.sha }}' \
-            --destination=${{ needs.aws.outputs.registry }}/holos-run/container-images/debian:bullseye \
+            --destination=${{ vars.REGISTRY }}/holos-run/container-images/debian:bullseye \
             --push-retry 5 \
             --image-name-with-digest-file /workspace/image-digest.txt
 

diff --git a/.github/workflows/ecr.yaml b/.github/workflows/ecr.yaml
@@ -0,0 +1,56 @@
+# Periodically refresh AWS ECR creds
+name: ECR
+
+on:
+  workflow_dispatch: {}
+  schedule:
+    - cron: "10 * * * *" # Hourly
+
+jobs:
+  ecr:
+    runs-on: [ubuntu-22.04]
+    permissions:
+      id-token: write # Necessary to get aws creds via oidc token exchange
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: AWS us-east-1 credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          # Defined at https://github.com/holos-run/holos-infra/blob/main/terraform/projects/nonprod-holos/shared_services/aws/github_oidc/main.tf#L90-L106
+          role-to-assume: arn:aws:iam::271053619184:role/gha-app-role
+          aws-region: us-east-1
+          output-credentials: true
+      - name: Login to Amazon ECR Public
+        id: login-ecr-public
+        uses: aws-actions/amazon-ecr-login@v2
+        with:
+          registry-type: public
+      - name: AWS us-east2 credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          # Defined at https://github.com/holos-run/holos-infra/blob/main/terraform/projects/nonprod-holos/shared_services/aws/github_oidc/main.tf#L90-L106
+          role-to-assume: arn:aws:iam::271053619184:role/gha-app-role
+          aws-region: us-east-2
+          output-credentials: true
+      - name: Login to Amazon ECR Private
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+      - name: Get Secret Writer Token # GITHUB_TOKEN does not have sufficient permission.
+        # Refer to https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/making-authenticated-api-requests-with-a-github-app-in-a-github-actions-workflow
+        id: generate-token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ vars.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+      - name: Docker Login
+        id: docker-login
+        env:
+          GH_TOKEN: ${{ steps.generate-token.outputs.token }}
+        run: |
+          echo -n ${{ steps.login-ecr.outputs.docker_password_271053619184_dkr_ecr_us_east_2_amazonaws_com }} | docker login --password-stdin --username ${{ steps.login-ecr.outputs.docker_username_271053619184_dkr_ecr_us_east_2_amazonaws_com }} ${{ steps.login-ecr.outputs.registry }}
+          echo -n ${{ steps.login-ecr-public.outputs.docker_password_public_ecr_aws }} | docker login --password-stdin --username ${{ steps.login-ecr-public.outputs.docker_username_public_ecr_aws }} ${{ steps.login-ecr-public.outputs.registry }}
+
+          base64 -w 0 ~/.docker/config.json | gh secret set DOCKER_CONFIG_BASE64
+          echo -n ${{ steps.login-ecr.outputs.registry }} | gh variable set REGISTRY
diff --git a/.github/workflows/golang.yaml b/.github/workflows/golang.yaml
@@ -3,60 +3,40 @@ name: Golang
 on:
   workflow_dispatch: {}
   schedule:
-    - cron: "30 1 * * *" # 1:30AM UTC, 6:30PM PST
+    - cron: "30 1 * * *" # 1 AM UTC, 6 PM PST
 
 jobs:
   git:
-    runs-on: [ubuntu-22.04]
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-  aws:
-    runs-on: [ubuntu-22.04]
-    permissions:
-      id-token: write # Necessary to get aws creds via oidc token exchange
-      contents: read
-    steps:
-      - name: AWS Credentials
-        id: login-aws
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          # Defined at https://github.com/holos-run/holos-infra/blob/main/terraform/projects/nonprod-holos/shared_services/aws/github_oidc/main.tf#L90-L106
-          role-to-assume: arn:aws:iam::271053619184:role/gha-app-role
-          aws-region: us-east-2
-          output-credentials: true
-      - name: AWS ECR Credentials
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
-      - name: Docker Login
-        id: docker-login
-        run: |
-          echo -n ${{ steps.login-ecr.outputs.docker_password_271053619184_dkr_ecr_us_east_2_amazonaws_com }} | docker login --password-stdin --username ${{ steps.login-ecr.outputs.docker_username_271053619184_dkr_ecr_us_east_2_amazonaws_com }} ${{ steps.login-ecr.outputs.registry }}
-          echo "docker-config=$(cat ~/.docker/config.json | base64 -w 0)" >> $GITHUB_OUTPUT
+      - name: SHA
+        id: sha
+        run: echo "sha=$(/usr/bin/git log -1 --format='%H')" >> $GITHUB_OUTPUT
     outputs:
-      registry: ${{ steps.login-ecr.outputs.registry }}
-      docker-config: ${{ steps.docker-login.outputs.docker-config }}
+      sha: ${{ steps.sha.outputs.sha }}
   kaniko:
-    needs: [git, aws]
-    runs-on: [ubuntu-22.04]
+    runs-on: ubuntu-latest
+    needs: [git]
     container:
       image: gcr.io/kaniko-project/executor:v1.23.2-debug
     permissions:
       contents: read # read the repository
     steps:
-      - name: Build and push container image
+      - name: Build and Push Container
         run: |
-          # Kaniko
-          echo -n ${{ needs.aws.outputs.docker-config }} | base64 -d > /kaniko/.docker/config.json
+          # Docker config provided by ecr.yaml workflow.
+          echo -n ${{ secrets.DOCKER_CONFIG_BASE64 }} | base64 -d > /kaniko/.docker/config.json
 
           # Configure git credentials to access github private repositories.
-          export GIT_USERNAME='holos-server-go'
+          export GIT_USERNAME='GITHUB_TOKEN'
           export GIT_PASSWORD='${{ secrets.GITHUB_TOKEN }}'
 
           # Build and push
           /kaniko/executor --dockerfile=golang/Dockerfile \
             --context='${{ github.repositoryUrl }}#${{ needs.git.outputs.sha }}' \
-            --destination=${{ needs.aws.outputs.registry }}/holos-run/container-images/golang:1.21-bullseye \
+            --destination=${{ vars.REGISTRY }}/holos-run/container-images/golang:1.21-bullseye \
             --push-retry 5 \
             --image-name-with-digest-file /workspace/image-digest.txt
 

diff --git a/.github/workflows/toolkit.yaml b/.github/workflows/toolkit.yaml
@@ -3,72 +3,40 @@ name: Toolkit
 on:
   workflow_dispatch: {}
   schedule:
-    - cron: "30 2 * * *" # 2:30AM UTC, 7:30PM PST
+    - cron: "50 1 * * *" # 1 AM UTC, 6 PM PST
 
 jobs:
   git:
-    runs-on: [ubuntu-22.04]
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-  aws:
-    runs-on: [ubuntu-22.04]
-    permissions:
-      id-token: write # Necessary to get aws creds via oidc token exchange
-      contents: read
-    steps:
-      - name: AWS us-east-1 credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          # Defined at https://github.com/holos-run/holos-infra/blob/main/terraform/projects/nonprod-holos/shared_services/aws/github_oidc/main.tf#L90-L106
-          role-to-assume: arn:aws:iam::271053619184:role/gha-app-role
-          aws-region: us-east-1
-          output-credentials: true
-      - name: Login to Amazon ECR Public
-        id: login-ecr-public
-        uses: aws-actions/amazon-ecr-login@v2
-        with:
-          registry-type: public
-      - name: AWS us-east2 credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          # Defined at https://github.com/holos-run/holos-infra/blob/main/terraform/projects/nonprod-holos/shared_services/aws/github_oidc/main.tf#L90-L106
-          role-to-assume: arn:aws:iam::271053619184:role/gha-app-role
-          aws-region: us-east-2
-          output-credentials: true
-      - name: Login to Amazon ECR Private
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
-      - name: Docker Login
-        id: docker-login
-        run: |
-          echo -n ${{ steps.login-ecr.outputs.docker_password_271053619184_dkr_ecr_us_east_2_amazonaws_com }} | docker login --password-stdin --username ${{ steps.login-ecr.outputs.docker_username_271053619184_dkr_ecr_us_east_2_amazonaws_com }} ${{ steps.login-ecr.outputs.registry }}
-          echo -n ${{ steps.login-ecr-public.outputs.docker_password_public_ecr_aws }} | docker login --password-stdin --username ${{ steps.login-ecr-public.outputs.docker_username_public_ecr_aws }} ${{ steps.login-ecr-public.outputs.registry }}
-          echo "docker-config=$(cat ~/.docker/config.json | base64 -w 0)" >> $GITHUB_OUTPUT
+      - name: SHA
+        id: sha
+        run: echo "sha=$(/usr/bin/git log -1 --format='%H')" >> $GITHUB_OUTPUT
     outputs:
-      registry: ${{ steps.login-ecr.outputs.registry }}
-      docker-config: ${{ steps.docker-login.outputs.docker-config }}
+      sha: ${{ steps.sha.outputs.sha }}
   kaniko:
-    needs: [git, aws]
-    runs-on: [ubuntu-22.04]
+    runs-on: ubuntu-latest
+    needs: [git]
     container:
       image: gcr.io/kaniko-project/executor:v1.23.2-debug
     permissions:
       contents: read # read the repository
     steps:
-      - name: Build and push container image
+      - name: Build and Push Container
         run: |
-          # Kaniko
-          echo -n ${{ needs.aws.outputs.docker-config }} | base64 -d > /kaniko/.docker/config.json
+          # Docker config provided by ecr.yaml workflow.
+          echo -n ${{ secrets.DOCKER_CONFIG_BASE64 }} | base64 -d > /kaniko/.docker/config.json
 
           # Configure git credentials to access github private repositories.
-          export GIT_USERNAME='holos-server-go'
+          export GIT_USERNAME='GITHUB_TOKEN'
           export GIT_PASSWORD='${{ secrets.GITHUB_TOKEN }}'
 
           # Build and push
           /kaniko/executor --dockerfile=toolkit/Dockerfile \
             --context='${{ github.repositoryUrl }}#${{ needs.git.outputs.sha }}' \
-            --destination=${{ needs.aws.outputs.registry }}/holos-run/container-images/toolkit:latest \
+            --destination=${{ vars.REGISTRY }}/holos-run/container-images/toolkit:latest \
             --push-retry 5 \
             --image-name-with-digest-file /workspace/image-digest.txt