From 52bdca127a2e05b4c07aa76dacbb0f6f3bc50558 Mon Sep 17 00:00:00 2001 From: Jeff McCune Date: Tue, 3 Sep 2024 08:19:03 -0700 Subject: [PATCH] aws: fix TOOMANYREQUESTS error, log into ecr public Without this patch kaniko is failing with the following TOOMANYREQUESTS error: INFO[0000] Retrieving image public.ecr.aws/docker/library/golang:1.21-bullseye from registry public.ecr.aws error building image: unable to complete operation after 0 attempts, last error: GET https://public.ecr.aws/v2/docker/library/golang/manifests/sha256:301b0f36ff74f5b3b0fcae9a158b6338fd6b6d1ed8231b0fff6460a065cebeb3: TOOMANYREQUESTS: Rate exceeded This change refactors the jobs. A new ecr job executes every hour and sores the docker login credentials into a repository secret. These credentials are then read by each job without needing to log into ECR because the credentials stored in the secret should always be valid and up to date. --- .github/workflows/debian.yaml | 46 ++++++++------------------- .github/workflows/ecr.yaml | 56 ++++++++++++++++++++++++++++++++ .github/workflows/golang.yaml | 46 ++++++++------------------- .github/workflows/toolkit.yaml | 58 ++++++++-------------------------- 4 files changed, 95 insertions(+), 111 deletions(-) create mode 100644 .github/workflows/ecr.yaml diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 22f2c42..ab66865 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -3,60 +3,40 @@ name: Debian on: workflow_dispatch: {} schedule: - - cron: "30 1 * * *" # 1:30AM UTC, 6:30PM PST + - cron: "40 1 * * *" # 1 AM UTC, 6 PM PST jobs: git: - runs-on: [ubuntu-22.04] + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - aws: - runs-on: [ubuntu-22.04] - permissions: - id-token: write # Necessary to get aws creds via oidc token exchange - contents: read - steps: - - name: AWS Credentials - id: login-aws - uses: aws-actions/configure-aws-credentials@v4 - with: - # Defined at https://github.com/holos-run/holos-infra/blob/main/terraform/projects/nonprod-holos/shared_services/aws/github_oidc/main.tf#L90-L106 - role-to-assume: arn:aws:iam::271053619184:role/gha-app-role - aws-region: us-east-2 - output-credentials: true - - name: AWS ECR Credentials - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - name: Docker Login - id: docker-login - run: | - echo -n ${{ steps.login-ecr.outputs.docker_password_271053619184_dkr_ecr_us_east_2_amazonaws_com }} | docker login --password-stdin --username ${{ steps.login-ecr.outputs.docker_username_271053619184_dkr_ecr_us_east_2_amazonaws_com }} ${{ steps.login-ecr.outputs.registry }} - echo "docker-config=$(cat ~/.docker/config.json | base64 -w 0)" >> $GITHUB_OUTPUT + - name: SHA + id: sha + run: echo "sha=$(/usr/bin/git log -1 --format='%H')" >> $GITHUB_OUTPUT outputs: - registry: ${{ steps.login-ecr.outputs.registry }} - docker-config: ${{ steps.docker-login.outputs.docker-config }} + sha: ${{ steps.sha.outputs.sha }} kaniko: - needs: [git, aws] - runs-on: [ubuntu-22.04] + runs-on: ubuntu-latest + needs: [git] container: image: gcr.io/kaniko-project/executor:v1.23.2-debug permissions: contents: read # read the repository steps: - - name: Build and push container image + - name: Build and Push Container run: | - # Kaniko - echo -n ${{ needs.aws.outputs.docker-config }} | base64 -d > /kaniko/.docker/config.json + # Docker config provided by ecr.yaml workflow. + echo -n ${{ secrets.DOCKER_CONFIG_BASE64 }} | base64 -d > /kaniko/.docker/config.json # Configure git credentials to access github private repositories. - export GIT_USERNAME='holos-server-go' + export GIT_USERNAME='GITHUB_TOKEN' export GIT_PASSWORD='${{ secrets.GITHUB_TOKEN }}' # Build and push /kaniko/executor --dockerfile=debian/Dockerfile \ --context='${{ github.repositoryUrl }}#${{ needs.git.outputs.sha }}' \ - --destination=${{ needs.aws.outputs.registry }}/holos-run/container-images/debian:bullseye \ + --destination=${{ vars.REGISTRY }}/holos-run/container-images/debian:bullseye \ --push-retry 5 \ --image-name-with-digest-file /workspace/image-digest.txt diff --git a/.github/workflows/ecr.yaml b/.github/workflows/ecr.yaml new file mode 100644 index 0000000..2340e4d --- /dev/null +++ b/.github/workflows/ecr.yaml @@ -0,0 +1,56 @@ +# Periodically refresh AWS ECR creds +name: ECR + +on: + workflow_dispatch: {} + schedule: + - cron: "10 * * * *" # Hourly + +jobs: + ecr: + runs-on: [ubuntu-22.04] + permissions: + id-token: write # Necessary to get aws creds via oidc token exchange + contents: read + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: AWS us-east-1 credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + # Defined at https://github.com/holos-run/holos-infra/blob/main/terraform/projects/nonprod-holos/shared_services/aws/github_oidc/main.tf#L90-L106 + role-to-assume: arn:aws:iam::271053619184:role/gha-app-role + aws-region: us-east-1 + output-credentials: true + - name: Login to Amazon ECR Public + id: login-ecr-public + uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: public + - name: AWS us-east2 credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + # Defined at https://github.com/holos-run/holos-infra/blob/main/terraform/projects/nonprod-holos/shared_services/aws/github_oidc/main.tf#L90-L106 + role-to-assume: arn:aws:iam::271053619184:role/gha-app-role + aws-region: us-east-2 + output-credentials: true + - name: Login to Amazon ECR Private + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + - name: Get Secret Writer Token # GITHUB_TOKEN does not have sufficient permission. + # Refer to https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/making-authenticated-api-requests-with-a-github-app-in-a-github-actions-workflow + id: generate-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ vars.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + - name: Docker Login + id: docker-login + env: + GH_TOKEN: ${{ steps.generate-token.outputs.token }} + run: | + echo -n ${{ steps.login-ecr.outputs.docker_password_271053619184_dkr_ecr_us_east_2_amazonaws_com }} | docker login --password-stdin --username ${{ steps.login-ecr.outputs.docker_username_271053619184_dkr_ecr_us_east_2_amazonaws_com }} ${{ steps.login-ecr.outputs.registry }} + echo -n ${{ steps.login-ecr-public.outputs.docker_password_public_ecr_aws }} | docker login --password-stdin --username ${{ steps.login-ecr-public.outputs.docker_username_public_ecr_aws }} ${{ steps.login-ecr-public.outputs.registry }} + + base64 -w 0 ~/.docker/config.json | gh secret set DOCKER_CONFIG_BASE64 + echo -n ${{ steps.login-ecr.outputs.registry }} | gh variable set REGISTRY diff --git a/.github/workflows/golang.yaml b/.github/workflows/golang.yaml index dc61969..6c259f6 100644 --- a/.github/workflows/golang.yaml +++ b/.github/workflows/golang.yaml @@ -3,60 +3,40 @@ name: Golang on: workflow_dispatch: {} schedule: - - cron: "30 1 * * *" # 1:30AM UTC, 6:30PM PST + - cron: "30 1 * * *" # 1 AM UTC, 6 PM PST jobs: git: - runs-on: [ubuntu-22.04] + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - aws: - runs-on: [ubuntu-22.04] - permissions: - id-token: write # Necessary to get aws creds via oidc token exchange - contents: read - steps: - - name: AWS Credentials - id: login-aws - uses: aws-actions/configure-aws-credentials@v4 - with: - # Defined at https://github.com/holos-run/holos-infra/blob/main/terraform/projects/nonprod-holos/shared_services/aws/github_oidc/main.tf#L90-L106 - role-to-assume: arn:aws:iam::271053619184:role/gha-app-role - aws-region: us-east-2 - output-credentials: true - - name: AWS ECR Credentials - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - name: Docker Login - id: docker-login - run: | - echo -n ${{ steps.login-ecr.outputs.docker_password_271053619184_dkr_ecr_us_east_2_amazonaws_com }} | docker login --password-stdin --username ${{ steps.login-ecr.outputs.docker_username_271053619184_dkr_ecr_us_east_2_amazonaws_com }} ${{ steps.login-ecr.outputs.registry }} - echo "docker-config=$(cat ~/.docker/config.json | base64 -w 0)" >> $GITHUB_OUTPUT + - name: SHA + id: sha + run: echo "sha=$(/usr/bin/git log -1 --format='%H')" >> $GITHUB_OUTPUT outputs: - registry: ${{ steps.login-ecr.outputs.registry }} - docker-config: ${{ steps.docker-login.outputs.docker-config }} + sha: ${{ steps.sha.outputs.sha }} kaniko: - needs: [git, aws] - runs-on: [ubuntu-22.04] + runs-on: ubuntu-latest + needs: [git] container: image: gcr.io/kaniko-project/executor:v1.23.2-debug permissions: contents: read # read the repository steps: - - name: Build and push container image + - name: Build and Push Container run: | - # Kaniko - echo -n ${{ needs.aws.outputs.docker-config }} | base64 -d > /kaniko/.docker/config.json + # Docker config provided by ecr.yaml workflow. + echo -n ${{ secrets.DOCKER_CONFIG_BASE64 }} | base64 -d > /kaniko/.docker/config.json # Configure git credentials to access github private repositories. - export GIT_USERNAME='holos-server-go' + export GIT_USERNAME='GITHUB_TOKEN' export GIT_PASSWORD='${{ secrets.GITHUB_TOKEN }}' # Build and push /kaniko/executor --dockerfile=golang/Dockerfile \ --context='${{ github.repositoryUrl }}#${{ needs.git.outputs.sha }}' \ - --destination=${{ needs.aws.outputs.registry }}/holos-run/container-images/golang:1.21-bullseye \ + --destination=${{ vars.REGISTRY }}/holos-run/container-images/golang:1.21-bullseye \ --push-retry 5 \ --image-name-with-digest-file /workspace/image-digest.txt diff --git a/.github/workflows/toolkit.yaml b/.github/workflows/toolkit.yaml index b7b438c..5354646 100644 --- a/.github/workflows/toolkit.yaml +++ b/.github/workflows/toolkit.yaml @@ -3,72 +3,40 @@ name: Toolkit on: workflow_dispatch: {} schedule: - - cron: "30 2 * * *" # 2:30AM UTC, 7:30PM PST + - cron: "50 1 * * *" # 1 AM UTC, 6 PM PST jobs: git: - runs-on: [ubuntu-22.04] + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - aws: - runs-on: [ubuntu-22.04] - permissions: - id-token: write # Necessary to get aws creds via oidc token exchange - contents: read - steps: - - name: AWS us-east-1 credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - # Defined at https://github.com/holos-run/holos-infra/blob/main/terraform/projects/nonprod-holos/shared_services/aws/github_oidc/main.tf#L90-L106 - role-to-assume: arn:aws:iam::271053619184:role/gha-app-role - aws-region: us-east-1 - output-credentials: true - - name: Login to Amazon ECR Public - id: login-ecr-public - uses: aws-actions/amazon-ecr-login@v2 - with: - registry-type: public - - name: AWS us-east2 credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - # Defined at https://github.com/holos-run/holos-infra/blob/main/terraform/projects/nonprod-holos/shared_services/aws/github_oidc/main.tf#L90-L106 - role-to-assume: arn:aws:iam::271053619184:role/gha-app-role - aws-region: us-east-2 - output-credentials: true - - name: Login to Amazon ECR Private - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - name: Docker Login - id: docker-login - run: | - echo -n ${{ steps.login-ecr.outputs.docker_password_271053619184_dkr_ecr_us_east_2_amazonaws_com }} | docker login --password-stdin --username ${{ steps.login-ecr.outputs.docker_username_271053619184_dkr_ecr_us_east_2_amazonaws_com }} ${{ steps.login-ecr.outputs.registry }} - echo -n ${{ steps.login-ecr-public.outputs.docker_password_public_ecr_aws }} | docker login --password-stdin --username ${{ steps.login-ecr-public.outputs.docker_username_public_ecr_aws }} ${{ steps.login-ecr-public.outputs.registry }} - echo "docker-config=$(cat ~/.docker/config.json | base64 -w 0)" >> $GITHUB_OUTPUT + - name: SHA + id: sha + run: echo "sha=$(/usr/bin/git log -1 --format='%H')" >> $GITHUB_OUTPUT outputs: - registry: ${{ steps.login-ecr.outputs.registry }} - docker-config: ${{ steps.docker-login.outputs.docker-config }} + sha: ${{ steps.sha.outputs.sha }} kaniko: - needs: [git, aws] - runs-on: [ubuntu-22.04] + runs-on: ubuntu-latest + needs: [git] container: image: gcr.io/kaniko-project/executor:v1.23.2-debug permissions: contents: read # read the repository steps: - - name: Build and push container image + - name: Build and Push Container run: | - # Kaniko - echo -n ${{ needs.aws.outputs.docker-config }} | base64 -d > /kaniko/.docker/config.json + # Docker config provided by ecr.yaml workflow. + echo -n ${{ secrets.DOCKER_CONFIG_BASE64 }} | base64 -d > /kaniko/.docker/config.json # Configure git credentials to access github private repositories. - export GIT_USERNAME='holos-server-go' + export GIT_USERNAME='GITHUB_TOKEN' export GIT_PASSWORD='${{ secrets.GITHUB_TOKEN }}' # Build and push /kaniko/executor --dockerfile=toolkit/Dockerfile \ --context='${{ github.repositoryUrl }}#${{ needs.git.outputs.sha }}' \ - --destination=${{ needs.aws.outputs.registry }}/holos-run/container-images/toolkit:latest \ + --destination=${{ vars.REGISTRY }}/holos-run/container-images/toolkit:latest \ --push-retry 5 \ --image-name-with-digest-file /workspace/image-digest.txt