Merge branch 'main' into TIMO/readme-updates

.github/actions/configure-aws-credentials/action.yml

@@ -1,19 +1,59 @@
 name: 'Configure AWS Credentials'
-description: 'Configure AWS Credentials for a given application and |
-  environment so that the GitHub Actions workflow can access AWS resources. |
+description: 'Configure AWS Credentials for an AWS account so that |
+  the GitHub Actions workflow can access AWS resources. |
   This is a wrapper around https://github.com/aws-actions/configure-aws-credentials |
-  that first determines the account, role, and region based on the |
-  account_names_by_environment configuration in app-config'
+  that first determines the account, role, and region. |
+  Chose one of the following three authentication options: |
+  1. Authenticate by account_name |
+  2. Authenticate by network_name |
+  3. Authenticate by app_name and environment.'
+
 inputs:
+  account_name:
+    description: 'Name of account, must match <ACCOUNT_NAME> in <ACCOUNT_NAME>.<ACCOUNT_ID>.s3.tfbackend file in /infra/accounts'
+  network_name:
+    description: 'Name of network, must match <NETWORK_NAME> in <NETWORK_NAME>.s3.tfbackend file in /infra/networks'
   app_name:
     description: 'Name of application folder under /infra'
-    required: true
   environment:
     description: 'Name of environment (dev, staging, prod) that AWS resources live in, or "shared" for resources that are shared across environments'
-    required: true
 runs:
   using: "composite"
   steps:
+    - name: Get network name from app and environment
+      id: get-network-name
+      if: ${{ inputs.app_name && inputs.environment }}
+      run: |
+        echo "Get network name for app_name=${{ inputs.app_name }} and environment=${{ inputs.environment }}"
+
+        terraform -chdir="infra/${{ inputs.app_name }}/app-config" init > /dev/null
+        terraform -chdir="infra/${{ inputs.app_name }}/app-config" apply -auto-approve > /dev/null
+
+        if [[ "${{ inputs.environment }}" == "shared" ]]; then
+          network_name=$(terraform -chdir="infra/${{ inputs.app_name }}/app-config" output -raw shared_network_name)
+        else
+          network_name=$(terraform -chdir="infra/${{ inputs.app_name }}/app-config" output -json environment_configs | jq -r ".${{ inputs.environment }}.network_name")
+        fi
+
+        echo "Network name retrieved: ${network_name}"
+        echo "network_name=${network_name}" >> "$GITHUB_OUTPUT"
+      shell: bash
+
+    - name: Get account name from network
+      id: get-account-name
+      if: ${{ inputs.network_name || steps.get-network-name.outputs.network_name }}
+      run: |
+        network_name="${{ inputs.network_name || steps.get-network-name.outputs.network_name }}"
+        echo "Get account name for network: ${network_name}"
+
+        terraform -chdir="infra/project-config" init > /dev/null
+        terraform -chdir="infra/project-config" apply -auto-approve > /dev/null
+        account_name=$(terraform -chdir="infra/project-config" output -json network_configs | jq -r ".[\"${network_name}\"].account_name")
+
+        echo "Account name retrieved: ${account_name}"
+        echo "account_name=${account_name}" >> "$GITHUB_OUTPUT"
+      shell: bash
+
     - name: Get AWS account authentication details (AWS account, IAM role, AWS region)
       run: |
         # Get AWS account authentication details (AWS account, IAM role, AWS region)
@@ -22,34 +62,31 @@ runs:
 
         echo "::group::AWS account authentication details"
 
-        terraform -chdir=infra/project-config init > /dev/null
-        terraform -chdir=infra/project-config apply -auto-approve > /dev/null
-        AWS_REGION=$(terraform -chdir=infra/project-config output -raw default_region)
-        echo "AWS_REGION=$AWS_REGION"
-        GITHUB_ACTIONS_ROLE_NAME=$(terraform -chdir=infra/project-config output -raw github_actions_role_name)
-        echo "GITHUB_ACTIONS_ROLE_NAME=$GITHUB_ACTIONS_ROLE_NAME"
+        account_name="${{ inputs.account_name || steps.get-account-name.outputs.account_name }}"
 
-        terraform -chdir=infra/${{ inputs.app_name }}/app-config init > /dev/null
-        terraform -chdir=infra/${{ inputs.app_name }}/app-config apply -auto-approve > /dev/null
-        ACCOUNT_NAME=$(terraform -chdir=infra/${{ inputs.app_name }}/app-config output -json account_names_by_environment | jq -r .${{ inputs.environment }})
-        echo "ACCOUNT_NAME=$ACCOUNT_NAME"
+        terraform -chdir="infra/project-config" init > /dev/null
+        terraform -chdir="infra/project-config" apply -auto-approve > /dev/null
+        aws_region=$(terraform -chdir="infra/project-config" output -raw default_region)
+        echo "aws_region=${aws_region}"
+        github_actions_role_name=$(terraform -chdir="infra/project-config" output -raw github_actions_role_name)
+        echo "github_actions_role_name=${github_actions_role_name}"
 
         # Get the account id associated with the account name extracting the
         # ACCOUNT_ID part of the tfbackend file name which looks like
         # <ACCOUNT_NAME>.<ACCOUNT_ID>.s3.tfbackend.
-        # The cut command splits the string with period as the delimeter and
+        # The cut command splits the string with period as the delimiter and
         # extracts the second field.
-        ACCOUNT_ID=$(ls infra/accounts/$ACCOUNT_NAME.*.s3.tfbackend | cut -d. -f2)
-        echo "ACCOUNT_ID=$ACCOUNT_ID"
+        account_id=$(ls infra/accounts/${account_name}.*.s3.tfbackend | cut -d. -f2)
+        echo "account_id=${account_id}"
 
-        AWS_ROLE_TO_ASSUME=arn:aws:iam::$ACCOUNT_ID:role/$GITHUB_ACTIONS_ROLE_NAME
-        echo "AWS_ROLE_TO_ASSUME=$AWS_ROLE_TO_ASSUME"
+        aws_role_to_assume="arn:aws:iam::${account_id}:role/${github_actions_role_name}"
+        echo "aws_role_to_assume=${aws_role_to_assume}"
 
         echo "::endgroup::"
 
         echo "Setting env vars AWS_ROLE_TO_ASSUME and AWS_REGION..."
-        echo "AWS_ROLE_TO_ASSUME=$AWS_ROLE_TO_ASSUME" >> "$GITHUB_ENV"
-        echo "AWS_REGION=$AWS_REGION" >> "$GITHUB_ENV"
+        echo "AWS_ROLE_TO_ASSUME=${aws_role_to_assume}" >> "$GITHUB_ENV"
+        echo "AWS_REGION=${aws_region}" >> "$GITHUB_ENV"
       shell: bash
     - name: Configure AWS credentials
       uses: aws-actions/configure-aws-credentials@v3

.github/workflows/README.md

@@ -11,6 +11,10 @@ Each app should have:
 - `ci-[app_name]`: must be created; should run linting and testing
 - `ci-[app_name]-vulnerability-scans`: calls `vulnerability-scans`
   - Based on [ci-app-vulnerability-scans](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-vulnerability-scans.yml)
+- `ci-[app_name]-pr-environment-checks.yml`: calls `pr-environment-checks.yml` to create or update a pull request environment (see [pull request environments](/docs/infra/pull-request-environments.md))
+  - Based on [ci-app-pr-environment-checks.yml](/.github/workflows/ci-app-pr-environment-checks.yml)
+- `ci-[app_name]-pr-environment-destroy.yml`: calls `pr-environment-destroy.yml` to destroy the pull request environment (see [pull request environments](/docs/infra/pull-request-environments.md))
+  - Based on [ci-app-pr-environment-destroy.yml](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-pr-environment-destroy.yml)
 
 ### App-agnostic workflows
 
@@ -43,5 +47,4 @@ graph TD
 
 ## ⛑️ Helper workflows
 
-- [`check-infra-auth`](./check-infra-auth.yml): verifes that the project's Github repo is able to connect to AWS
-
+- [`check-ci-cd-auth`](./check-ci-cd-auth.yml): verifes that the project's Github repo is able to connect to AWS

.github/workflows/build-and-publish.yml

@@ -24,10 +24,26 @@ on:
         type: string
 
 jobs:
+  get-commit-hash:
+    name: Get commit hash
+    runs-on: ubuntu-latest
+    outputs:
+      commit_hash: ${{ steps.get-commit-hash.outputs.commit_hash }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+      - name: Get commit hash
+        id: get-commit-hash
+        run: |
+          COMMIT_HASH=$(git rev-parse ${{ inputs.ref }})
+          echo "Commit hash: $COMMIT_HASH"
+          echo "commit_hash=$COMMIT_HASH" >> "$GITHUB_OUTPUT"
   build-and-publish:
     name: Build and publish
     runs-on: ubuntu-latest
-    concurrency: ${{ github.workflow }}-${{ github.sha }}
+    needs: get-commit-hash
+    concurrency: ${{ github.workflow }}-${{ needs.get-commit-hash.outputs.commit_hash }}
 
     permissions:
       contents: read
@@ -38,14 +54,23 @@ jobs:
         with:
           ref: ${{ inputs.ref }}
 
-      - name: Build release
-        run: make APP_NAME=${{ inputs.app_name }} release-build
-
       - name: Configure AWS credentials
         uses: ./.github/actions/configure-aws-credentials
         with:
           app_name: ${{ inputs.app_name }}
           environment: shared
 
+      - name: Check if image is already published
+        id: check-image-published
+        run: |
+          is_image_published=$(./bin/is-image-published "${{ inputs.app_name }}" "${{ inputs.ref }}")
+          echo "Is image published: $is_image_published"
+          echo "is_image_published=$is_image_published" >> "$GITHUB_OUTPUT"
+
+      - name: Build release
+        if: steps.check-image-published.outputs.is_image_published == 'false'
+        run: make APP_NAME=${{ inputs.app_name }} release-build
+
       - name: Publish release
+        if: steps.check-image-published.outputs.is_image_published == 'false'
         run: make APP_NAME=${{ inputs.app_name }} release-publish

.github/workflows/check-infra-auth.yml → .github/workflows/check-ci-cd-auth.yml

@@ -1,4 +1,4 @@
-name: Check GitHub Actions AWS Authentication
+name: Check CI/CD AWS authentication
 
 on:
   workflow_dispatch:

.github/workflows/check-infra-deploy-status.yml

@@ -0,0 +1,72 @@
+# This workflow checks the status of infrastructure deployments to see whether
+# infrastructure code configuration matches the actual state of the infrastructure.
+# It does this by checking that Terraform plans show an empty diff (no changes)
+# across all root modules and backend configurations.
+name: Check infra deploy status
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Run every day at 07:00 UTC (3am ET, 12am PT) after engineers are likely done with work
+    - cron: "0 7 * * *"
+
+jobs:
+  collect-configs:
+    name: Collect configs
+    runs-on: ubuntu-latest
+    outputs:
+      root_module_configs: ${{ steps.collect-infra-deploy-status-check-configs.outputs.root_module_configs }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Collect root module configurations
+        id: collect-infra-deploy-status-check-configs
+        run: |
+          root_module_configs="$(./bin/infra-deploy-status-check-configs)"
+          echo "${root_module_configs}"
+          echo "root_module_configs=${root_module_configs}" >> "$GITHUB_OUTPUT"
+  check:
+    name: ${{ matrix.root_module_subdir }} ${{ matrix.backend_config_name }}
+    runs-on: ubuntu-latest
+    needs: collect-configs
+
+    # Skip this job if there are no root module configurations to check,
+    # otherwise the GitHub actions will give the error: "Matrix must define at least one vector"
+    if: ${{ needs.collect-configs.outputs.root_module_configs != '[]' }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.collect-configs.outputs.root_module_configs) }}
+
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: hashicorp/setup-terraform@v2
+        with:
+          terraform_version: 1.8.3
+          terraform_wrapper: false
+
+      - name: Configure AWS credentials
+        uses: ./.github/actions/configure-aws-credentials
+        with:
+          account_name: ${{ matrix.infra_layer == 'accounts' && matrix.account_name || null }}
+          network_name: ${{ matrix.infra_layer == 'networks' && matrix.backend_config_name || null }}
+          app_name: ${{ contains(fromJSON('["build-repository", "database", "service"]'), matrix.infra_layer) && matrix.app_name || null }}
+          environment: ${{ contains(fromJSON('["build-repository", "database", "service"]'), matrix.infra_layer) && matrix.backend_config_name || null }}
+
+      - name: Check Terraform plan
+        run: |
+          echo "::group::Initialize Terraform"
+          echo terraform -chdir="infra/${{ matrix.root_module_subdir }}" init -input=false -reconfigure -backend-config="${{ matrix.backend_config_name }}.s3.tfbackend"
+          terraform -chdir="infra/${{ matrix.root_module_subdir }}" init -input=false -reconfigure -backend-config="${{ matrix.backend_config_name }}.s3.tfbackend"
+          echo "::endgroup::"
+
+          echo "::group::Check Terraform plan"
+          echo terraform -chdir="infra/${{ matrix.root_module_subdir }}" plan -input=false -detailed-exitcode ${{ matrix.extra_params }}
+          terraform -chdir="infra/${{ matrix.root_module_subdir }}" plan -input=false -detailed-exitcode ${{ matrix.extra_params }}
+          echo "::endgroup::"
+        env:
+          TF_IN_AUTOMATION: "true"

.github/workflows/ci-app-pr-environment-checks.yml

@@ -0,0 +1,21 @@
+name: CI App PR Environment Checks
+on:
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        required: true
+        type: string
+      commit_hash:
+        required: true
+        type: string
+  # !! Uncomment the following lines once you've set up the dev environment and are ready to enable PR environments
+  # pull_request:
+jobs:
+  update:
+    name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise
+    uses: ./.github/workflows/pr-environment-checks.yml
+    with:
+      app_name: "app"
+      environment: "dev"
+      pr_number: ${{ inputs.pr_number || github.event.number }}
+      commit_hash: ${{ inputs.commit_hash || github.event.pull_request.head.sha }}

.github/workflows/ci-app-pr-environment-destroy.yml

@@ -0,0 +1,18 @@
+name: CI App PR Environment Destroy
+on:
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        required: true
+        type: string
+  # !! Uncomment the following lines once you've set up the dev environment and are ready to enable PR environments
+  # pull_request:
+  #   types: [closed]
+jobs:
+  destroy:
+    name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise
+    uses: ./.github/workflows/pr-environment-destroy.yml
+    with:
+      app_name: "app"
+      environment: "dev"
+      pr_number: ${{ inputs.pr_number || github.event.number }}

.github/workflows/ci-docs.yml

@@ -14,9 +14,9 @@ jobs:
     name: Lint markdown
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       # This is the GitHub Actions-friendly port of the linter used in the Makefile.
       - uses: gaurav-nelson/[email protected]
         with:
-          use-quiet-mode: 'yes'  # errors only.
-          config-file: '.github/workflows/markdownlint-config.json'
+          use-quiet-mode: "yes" # errors only.
+          config-file: ".github/workflows/markdownlint-config.json"

.github/workflows/ci-infra-service.yml

@@ -28,7 +28,7 @@ jobs:
       id-token: write
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: hashicorp/setup-terraform@v2
         with:

.github/workflows/ci-infra.yml

@@ -21,7 +21,7 @@ jobs:
     name: Lint GitHub Actions workflows
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Download actionlint
         id: get_actionlint
         run: bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash)
@@ -33,14 +33,14 @@ jobs:
     name: Lint scripts
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Shellcheck
         run: make infra-lint-scripts
   check-terraform-format:
     name: Check Terraform format
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: hashicorp/setup-terraform@v2
         with:
           terraform_version: 1.8.3
@@ -53,7 +53,7 @@ jobs:
     name: Validate Terraform modules
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: hashicorp/setup-terraform@v2
         with:
           terraform_version: 1.8.3
@@ -64,7 +64,7 @@ jobs:
     name: Check compliance with checkov
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
@@ -88,7 +88,7 @@ jobs:
       pull-requests: write
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Run tfsec check
         uses: aquasecurity/[email protected]
         with:

.github/workflows/database-migrations.yml

@@ -31,7 +31,7 @@ jobs:
       id-token: write
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Terraform
         uses: ./.github/actions/setup-terraform