diff --git a/.github/actions/configure-aws-credentials/action.yml b/.github/actions/configure-aws-credentials/action.yml index 42f1e963..6b573ce9 100644 --- a/.github/actions/configure-aws-credentials/action.yml +++ b/.github/actions/configure-aws-credentials/action.yml @@ -1,19 +1,59 @@ name: 'Configure AWS Credentials' -description: 'Configure AWS Credentials for a given application and | - environment so that the GitHub Actions workflow can access AWS resources. | +description: 'Configure AWS Credentials for an AWS account so that | + the GitHub Actions workflow can access AWS resources. | This is a wrapper around https://github.com/aws-actions/configure-aws-credentials | - that first determines the account, role, and region based on the | - account_names_by_environment configuration in app-config' + that first determines the account, role, and region. | + Chose one of the following three authentication options: | + 1. Authenticate by account_name | + 2. Authenticate by network_name | + 3. Authenticate by app_name and environment.' + inputs: + account_name: + description: 'Name of account, must match in ..s3.tfbackend file in /infra/accounts' + network_name: + description: 'Name of network, must match in .s3.tfbackend file in /infra/networks' app_name: description: 'Name of application folder under /infra' - required: true environment: description: 'Name of environment (dev, staging, prod) that AWS resources live in, or "shared" for resources that are shared across environments' - required: true runs: using: "composite" steps: + - name: Get network name from app and environment + id: get-network-name + if: ${{ inputs.app_name && inputs.environment }} + run: | + echo "Get network name for app_name=${{ inputs.app_name }} and environment=${{ inputs.environment }}" + + terraform -chdir="infra/${{ inputs.app_name }}/app-config" init > /dev/null + terraform -chdir="infra/${{ inputs.app_name }}/app-config" apply -auto-approve > /dev/null + + if [[ "${{ inputs.environment }}" == "shared" ]]; then + network_name=$(terraform -chdir="infra/${{ inputs.app_name }}/app-config" output -raw shared_network_name) + else + network_name=$(terraform -chdir="infra/${{ inputs.app_name }}/app-config" output -json environment_configs | jq -r ".${{ inputs.environment }}.network_name") + fi + + echo "Network name retrieved: ${network_name}" + echo "network_name=${network_name}" >> "$GITHUB_OUTPUT" + shell: bash + + - name: Get account name from network + id: get-account-name + if: ${{ inputs.network_name || steps.get-network-name.outputs.network_name }} + run: | + network_name="${{ inputs.network_name || steps.get-network-name.outputs.network_name }}" + echo "Get account name for network: ${network_name}" + + terraform -chdir="infra/project-config" init > /dev/null + terraform -chdir="infra/project-config" apply -auto-approve > /dev/null + account_name=$(terraform -chdir="infra/project-config" output -json network_configs | jq -r ".[\"${network_name}\"].account_name") + + echo "Account name retrieved: ${account_name}" + echo "account_name=${account_name}" >> "$GITHUB_OUTPUT" + shell: bash + - name: Get AWS account authentication details (AWS account, IAM role, AWS region) run: | # Get AWS account authentication details (AWS account, IAM role, AWS region) @@ -22,34 +62,31 @@ runs: echo "::group::AWS account authentication details" - terraform -chdir=infra/project-config init > /dev/null - terraform -chdir=infra/project-config apply -auto-approve > /dev/null - AWS_REGION=$(terraform -chdir=infra/project-config output -raw default_region) - echo "AWS_REGION=$AWS_REGION" - GITHUB_ACTIONS_ROLE_NAME=$(terraform -chdir=infra/project-config output -raw github_actions_role_name) - echo "GITHUB_ACTIONS_ROLE_NAME=$GITHUB_ACTIONS_ROLE_NAME" + account_name="${{ inputs.account_name || steps.get-account-name.outputs.account_name }}" - terraform -chdir=infra/${{ inputs.app_name }}/app-config init > /dev/null - terraform -chdir=infra/${{ inputs.app_name }}/app-config apply -auto-approve > /dev/null - ACCOUNT_NAME=$(terraform -chdir=infra/${{ inputs.app_name }}/app-config output -json account_names_by_environment | jq -r .${{ inputs.environment }}) - echo "ACCOUNT_NAME=$ACCOUNT_NAME" + terraform -chdir="infra/project-config" init > /dev/null + terraform -chdir="infra/project-config" apply -auto-approve > /dev/null + aws_region=$(terraform -chdir="infra/project-config" output -raw default_region) + echo "aws_region=${aws_region}" + github_actions_role_name=$(terraform -chdir="infra/project-config" output -raw github_actions_role_name) + echo "github_actions_role_name=${github_actions_role_name}" # Get the account id associated with the account name extracting the # ACCOUNT_ID part of the tfbackend file name which looks like # ..s3.tfbackend. - # The cut command splits the string with period as the delimeter and + # The cut command splits the string with period as the delimiter and # extracts the second field. - ACCOUNT_ID=$(ls infra/accounts/$ACCOUNT_NAME.*.s3.tfbackend | cut -d. -f2) - echo "ACCOUNT_ID=$ACCOUNT_ID" + account_id=$(ls infra/accounts/${account_name}.*.s3.tfbackend | cut -d. -f2) + echo "account_id=${account_id}" - AWS_ROLE_TO_ASSUME=arn:aws:iam::$ACCOUNT_ID:role/$GITHUB_ACTIONS_ROLE_NAME - echo "AWS_ROLE_TO_ASSUME=$AWS_ROLE_TO_ASSUME" + aws_role_to_assume="arn:aws:iam::${account_id}:role/${github_actions_role_name}" + echo "aws_role_to_assume=${aws_role_to_assume}" echo "::endgroup::" echo "Setting env vars AWS_ROLE_TO_ASSUME and AWS_REGION..." - echo "AWS_ROLE_TO_ASSUME=$AWS_ROLE_TO_ASSUME" >> "$GITHUB_ENV" - echo "AWS_REGION=$AWS_REGION" >> "$GITHUB_ENV" + echo "AWS_ROLE_TO_ASSUME=${aws_role_to_assume}" >> "$GITHUB_ENV" + echo "AWS_REGION=${aws_region}" >> "$GITHUB_ENV" shell: bash - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v3 diff --git a/.github/workflows/README.md b/.github/workflows/README.md index d10928f7..db3692f6 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -11,6 +11,10 @@ Each app should have: - `ci-[app_name]`: must be created; should run linting and testing - `ci-[app_name]-vulnerability-scans`: calls `vulnerability-scans` - Based on [ci-app-vulnerability-scans](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-vulnerability-scans.yml) +- `ci-[app_name]-pr-environment-checks.yml`: calls `pr-environment-checks.yml` to create or update a pull request environment (see [pull request environments](/docs/infra/pull-request-environments.md)) + - Based on [ci-app-pr-environment-checks.yml](/.github/workflows/ci-app-pr-environment-checks.yml) +- `ci-[app_name]-pr-environment-destroy.yml`: calls `pr-environment-destroy.yml` to destroy the pull request environment (see [pull request environments](/docs/infra/pull-request-environments.md)) + - Based on [ci-app-pr-environment-destroy.yml](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-pr-environment-destroy.yml) ### App-agnostic workflows @@ -43,5 +47,4 @@ graph TD ## ⛑️ Helper workflows -- [`check-infra-auth`](./check-infra-auth.yml): verifes that the project's Github repo is able to connect to AWS - +- [`check-ci-cd-auth`](./check-ci-cd-auth.yml): verifes that the project's Github repo is able to connect to AWS diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 8f129609..feb7f9af 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -24,10 +24,26 @@ on: type: string jobs: + get-commit-hash: + name: Get commit hash + runs-on: ubuntu-latest + outputs: + commit_hash: ${{ steps.get-commit-hash.outputs.commit_hash }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + - name: Get commit hash + id: get-commit-hash + run: | + COMMIT_HASH=$(git rev-parse ${{ inputs.ref }}) + echo "Commit hash: $COMMIT_HASH" + echo "commit_hash=$COMMIT_HASH" >> "$GITHUB_OUTPUT" build-and-publish: name: Build and publish runs-on: ubuntu-latest - concurrency: ${{ github.workflow }}-${{ github.sha }} + needs: get-commit-hash + concurrency: ${{ github.workflow }}-${{ needs.get-commit-hash.outputs.commit_hash }} permissions: contents: read @@ -38,14 +54,23 @@ jobs: with: ref: ${{ inputs.ref }} - - name: Build release - run: make APP_NAME=${{ inputs.app_name }} release-build - - name: Configure AWS credentials uses: ./.github/actions/configure-aws-credentials with: app_name: ${{ inputs.app_name }} environment: shared + - name: Check if image is already published + id: check-image-published + run: | + is_image_published=$(./bin/is-image-published "${{ inputs.app_name }}" "${{ inputs.ref }}") + echo "Is image published: $is_image_published" + echo "is_image_published=$is_image_published" >> "$GITHUB_OUTPUT" + + - name: Build release + if: steps.check-image-published.outputs.is_image_published == 'false' + run: make APP_NAME=${{ inputs.app_name }} release-build + - name: Publish release + if: steps.check-image-published.outputs.is_image_published == 'false' run: make APP_NAME=${{ inputs.app_name }} release-publish diff --git a/.github/workflows/check-infra-auth.yml b/.github/workflows/check-ci-cd-auth.yml similarity index 93% rename from .github/workflows/check-infra-auth.yml rename to .github/workflows/check-ci-cd-auth.yml index 5b85d560..4a54b9e9 100644 --- a/.github/workflows/check-infra-auth.yml +++ b/.github/workflows/check-ci-cd-auth.yml @@ -1,4 +1,4 @@ -name: Check GitHub Actions AWS Authentication +name: Check CI/CD AWS authentication on: workflow_dispatch: diff --git a/.github/workflows/check-infra-deploy-status.yml b/.github/workflows/check-infra-deploy-status.yml new file mode 100644 index 00000000..be2f62aa --- /dev/null +++ b/.github/workflows/check-infra-deploy-status.yml @@ -0,0 +1,72 @@ +# This workflow checks the status of infrastructure deployments to see whether +# infrastructure code configuration matches the actual state of the infrastructure. +# It does this by checking that Terraform plans show an empty diff (no changes) +# across all root modules and backend configurations. +name: Check infra deploy status + +on: + workflow_dispatch: + schedule: + # Run every day at 07:00 UTC (3am ET, 12am PT) after engineers are likely done with work + - cron: "0 7 * * *" + +jobs: + collect-configs: + name: Collect configs + runs-on: ubuntu-latest + outputs: + root_module_configs: ${{ steps.collect-infra-deploy-status-check-configs.outputs.root_module_configs }} + steps: + - uses: actions/checkout@v4 + - name: Collect root module configurations + id: collect-infra-deploy-status-check-configs + run: | + root_module_configs="$(./bin/infra-deploy-status-check-configs)" + echo "${root_module_configs}" + echo "root_module_configs=${root_module_configs}" >> "$GITHUB_OUTPUT" + check: + name: ${{ matrix.root_module_subdir }} ${{ matrix.backend_config_name }} + runs-on: ubuntu-latest + needs: collect-configs + + # Skip this job if there are no root module configurations to check, + # otherwise the GitHub actions will give the error: "Matrix must define at least one vector" + if: ${{ needs.collect-configs.outputs.root_module_configs != '[]' }} + + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.collect-configs.outputs.root_module_configs) }} + + permissions: + contents: read + id-token: write + + steps: + - uses: actions/checkout@v4 + - uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.8.3 + terraform_wrapper: false + + - name: Configure AWS credentials + uses: ./.github/actions/configure-aws-credentials + with: + account_name: ${{ matrix.infra_layer == 'accounts' && matrix.account_name || null }} + network_name: ${{ matrix.infra_layer == 'networks' && matrix.backend_config_name || null }} + app_name: ${{ contains(fromJSON('["build-repository", "database", "service"]'), matrix.infra_layer) && matrix.app_name || null }} + environment: ${{ contains(fromJSON('["build-repository", "database", "service"]'), matrix.infra_layer) && matrix.backend_config_name || null }} + + - name: Check Terraform plan + run: | + echo "::group::Initialize Terraform" + echo terraform -chdir="infra/${{ matrix.root_module_subdir }}" init -input=false -reconfigure -backend-config="${{ matrix.backend_config_name }}.s3.tfbackend" + terraform -chdir="infra/${{ matrix.root_module_subdir }}" init -input=false -reconfigure -backend-config="${{ matrix.backend_config_name }}.s3.tfbackend" + echo "::endgroup::" + + echo "::group::Check Terraform plan" + echo terraform -chdir="infra/${{ matrix.root_module_subdir }}" plan -input=false -detailed-exitcode ${{ matrix.extra_params }} + terraform -chdir="infra/${{ matrix.root_module_subdir }}" plan -input=false -detailed-exitcode ${{ matrix.extra_params }} + echo "::endgroup::" + env: + TF_IN_AUTOMATION: "true" diff --git a/.github/workflows/ci-app-pr-environment-checks.yml b/.github/workflows/ci-app-pr-environment-checks.yml new file mode 100644 index 00000000..345bddd8 --- /dev/null +++ b/.github/workflows/ci-app-pr-environment-checks.yml @@ -0,0 +1,21 @@ +name: CI App PR Environment Checks +on: + workflow_dispatch: + inputs: + pr_number: + required: true + type: string + commit_hash: + required: true + type: string + # !! Uncomment the following lines once you've set up the dev environment and are ready to enable PR environments + # pull_request: +jobs: + update: + name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise + uses: ./.github/workflows/pr-environment-checks.yml + with: + app_name: "app" + environment: "dev" + pr_number: ${{ inputs.pr_number || github.event.number }} + commit_hash: ${{ inputs.commit_hash || github.event.pull_request.head.sha }} diff --git a/.github/workflows/ci-app-pr-environment-destroy.yml b/.github/workflows/ci-app-pr-environment-destroy.yml new file mode 100644 index 00000000..ad4cbb01 --- /dev/null +++ b/.github/workflows/ci-app-pr-environment-destroy.yml @@ -0,0 +1,18 @@ +name: CI App PR Environment Destroy +on: + workflow_dispatch: + inputs: + pr_number: + required: true + type: string + # !! Uncomment the following lines once you've set up the dev environment and are ready to enable PR environments + # pull_request: + # types: [closed] +jobs: + destroy: + name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise + uses: ./.github/workflows/pr-environment-destroy.yml + with: + app_name: "app" + environment: "dev" + pr_number: ${{ inputs.pr_number || github.event.number }} diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index 1700033e..93ac1e3a 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -14,9 +14,9 @@ jobs: name: Lint markdown runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # This is the GitHub Actions-friendly port of the linter used in the Makefile. - uses: gaurav-nelson/github-action-markdown-link-check@1.0.15 with: - use-quiet-mode: 'yes' # errors only. - config-file: '.github/workflows/markdownlint-config.json' + use-quiet-mode: "yes" # errors only. + config-file: ".github/workflows/markdownlint-config.json" diff --git a/.github/workflows/ci-infra-service.yml b/.github/workflows/ci-infra-service.yml index 6e3bed7c..7c486c99 100644 --- a/.github/workflows/ci-infra-service.yml +++ b/.github/workflows/ci-infra-service.yml @@ -28,7 +28,7 @@ jobs: id-token: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: hashicorp/setup-terraform@v2 with: diff --git a/.github/workflows/ci-infra.yml b/.github/workflows/ci-infra.yml index f7f9a4bf..d9e4698d 100644 --- a/.github/workflows/ci-infra.yml +++ b/.github/workflows/ci-infra.yml @@ -21,7 +21,7 @@ jobs: name: Lint GitHub Actions workflows runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Download actionlint id: get_actionlint run: bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) @@ -33,14 +33,14 @@ jobs: name: Lint scripts runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Shellcheck run: make infra-lint-scripts check-terraform-format: name: Check Terraform format runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: hashicorp/setup-terraform@v2 with: terraform_version: 1.8.3 @@ -53,7 +53,7 @@ jobs: name: Validate Terraform modules runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: hashicorp/setup-terraform@v2 with: terraform_version: 1.8.3 @@ -64,7 +64,7 @@ jobs: name: Check compliance with checkov runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "3.10" @@ -88,7 +88,7 @@ jobs: pull-requests: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Run tfsec check uses: aquasecurity/tfsec-pr-commenter-action@v1.2.0 with: diff --git a/.github/workflows/database-migrations.yml b/.github/workflows/database-migrations.yml index 669d0bc0..12185189 100644 --- a/.github/workflows/database-migrations.yml +++ b/.github/workflows/database-migrations.yml @@ -31,7 +31,7 @@ jobs: id-token: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Terraform uses: ./.github/actions/setup-terraform diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4ce48e8c..6e37dffb 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -34,7 +34,7 @@ jobs: contents: read id-token: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Terraform uses: ./.github/actions/setup-terraform diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml new file mode 100644 index 00000000..8b3c9ffe --- /dev/null +++ b/.github/workflows/e2e-tests.yml @@ -0,0 +1,37 @@ +name: E2E Tests + +on: + workflow_call: + inputs: + service_endpoint: + required: true + type: string + app_name: + required: false + type: string + +jobs: + e2e: + name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install Playwright browsers + run: make e2e-setup-ci + + - name: Run e2e tests + run: make e2e-test APP_NAME=${{ inputs.app_name }} BASE_URL=${{ inputs.service_endpoint }} + + - name: Upload Playwright report + uses: actions/upload-artifact@v4 + with: + name: playwright-report + path: ./e2e/playwright-report diff --git a/.github/workflows/markdownlint-config.json b/.github/workflows/markdownlint-config.json index 48f3a6b4..db31cb9f 100644 --- a/.github/workflows/markdownlint-config.json +++ b/.github/workflows/markdownlint-config.json @@ -11,6 +11,9 @@ }, { "pattern": "^https://confluenceent.cms.gov" + }, + { + "pattern": "^https://www.hhs.gov" } ], "replacementPatterns": [ diff --git a/.github/workflows/pr-environment-checks.yml b/.github/workflows/pr-environment-checks.yml new file mode 100644 index 00000000..9e0fcbe7 --- /dev/null +++ b/.github/workflows/pr-environment-checks.yml @@ -0,0 +1,73 @@ +name: PR Environment Update +run-name: Update PR Environment ${{ inputs.pr_number }} +on: + workflow_call: + inputs: + app_name: + required: true + type: string + environment: + required: true + type: string + pr_number: + required: true + type: string + commit_hash: + required: true + type: string +jobs: + build-and-publish: + name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise + uses: ./.github/workflows/build-and-publish.yml + with: + app_name: ${{ inputs.app_name }} + ref: ${{ inputs.commit_hash }} + + update: + name: Update environment + needs: [build-and-publish] + runs-on: ubuntu-latest + + permissions: + contents: read + id-token: write + pull-requests: write # Needed to comment on PR + repository-projects: read # Workaround for GitHub CLI bug https://github.com/cli/cli/issues/6274 + + concurrency: pr-environment-${{ inputs.pr_number }} + + outputs: + service_endpoint: ${{ steps.update-environment.outputs.service_endpoint }} + + steps: + - uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.8.3 + terraform_wrapper: false + + - name: Configure AWS credentials + uses: ./.github/actions/configure-aws-credentials + with: + app_name: ${{ inputs.app_name }} + environment: ${{ inputs.environment }} + + - name: Update environment + id: update-environment + run: | + ./bin/update-pr-environment "${{ inputs.app_name }}" "${{ inputs.environment }}" "${{ inputs.pr_number }}" "${{ inputs.commit_hash }}" + service_endpoint=$(terraform -chdir="infra/${{ inputs.app_name }}/service" output -raw service_endpoint) + echo "service_endpoint=${service_endpoint}" + echo "service_endpoint=${service_endpoint}" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ github.token }} + + e2e-tests: + name: Run E2E Tests + needs: [update] + uses: ./.github/workflows/e2e-tests.yml + with: + service_endpoint: ${{ needs.update.outputs.service_endpoint }} + app_name: ${{ inputs.app_name }} diff --git a/.github/workflows/pr-environment-destroy.yml b/.github/workflows/pr-environment-destroy.yml new file mode 100644 index 00000000..346b8c11 --- /dev/null +++ b/.github/workflows/pr-environment-destroy.yml @@ -0,0 +1,46 @@ +name: PR Environment Destroy +run-name: Destroy PR Environment ${{ inputs.pr_number }} +on: + workflow_call: + inputs: + app_name: + required: true + type: string + environment: + required: true + type: string + pr_number: + required: true + type: string +jobs: + destroy: + name: Destroy environment + runs-on: ubuntu-latest + + permissions: + contents: read + id-token: write + pull-requests: write # Needed to comment on PR + repository-projects: read # Workaround for GitHub CLI bug https://github.com/cli/cli/issues/6274 + + concurrency: pr-environment-${{ inputs.pr_number }} + + steps: + - uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.8.3 + terraform_wrapper: false + + - name: Configure AWS credentials + uses: ./.github/actions/configure-aws-credentials + with: + app_name: ${{ inputs.app_name }} + environment: ${{ inputs.environment }} + + - name: Destroy environment + run: ./bin/destroy-pr-environment "${{ inputs.app_name }}" "${{ inputs.environment }}" "${{ inputs.pr_number }}" + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/vulnerability-scans.yml b/.github/workflows/vulnerability-scans.yml index 53e5968f..232f34b6 100644 --- a/.github/workflows/vulnerability-scans.yml +++ b/.github/workflows/vulnerability-scans.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # Scans Dockerfile for any bad practices or issues - name: Scan Dockerfile by hadolint @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build and tag Docker image for scanning id: build-image @@ -67,7 +67,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build and tag Docker image for scanning id: build-image @@ -91,7 +91,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build and tag Docker image for scanning id: build-image diff --git a/.template-version b/.template-version index 5b17916e..ab6d47de 100644 --- a/.template-version +++ b/.template-version @@ -1 +1 @@ -a16c6247afc979c69511316c89bc79d940362476 +9616fcf8f156206aea4c3cb0a81459d7becef1ef diff --git a/Makefile b/Makefile index 7812952f..29424d56 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ PROJECT_ROOT ?= $(notdir $(PWD)) -# Use `=` instead of `:=` so that we only execute `./bin/current-account-alias.sh` when needed +# Use `=` instead of `:=` so that we only execute `./bin/current-account-alias` when needed # See https://www.gnu.org/software/make/manual/html_node/Flavors.html#Flavors -CURRENT_ACCOUNT_ALIAS = `./bin/current-account-alias.sh` +CURRENT_ACCOUNT_ALIAS = `./bin/current-account-alias` -CURRENT_ACCOUNT_ID = $(./bin/current-account-id.sh) +CURRENT_ACCOUNT_ID = $(./bin/current-account-id) # Get the list of reusable terraform modules by getting out all the modules # in infra/modules and then stripping out the "infra/modules/" prefix @@ -32,6 +32,7 @@ __check_defined = \ infra-check-compliance-checkov \ infra-check-compliance-tfsec \ infra-check-compliance \ + infra-check-github-actions-auth \ infra-configure-app-build-repository \ infra-configure-app-database \ infra-configure-app-service \ @@ -42,6 +43,7 @@ __check_defined = \ infra-lint-scripts \ infra-lint-terraform \ infra-lint-workflows \ + infra-module-database-role-manager \ infra-set-up-account \ infra-test-service \ infra-update-app-build-repository \ @@ -57,40 +59,40 @@ __check_defined = \ release-image-name \ release-image-tag \ release-publish \ - release-run-database-migrations - - + release-run-database-migrations \ + e2e-setup \ + e2e-test infra-set-up-account: ## Configure and create resources for current AWS profile and save tfbackend file to infra/accounts/$ACCOUNT_NAME.ACCOUNT_ID.s3.tfbackend @:$(call check_defined, ACCOUNT_NAME, human readable name for account e.g. "prod" or the AWS account alias) - ./bin/set-up-current-account.sh $(ACCOUNT_NAME) + ./bin/set-up-current-account $(ACCOUNT_NAME) infra-configure-network: ## Configure network $NETWORK_NAME @:$(call check_defined, NETWORK_NAME, the name of the network in /infra/networks) - ./bin/create-tfbackend.sh infra/networks $(NETWORK_NAME) + ./bin/create-tfbackend infra/networks $(NETWORK_NAME) infra-configure-app-build-repository: ## Configure infra/$APP_NAME/build-repository tfbackend and tfvars files @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) - ./bin/create-tfbackend.sh "infra/$(APP_NAME)/build-repository" shared + ./bin/create-tfbackend "infra/$(APP_NAME)/build-repository" shared infra-configure-app-database: ## Configure infra/$APP_NAME/database module's tfbackend and tfvars files for $ENVIRONMENT @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) @:$(call check_defined, ENVIRONMENT, the name of the application environment e.g. "prod" or "staging") - ./bin/create-tfbackend.sh "infra/$(APP_NAME)/database" "$(ENVIRONMENT)" + ./bin/create-tfbackend "infra/$(APP_NAME)/database" "$(ENVIRONMENT)" infra-configure-monitoring-secrets: ## Set $APP_NAME's incident management service integration URL for $ENVIRONMENT @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) @:$(call check_defined, ENVIRONMENT, the name of the application environment e.g. "prod" or "staging") @:$(call check_defined, URL, incident management service (PagerDuty or VictorOps) integration URL) - ./bin/configure-monitoring-secret.sh $(APP_NAME) $(ENVIRONMENT) $(URL) + ./bin/configure-monitoring-secret $(APP_NAME) $(ENVIRONMENT) $(URL) infra-configure-app-service: ## Configure infra/$APP_NAME/service module's tfbackend and tfvars files for $ENVIRONMENT @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) @:$(call check_defined, ENVIRONMENT, the name of the application environment e.g. "prod" or "staging") - ./bin/create-tfbackend.sh "infra/$(APP_NAME)/service" "$(ENVIRONMENT)" + ./bin/create-tfbackend "infra/$(APP_NAME)/service" "$(ENVIRONMENT)" infra-update-current-account: ## Update infra resources for current AWS profile - ./bin/terraform-init-and-apply.sh infra/accounts `./bin/current-account-config-name.sh` + ./bin/terraform-init-and-apply infra/accounts `./bin/current-account-config-name` infra-update-network: ## Update network @:$(call check_defined, NETWORK_NAME, the name of the network in /infra/networks) @@ -99,7 +101,7 @@ infra-update-network: ## Update network infra-update-app-build-repository: ## Create or update $APP_NAME's build repository @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) - ./bin/terraform-init-and-apply.sh infra/$(APP_NAME)/build-repository shared + ./bin/terraform-init-and-apply infra/$(APP_NAME)/build-repository shared infra-update-app-database: ## Create or update $APP_NAME's database module for $ENVIRONMENT @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) @@ -107,10 +109,14 @@ infra-update-app-database: ## Create or update $APP_NAME's database module for $ terraform -chdir="infra/$(APP_NAME)/database" init -input=false -reconfigure -backend-config="$(ENVIRONMENT).s3.tfbackend" terraform -chdir="infra/$(APP_NAME)/database" apply -var="environment_name=$(ENVIRONMENT)" +infra-module-database-role-manager-archive: ## Build/rebuild role manager code package for Lambda deploys + pip3 install -r infra/modules/database/role_manager/requirements.txt -t infra/modules/database/role_manager/vendor --upgrade + zip -r infra/modules/database/role_manager.zip infra/modules/database/role_manager + infra-update-app-database-roles: ## Create or update database roles and schemas for $APP_NAME's database in $ENVIRONMENT @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) @:$(call check_defined, ENVIRONMENT, the name of the application environment e.g. "prod" or "staging") - ./bin/create-or-update-database-roles.sh $(APP_NAME) $(ENVIRONMENT) + ./bin/create-or-update-database-roles $(APP_NAME) $(ENVIRONMENT) infra-update-app-service: ## Create or update $APP_NAME's web service module @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) @@ -131,11 +137,16 @@ infra-validate-module-%: infra-check-app-database-roles: ## Check that app database roles have been configured properly @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) @:$(call check_defined, ENVIRONMENT, the name of the application environment e.g. "prod" or "staging") - ./bin/check-database-roles.sh $(APP_NAME) $(ENVIRONMENT) + ./bin/check-database-roles $(APP_NAME) $(ENVIRONMENT) infra-check-compliance: ## Run compliance checks infra-check-compliance: infra-check-compliance-checkov infra-check-compliance-tfsec +infra-check-github-actions-auth: ## Check that GitHub actions can authenticate to the AWS account + @:$(call check_defined, ACCOUNT_NAME, the name of account in infra/accounts) + ./bin/check-github-actions-auth $(ACCOUNT_NAME) + + infra-check-compliance-checkov: ## Run checkov compliance checks checkov --directory infra @@ -161,7 +172,7 @@ infra-test-service: ## Run service layer infra test suite cd infra/test && go test -run TestService -v -timeout 30m lint-markdown: ## Lint Markdown docs for broken links - ./bin/lint-markdown.sh + ./bin/lint-markdown ######################## ## Release Management ## @@ -192,17 +203,17 @@ release-build: ## Build release for $APP_NAME and tag it with current git hash release-publish: ## Publish release to $APP_NAME's build repository @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) - ./bin/publish-release.sh $(APP_NAME) $(IMAGE_NAME) $(IMAGE_TAG) + ./bin/publish-release $(APP_NAME) $(IMAGE_NAME) $(IMAGE_TAG) release-run-database-migrations: ## Run $APP_NAME's database migrations in $ENVIRONMENT @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) @:$(call check_defined, ENVIRONMENT, the name of the application environment e.g. "prod" or "dev") - ./bin/run-database-migrations.sh $(APP_NAME) $(IMAGE_TAG) $(ENVIRONMENT) + ./bin/run-database-migrations $(APP_NAME) $(IMAGE_TAG) $(ENVIRONMENT) release-deploy: ## Deploy release to $APP_NAME's web service in $ENVIRONMENT @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) @:$(call check_defined, ENVIRONMENT, the name of the application environment e.g. "prod" or "dev") - ./bin/deploy-release.sh $(APP_NAME) $(IMAGE_TAG) $(ENVIRONMENT) + ./bin/deploy-release $(APP_NAME) $(IMAGE_TAG) $(ENVIRONMENT) release-image-name: ## Prints the image name of the release image @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) @@ -211,6 +222,28 @@ release-image-name: ## Prints the image name of the release image release-image-tag: ## Prints the image tag of the release image @echo $(IMAGE_TAG) +############################## +## End-to-end (E2E) Testing ## +############################## + +e2e-setup: ## Setup end-to-end tests + @cd e2e && npm install + @cd e2e && npx playwright install --with-deps + +e2e-setup-ci: ## Install system dependencies, Node dependencies, and Playwright browsers + sudo apt-get update + sudo apt-get install -y libwoff1 libopus0 libvpx7 libevent-2.1-7 libopus0 libgstreamer1.0-0 \ + libgstreamer-plugins-base1.0-0 libgstreamer-plugins-good1.0-0 libharfbuzz-icu0 libhyphen0 \ + libenchant-2-2 libflite1 libgles2 libx264-dev + cd e2e && npm ci + cd e2e && npx playwright install --with-deps + + +e2e-test: ## Run end-to-end tests + @:$(call check_defined, APP_NAME, You must pass in a specific APP_NAME) + @:$(call check_defined, BASE_URL, You must pass in a BASE_URL) + @cd e2e/$(APP_NAME) && APP_NAME=$(APP_NAME) BASE_URL=$(BASE_URL) npx playwright test $(E2E_ARGS) + ######################## ## Scripts and Helper ## ######################## diff --git a/app/app/assets/stylesheets/application.postcss.css b/app/app/assets/stylesheets/application.postcss.css index 91f43eb7..797336c5 100644 --- a/app/app/assets/stylesheets/application.postcss.css +++ b/app/app/assets/stylesheets/application.postcss.css @@ -2,5 +2,4 @@ @forward "uswds-settings.scss"; @forward "uswds"; @forward "uswds-overrides.scss"; -@forward "lds-ripple.scss"; @forward "cbv.scss"; diff --git a/app/app/assets/stylesheets/cbv.scss b/app/app/assets/stylesheets/cbv.scss index 6bc6c8aa..79a8c42d 100644 --- a/app/app/assets/stylesheets/cbv.scss +++ b/app/app/assets/stylesheets/cbv.scss @@ -107,3 +107,12 @@ html { .cbv-row-highlight td { @include u-bg('yellow-5v'); } + +.rotate { + animation: 1s linear infinite rotate-con; +} + +@keyframes rotate-con { + from {transform: rotate(0deg);} + to {transform: rotate(360deg);} +} diff --git a/app/app/assets/stylesheets/lds-ripple.scss b/app/app/assets/stylesheets/lds-ripple.scss deleted file mode 100644 index 74656c4e..00000000 --- a/app/app/assets/stylesheets/lds-ripple.scss +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Loading screen ripple - */ -.lds-ripple, -.lds-ripple div { - box-sizing: border-box; -} -.lds-ripple { - display: inline-block; - position: relative; - width: 80px; - height: 80px; -} -.lds-ripple div { - position: absolute; - border: 4px solid currentColor; - opacity: 1; - border-radius: 50%; - animation: lds-ripple 1s cubic-bezier(0, 0.2, 0.8, 1) infinite; -} -.lds-ripple div:nth-child(2) { - animation-delay: -0.5s; -} -@keyframes lds-ripple { - 0% { - top: 36px; - left: 36px; - width: 8px; - height: 8px; - opacity: 0; - } - 4.9% { - top: 36px; - left: 36px; - width: 8px; - height: 8px; - opacity: 0; - } - 5% { - top: 36px; - left: 36px; - width: 8px; - height: 8px; - opacity: 1; - } - 100% { - top: 0; - left: 0; - width: 80px; - height: 80px; - opacity: 0; - } -} diff --git a/app/app/assets/stylesheets/uswds-settings.scss b/app/app/assets/stylesheets/uswds-settings.scss index 63803646..0a39e22f 100644 --- a/app/app/assets/stylesheets/uswds-settings.scss +++ b/app/app/assets/stylesheets/uswds-settings.scss @@ -12,4 +12,5 @@ $theme-header-logo-text-width: 100%, $theme-header-min-width: "tablet", $theme-site-margins-breakpoint: "tablet", + $theme-step-indicator-segment-height: 0, ); diff --git a/app/app/channels/paystubs_channel.rb b/app/app/channels/paystubs_channel.rb index 65f3e39c..4de688ca 100644 --- a/app/app/channels/paystubs_channel.rb +++ b/app/app/channels/paystubs_channel.rb @@ -1,10 +1,26 @@ class PaystubsChannel < ApplicationCable::Channel + periodically :check_pinwheel_account_synchrony, every: 5.seconds + def subscribed - cbv_flow = CbvFlow.find(connection.session[:cbv_flow_id]) - stream_for cbv_flow + @cbv_flow = CbvFlow.find(connection.session[:cbv_flow_id]) + stream_for @cbv_flow end - def unsubscribed - # Any cleanup needed when channel is unsubscribed + private + + def check_pinwheel_account_synchrony + pinwheel_account = PinwheelAccount.find_by_pinwheel_account_id(params["account_id"]) + + if pinwheel_account.present? + broadcast_to(@cbv_flow, { + event: "cbv.status_update", + account_id: pinwheel_account.pinwheel_account_id, + employment: pinwheel_account.job_completed?("employment"), + identity: pinwheel_account.job_completed?("identity"), + paystubs: pinwheel_account.job_completed?("paystubs"), + income: pinwheel_account.job_completed?("income"), + has_fully_synced: pinwheel_account.has_fully_synced? + }) + end end end diff --git a/app/app/controllers/cbv/base_controller.rb b/app/app/controllers/cbv/base_controller.rb index d8d427ed..8f90568e 100644 --- a/app/app/controllers/cbv/base_controller.rb +++ b/app/app/controllers/cbv/base_controller.rb @@ -56,6 +56,8 @@ def next_path when "cbv/agreements" cbv_flow_employer_search_path when "cbv/employer_searches" + cbv_flow_synchronizations_path + when "cbv/synchronizations" cbv_flow_payment_details_path when "cbv/missing_results" cbv_flow_summary_path diff --git a/app/app/controllers/cbv/synchronizations_controller.rb b/app/app/controllers/cbv/synchronizations_controller.rb new file mode 100644 index 00000000..5c2bf3eb --- /dev/null +++ b/app/app/controllers/cbv/synchronizations_controller.rb @@ -0,0 +1,19 @@ +class Cbv::SynchronizationsController < Cbv::BaseController + helper_method :job_completed? + + def show + account_id = params[:user][:account_id] + + @pinwheel_account = @cbv_flow.pinwheel_accounts.find_by(pinwheel_account_id: account_id) + + if @pinwheel_account && @pinwheel_account.has_fully_synced? + redirect_to cbv_flow_payment_details_path(user: { account_id: @pinwheel_account.pinwheel_account_id }) + end + end + + private + + def job_completed?(job) + @pinwheel_account.present? && @pinwheel_account.job_completed?(job) + end +end diff --git a/app/app/javascript/controllers/cbv/employer_search.js b/app/app/javascript/controllers/cbv/employer_search.js index 1b0c8a97..c4dc19fb 100644 --- a/app/app/javascript/controllers/cbv/employer_search.js +++ b/app/app/javascript/controllers/cbv/employer_search.js @@ -1,36 +1,16 @@ import { Controller } from "@hotwired/stimulus" -import * as ActionCable from '@rails/actioncable' import { loadPinwheel, initializePinwheel, fetchToken } from "../../utilities/pinwheel" export default class extends Controller { static targets = [ "form", - "searchTerms", "userAccountId", - "modal", "employerButton" ]; pinwheel = loadPinwheel(); - cable = ActionCable.createConsumer(); - connect() { - this.cable.subscriptions.create({ channel: 'PaystubsChannel' }, { - connected: () => { - console.log("Connected to the channel:", this); - }, - disconnected: () => { - console.log("Disconnected"); - }, - received: (data) => { - if (data.event === 'cbv.payroll_data_available') { - const accountId = data.account_id - this.userAccountIdTarget.value = accountId - this.formTarget.submit(); - } - } - }); this.errorHandler = document.addEventListener("turbo:frame-missing", this.onTurboError) } @@ -38,12 +18,6 @@ export default class extends Controller { document.removeEventListener("turbo:frame-missing", this.errorHandler) } - onSignInSuccess() { - this.pinwheel.then(pinwheel => pinwheel.close()); - this.modalTarget.click(); - this.reenableButtons() - } - onTurboError(event) { console.warn("Got turbo error, redirecting:", event) @@ -61,6 +35,14 @@ export default class extends Controller { } } + onPinwheelEvent(eventName, eventPayload) { + if (eventName === 'success') { + const { accountId } = eventPayload; + this.userAccountIdTarget.value = accountId + this.formTarget.submit(); + } + } + async select(event) { const { responseType, id } = event.target.dataset; this.disableButtons() @@ -71,10 +53,9 @@ export default class extends Controller { submit(token) { this.pinwheel.then(Pinwheel => initializePinwheel(Pinwheel, token, { - onEvent: console.log, + onEvent: this.onPinwheelEvent.bind(this), onError: this.onPinwheelError.bind(this), onExit: this.reenableButtons.bind(this), - onSuccess: this.onSignInSuccess.bind(this), })); } diff --git a/app/app/javascript/controllers/cbv/synchronizations_controller.js b/app/app/javascript/controllers/cbv/synchronizations_controller.js new file mode 100644 index 00000000..8a56021b --- /dev/null +++ b/app/app/javascript/controllers/cbv/synchronizations_controller.js @@ -0,0 +1,43 @@ +import { Controller } from "@hotwired/stimulus" +import * as ActionCable from '@rails/actioncable' + +export default class extends Controller { + static targets = ["form", "userAccountId", "employmentJob", "identityJob", "paystubsJob", "incomeJob"]; + + cable = ActionCable.createConsumer(); + + updateIndicatorStatus(element, completed) { + if (completed) { + element.querySelector('.completed').classList.remove('display-none'); + element.querySelector('.in-progress').classList.add('display-none'); + } else { + element.querySelector('.completed').classList.add('display-none'); + element.querySelector('.in-progress').classList.remove('display-none'); + } + } + + connect() { + this.cable.subscriptions.create({ channel: 'PaystubsChannel', account_id: this.userAccountIdTarget.value }, { + connected: () => { + console.log("Connected to the channel:", this); + }, + disconnected: () => { + console.log("Disconnected"); + }, + received: (data) => { + if (data.event === 'cbv.status_update') { + if (data.has_fully_synced) { + const accountId = data.account_id; + this.userAccountIdTarget.value = accountId; + this.formTarget.submit(); + } + + this.updateIndicatorStatus(this.employmentJobTarget, data.employment); + this.updateIndicatorStatus(this.identityJobTarget, data.identity); + this.updateIndicatorStatus(this.paystubsJobTarget, data.paystubs); + this.updateIndicatorStatus(this.incomeJobTarget, data.income); + } + } + }); + } +} diff --git a/app/app/javascript/controllers/index.js b/app/app/javascript/controllers/index.js index 5ac5a50d..2b055491 100644 --- a/app/app/javascript/controllers/index.js +++ b/app/app/javascript/controllers/index.js @@ -5,4 +5,6 @@ import { application } from "./application" import CbvEmployerSearch from "./cbv/employer_search" +import CbvSynchronizationsController from "./cbv/synchronizations_controller" application.register("cbv-employer-search", CbvEmployerSearch) +application.register("cbv-synchronizations", CbvSynchronizationsController) diff --git a/app/app/models/pinwheel_account.rb b/app/app/models/pinwheel_account.rb index 472c1689..ad3d19da 100644 --- a/app/app/models/pinwheel_account.rb +++ b/app/app/models/pinwheel_account.rb @@ -23,10 +23,25 @@ def has_fully_synced? end def job_succeeded?(job) - error_column = EVENTS_ERRORS_MAP.select { |key| key.start_with? job }&.values.last - sync_column = EVENTS_MAP.select { |key| key.start_with? job }&.values.last + error_column, sync_column = event_columns_for(job) return nil unless error_column.present? supported_jobs.include?(job) && send(sync_column).present? && send(error_column).blank? end + + def job_completed?(job) + error_column, sync_column = event_columns_for(job) + return nil unless error_column.present? + + supported_jobs.include?(job) && (send(sync_column).present? || send(error_column).present?) + end + + private + + def event_columns_for(job) + error_column = EVENTS_ERRORS_MAP.select { |key| key.start_with? job }&.values.last + sync_column = EVENTS_MAP.select { |key| key.start_with? job }&.values.last + + [ error_column, sync_column ] + end end diff --git a/app/app/views/cbv/employer_searches/show.html.erb b/app/app/views/cbv/employer_searches/show.html.erb index 51e46c58..9de8c231 100644 --- a/app/app/views/cbv/employer_searches/show.html.erb +++ b/app/app/views/cbv/employer_searches/show.html.erb @@ -28,7 +28,7 @@ <%= form_with url: cbv_flow_employer_search_path, method: :get, class: "usa-search usa-search--big margin-y-4", html: { role: "search" }, data: { turbo_frame: "employers", turbo_action: "advance" } do |f| %> <%= f.label :query, "Search for your employer", class: "usa-sr-only" %> - <%= f.text_field :query, value: @query, class: "usa-input", type: "search", data: { "cbv-employer-search-target": "searchTerms" } %> + <%= f.text_field :query, value: @query, class: "usa-input", type: "search" %>