From 63f93663a01cfa21ff4a027b14922f7269a6e68b Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Wed, 10 Jan 2024 15:12:22 -0800 Subject: [PATCH] tmp --- .dockleconfig | 4 +- .../configure-aws-credentials/action.yml | 2 +- .github/pull_request_template.md | 15 +- .github/workflows/build-and-publish.yml | 13 +- .github/workflows/cd-app.yml | 33 ++++ .../workflows/ci-app-vulnerability-scans.yml | 26 +++ .github/workflows/ci-docs.yml | 20 +++ .github/workflows/ci-infra-service.yml | 50 ++++++ .github/workflows/database-migrations.yml | 1 - .github/workflows/markdownlint-config.json | 19 ++ .github/workflows/vulnerability-scans.yml | 118 +++--------- .grype.yml | 10 -- .hadolint.yaml | 6 - .template-version | 2 +- .trivyignore | 9 +- Makefile | 23 ++- bin/account-ids-by-name.sh | 2 +- bin/check-github-actions-auth.sh | 4 +- bin/configure-monitoring-secret.sh | 7 +- bin/lint-markdown.sh | 18 ++ bin/run-command.sh | 2 +- bin/set-up-current-account.sh | 22 ++- docs/code-reviews.md | 55 ++++++ docs/compliance.md | 29 +++ docs/decisions/index.md | 22 +++ ...markdown-architectural-decision-records.md | 26 +++ docs/decisions/infra/0001-ci-cd-interface.md | 113 ++++++++++++ ...se-custom-implementation-of-github-oidc.md | 38 ++++ .../0003-manage-ecr-in-prod-account-module.md | 33 ++++ ...kend-configs-into-separate-config-files.md | 36 ++++ ...base-infrastructure-into-separate-layer.md | 66 +++++++ ...database-users-with-serverless-function.md | 87 +++++++++ .../0007-database-migration-architecture.md | 92 ++++++++++ ...ig-from-tfvars-files-into-config-module.md | 30 ++++ ...separate-app-infrastructure-into-layers.md | 40 +++++ .../infra/0010-feature-flags-system-design.md | 124 +++++++++++++ .../infra/0011-network-layer-design.md | 135 ++++++++++++++ docs/decisions/template.md | 72 ++++++++ docs/feature-flags.md | 31 ++++ docs/infra/database-access-control.md | 24 +++ docs/infra/destroy-infrastructure.md | 59 ++++++ docs/infra/intro-to-terraform-workspaces.md | 59 ++++++ docs/infra/intro-to-terraform.md | 33 ++++ docs/infra/making-infra-changes.md | 56 ++++++ docs/infra/module-architecture.md | 92 ++++++++++ docs/infra/module-dependencies.md | 92 ++++++++++ docs/infra/set-up-app-build-repository.md | 32 ++++ docs/infra/set-up-app-env.md | 60 +++++++ docs/infra/set-up-aws-account.md | 57 ++++++ docs/infra/set-up-database.md | 88 +++++++++ docs/infra/set-up-infrastructure-tools.md | 102 +++++++++++ docs/infra/set-up-monitoring-alerts.md | 29 +++ docs/infra/set-up-network.md | 38 ++++ docs/infra/vulnerability-management.md | 35 ++++ docs/releases.md | 21 +++ docs/system-architecture.md | 21 +++ infra/README.md | 83 ++++++++- infra/accounts/main.tf | 2 +- infra/app/app-config/dev.tf | 9 + infra/app/app-config/env-config/outputs.tf | 30 ++++ infra/app/app-config/env-config/variables.tf | 41 +++++ infra/app/app-config/main.tf | 75 ++++++++ infra/app/app-config/outputs.tf | 39 ++++ infra/app/app-config/prod.tf | 16 ++ infra/app/app-config/staging.tf | 9 + .../app/build-repository/.terraform.lock.hcl | 22 +++ infra/app/build-repository/main.tf | 59 ++++++ infra/app/database/main.tf | 92 ++++++++++ infra/app/database/outputs.tf | 3 + infra/app/database/variables.tf | 4 + infra/app/service/image_tag.tf | 56 ++++++ infra/app/service/main.tf | 169 ++++++++++++++++++ infra/app/service/outputs.tf | 24 +++ infra/app/service/variables.tf | 10 ++ infra/modules/auth-github-actions/main.tf | 15 +- infra/modules/database/backups.tf | 2 +- infra/modules/database/main.tf | 10 +- infra/modules/database/monitoring.tf | 2 +- infra/modules/database/networking.tf | 28 --- infra/modules/database/role-manager.tf | 40 +++-- .../database/role_manager/requirements.txt | 2 +- .../database/role_manager/role_manager.py | 18 +- infra/modules/database/variables.tf | 10 +- infra/modules/feature-flags/access-policy.tf | 21 +++ infra/modules/feature-flags/logs.tf | 47 +++++ infra/modules/feature-flags/main.tf | 49 +++++ infra/modules/feature-flags/outputs.tf | 9 + infra/modules/feature-flags/variables.tf | 9 + infra/modules/monitoring/main.tf | 2 +- infra/modules/network/main.tf | 32 ++++ infra/modules/network/variables.tf | 26 +++ infra/modules/network/vpc-endpoints.tf | 92 ++++++++++ infra/modules/service/access-control.tf | 7 + infra/modules/service/database-access.tf | 2 +- infra/modules/service/load-balancer.tf | 59 +----- infra/modules/service/main.tf | 22 ++- infra/modules/service/networking.tf | 76 ++++---- infra/modules/service/variables.tf | 68 ++----- infra/modules/storage/access-control.tf | 58 ++++++ infra/modules/storage/encryption.tf | 18 ++ infra/modules/storage/events.tf | 7 + infra/modules/storage/lifecycle.tf | 11 ++ infra/modules/storage/main.tf | 10 ++ infra/modules/storage/outputs.tf | 3 + infra/modules/storage/variables.tf | 4 + infra/modules/terraform-backend-s3/main.tf | 4 +- infra/networks/main.tf | 119 ++++-------- infra/networks/variables.tf | 4 + infra/project-config/main.tf | 17 +- infra/project-config/outputs.tf | 6 +- infra/test/go.mod | 39 ++-- infra/test/go.sum | 50 ------ infra/test/infra_test.go | 35 +++- 113 files changed, 3536 insertions(+), 573 deletions(-) create mode 100644 .github/workflows/cd-app.yml create mode 100644 .github/workflows/ci-app-vulnerability-scans.yml create mode 100644 .github/workflows/ci-docs.yml create mode 100644 .github/workflows/ci-infra-service.yml create mode 100644 .github/workflows/markdownlint-config.json create mode 100755 bin/lint-markdown.sh create mode 100644 docs/code-reviews.md create mode 100644 docs/compliance.md create mode 100644 docs/decisions/index.md create mode 100644 docs/decisions/infra/0000-use-markdown-architectural-decision-records.md create mode 100644 docs/decisions/infra/0001-ci-cd-interface.md create mode 100644 docs/decisions/infra/0002-use-custom-implementation-of-github-oidc.md create mode 100644 docs/decisions/infra/0003-manage-ecr-in-prod-account-module.md create mode 100644 docs/decisions/infra/0004-separate-terraform-backend-configs-into-separate-config-files.md create mode 100644 docs/decisions/infra/0005-separate-database-infrastructure-into-separate-layer.md create mode 100644 docs/decisions/infra/0006-provision-database-users-with-serverless-function.md create mode 100644 docs/decisions/infra/0007-database-migration-architecture.md create mode 100644 docs/decisions/infra/0008-consolidate-infra-config-from-tfvars-files-into-config-module.md create mode 100644 docs/decisions/infra/0009-separate-app-infrastructure-into-layers.md create mode 100644 docs/decisions/infra/0010-feature-flags-system-design.md create mode 100644 docs/decisions/infra/0011-network-layer-design.md create mode 100644 docs/decisions/template.md create mode 100644 docs/feature-flags.md create mode 100644 docs/infra/database-access-control.md create mode 100644 docs/infra/destroy-infrastructure.md create mode 100644 docs/infra/intro-to-terraform-workspaces.md create mode 100644 docs/infra/intro-to-terraform.md create mode 100644 docs/infra/making-infra-changes.md create mode 100644 docs/infra/module-architecture.md create mode 100644 docs/infra/module-dependencies.md create mode 100644 docs/infra/set-up-app-build-repository.md create mode 100644 docs/infra/set-up-app-env.md create mode 100644 docs/infra/set-up-aws-account.md create mode 100644 docs/infra/set-up-database.md create mode 100644 docs/infra/set-up-infrastructure-tools.md create mode 100644 docs/infra/set-up-monitoring-alerts.md create mode 100644 docs/infra/set-up-network.md create mode 100644 docs/infra/vulnerability-management.md create mode 100644 docs/releases.md create mode 100644 docs/system-architecture.md create mode 100644 infra/app/app-config/dev.tf create mode 100644 infra/app/app-config/env-config/outputs.tf create mode 100644 infra/app/app-config/env-config/variables.tf create mode 100644 infra/app/app-config/main.tf create mode 100644 infra/app/app-config/outputs.tf create mode 100644 infra/app/app-config/prod.tf create mode 100644 infra/app/app-config/staging.tf create mode 100644 infra/app/build-repository/.terraform.lock.hcl create mode 100644 infra/app/build-repository/main.tf create mode 100644 infra/app/database/main.tf create mode 100644 infra/app/database/outputs.tf create mode 100644 infra/app/database/variables.tf create mode 100644 infra/app/service/image_tag.tf create mode 100644 infra/app/service/main.tf create mode 100644 infra/app/service/outputs.tf create mode 100644 infra/app/service/variables.tf create mode 100644 infra/modules/feature-flags/access-policy.tf create mode 100644 infra/modules/feature-flags/logs.tf create mode 100644 infra/modules/feature-flags/main.tf create mode 100644 infra/modules/feature-flags/outputs.tf create mode 100644 infra/modules/feature-flags/variables.tf create mode 100644 infra/modules/network/main.tf create mode 100644 infra/modules/network/variables.tf create mode 100644 infra/modules/network/vpc-endpoints.tf create mode 100644 infra/modules/storage/access-control.tf create mode 100644 infra/modules/storage/encryption.tf create mode 100644 infra/modules/storage/events.tf create mode 100644 infra/modules/storage/lifecycle.tf create mode 100644 infra/modules/storage/main.tf create mode 100644 infra/modules/storage/outputs.tf create mode 100644 infra/modules/storage/variables.tf create mode 100644 infra/networks/variables.tf diff --git a/.dockleconfig b/.dockleconfig index 15c1c8ce3..f7bc97678 100644 --- a/.dockleconfig +++ b/.dockleconfig @@ -1,6 +1,4 @@ # This file is allows you to specify a list of files that is acceptable to Dockle # To allow multiple files, use a list of names, example below. Make sure to remove the leading # # DOCKLE_ACCEPT_FILES="file1,path/to/file2,file3/path,etc" -# https://github.com/goodwithtech/dockle#accept-suspicious-environment-variables--files--file-extensions -# The apiflask/settings file is a stub file that apiflask creates, and has no sensitive data in. We are ignoring it since it is unused -DOCKLE_ACCEPT_FILES=api/.venv/lib/python3.12/site-packages/apiflask/settings.py +# https://github.com/goodwithtech/dockle#accept-suspicious-environment-variables--files--file-extensions \ No newline at end of file diff --git a/.github/actions/configure-aws-credentials/action.yml b/.github/actions/configure-aws-credentials/action.yml index 82f53b57a..a92990a58 100644 --- a/.github/actions/configure-aws-credentials/action.yml +++ b/.github/actions/configure-aws-credentials/action.yml @@ -30,7 +30,7 @@ runs: echo "GITHUB_ACTIONS_ROLE_NAME=$GITHUB_ACTIONS_ROLE_NAME" terraform -chdir=infra/${{ inputs.app_name }}/app-config init > /dev/null - terraform -chdir=infra/${{ inputs.app_name }}/app-config apply -refresh-only -auto-approve> /dev/null + terraform -chdir=infra/${{ inputs.app_name }}/app-config apply -refresh-only -auto-approve> /dev/null ACCOUNT_NAME=$(terraform -chdir=infra/${{ inputs.app_name }}/app-config output -json account_names_by_environment | jq -r .${{ inputs.environment }}) echo "ACCOUNT_NAME=$ACCOUNT_NAME" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index e435dbeed..a7db834a4 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,14 +1,15 @@ -## Summary -Fixes #{ISSUE} +## Ticket -### Time to review: __x mins__ +Resolves #{TICKET NUMBER OR URL} + +## Changes -## Changes proposed > What was added, updated, or removed in this PR. ## Context for reviewers -> Testing instructions, background context, more in-depth details of the implementation, and anything else you'd like to call out or ask reviewers. Explain how the changes were verified. -## Additional information -> Screenshots, GIF demos, code examples or output to help show the changes working as expected. +> Testing instructions, background context, more in-depth details of the implementation, and anything else you'd like to call out or ask reviewers. + +## Testing +> Provide evidence that the code works as expected. Explain what was done for testing and the results of the test plan. Include screenshots, [GIF demos](https://www.cockos.com/licecap/), shell commands or output to help show the changes working as expected. ProTip: you can drag and drop or paste images into this textbox. diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index c6bf4261e..57a46eab7 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -11,11 +11,6 @@ on: description: The branch, tag or SHA to checkout. When checking out the repository that triggered a workflow, this defaults to the reference or SHA for that event. Otherwise, use branch or tag that triggered the workflow run. required: true type: string - environment: - description: "The environment where the build will be deployed. eg. dev or prod. Will default to dev." - default: dev - required: false - type: string workflow_dispatch: inputs: app_name: @@ -26,16 +21,12 @@ on: description: The branch, tag or SHA to checkout. When checking out the repository that triggered a workflow, this defaults to the reference or SHA for that event. Otherwise, use branch or tag that triggered the workflow run. required: true type: string - environment: - description: "The environment where the build will be deployed. eg. dev or prod. Will default to dev." - default: dev - required: false - type: string jobs: build-and-publish: name: Build and publish runs-on: ubuntu-latest + concurrency: ${{ github.action }}-${{ inputs.ref }} permissions: contents: read @@ -47,7 +38,7 @@ jobs: ref: ${{ inputs.ref }} - name: Build release - run: make APP_NAME=${{ inputs.app_name }} ENVIRONMENT=${{ inputs.environment }} release-build + run: make APP_NAME=${{ inputs.app_name }} release-build - name: Configure AWS credentials uses: ./.github/actions/configure-aws-credentials diff --git a/.github/workflows/cd-app.yml b/.github/workflows/cd-app.yml new file mode 100644 index 000000000..abd4660fe --- /dev/null +++ b/.github/workflows/cd-app.yml @@ -0,0 +1,33 @@ +name: Deploy App +# Need to set a default value for when the workflow is triggered from a git push +# which bypasses the default configuration for inputs +run-name: Deploy ${{ github.ref_name }} to App ${{ inputs.environment || 'dev' }} + +on: + # !! Uncomment the following lines once you've set up the dev environment and ready to turn on continuous deployment + # push: + # branches: + # - "main" + # paths: + # - "app/**" + # - "bin/**" + # - "infra/**" + workflow_dispatch: + inputs: + environment: + description: "target environment" + required: true + default: "dev" + type: choice + options: + - dev + - staging + - prod + +jobs: + deploy: + name: Deploy + uses: ./.github/workflows/deploy.yml + with: + app_name: "app" + environment: ${{ inputs.environment || 'dev' }} diff --git a/.github/workflows/ci-app-vulnerability-scans.yml b/.github/workflows/ci-app-vulnerability-scans.yml new file mode 100644 index 000000000..ffcc55224 --- /dev/null +++ b/.github/workflows/ci-app-vulnerability-scans.yml @@ -0,0 +1,26 @@ +name: CI Vulnerability Scans + +on: + push: + branches: + - main + paths: + - app/** + - .grype.yml + - .hadolint.yaml + - .trivyignore + - .github/workflows/ci-vulnerability-scans.yml + pull_request: + paths: + - app/** + - .grype.yml + - .hadolint.yaml + - .trivyignore + - .github/workflows/ci-vulnerability-scans.yml + +jobs: + vulnerability-scans: + name: Vulnerability Scans + uses: ./.github/workflows/vulnerability-scans.yml + with: + app_name: "app" diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml new file mode 100644 index 000000000..8ce042bed --- /dev/null +++ b/.github/workflows/ci-docs.yml @@ -0,0 +1,20 @@ +name: CI Documentation Checks + +on: + push: + branches: + - main + pull_request: + + +jobs: + lint-markdown: + name: Lint markdown + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + # This is the GitHub Actions-friendly port of the linter used in the Makefile. + - uses: gaurav-nelson/github-action-markdown-link-check@1.0.15 + with: + use-quiet-mode: 'yes' # errors only. + config-file: '.github/workflows/markdownlint-config.json' diff --git a/.github/workflows/ci-infra-service.yml b/.github/workflows/ci-infra-service.yml new file mode 100644 index 000000000..dfa680bf4 --- /dev/null +++ b/.github/workflows/ci-infra-service.yml @@ -0,0 +1,50 @@ +name: CI Infra Service Checks + +on: + # !! Uncomment to trigger automated infra tests once dev environment is set up + # push: + # branches: + # - main + # paths: + # - infra/*/service/** + # - infra/modules/** + # - infra/test/** + # - .github/workflows/ci-infra-service.yml + # pull_request: + # paths: + # - infra/*/service/** + # - infra/modules/** + # - infra/test/** + # - .github/workflows/ci-infra-service.yml + workflow_dispatch: + +jobs: + infra-test-e2e: + name: Test service + runs-on: ubuntu-latest + + permissions: + contents: read + id-token: write + + steps: + - uses: actions/checkout@v3 + + - uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.2.1 + terraform_wrapper: false + + - uses: actions/setup-go@v3 + with: + go-version: ">=1.19.0" + + - name: Configure AWS credentials + uses: ./.github/actions/configure-aws-credentials + with: + app_name: app + # Run infra CI on dev environment + environment: dev + + - name: Run Terratest + run: make infra-test-service diff --git a/.github/workflows/database-migrations.yml b/.github/workflows/database-migrations.yml index 8c295a10e..f4e44413b 100644 --- a/.github/workflows/database-migrations.yml +++ b/.github/workflows/database-migrations.yml @@ -21,7 +21,6 @@ jobs: with: app_name: ${{ inputs.app_name }} ref: ${{ github.ref }} - environment: ${{ inputs.environment }} run-migrations: name: Run migrations runs-on: ubuntu-latest diff --git a/.github/workflows/markdownlint-config.json b/.github/workflows/markdownlint-config.json new file mode 100644 index 000000000..fec553041 --- /dev/null +++ b/.github/workflows/markdownlint-config.json @@ -0,0 +1,19 @@ +{ + "ignorePatterns" : [ + { + "pattern": "0005-example.md" + }, + { + "pattern": "localhost" + }, + { + "pattern": "127.0.0.1" + } + ], + "replacementPatterns": [ + { + "pattern": "^/", + "replacement": "{{BASEURL}}/" + } + ] +} diff --git a/.github/workflows/vulnerability-scans.yml b/.github/workflows/vulnerability-scans.yml index 3b0337aad..53e5968fa 100644 --- a/.github/workflows/vulnerability-scans.yml +++ b/.github/workflows/vulnerability-scans.yml @@ -1,6 +1,8 @@ # GitHub Actions CI workflow that runs vulnerability scans on the application's Docker image # to ensure images built are secure before they are deployed. +# NOTE: The workflow isn't able to pass the docker image between jobs, so each builds the image. +# A future PR will pass the image between the scans to reduce overhead and increase speed name: Vulnerability Scans on: @@ -13,7 +15,6 @@ on: jobs: hadolint-scan: - name: Hadolint Scan runs-on: ubuntu-latest steps: @@ -30,87 +31,27 @@ jobs: - name: Save output to workflow summary if: always() # Runs even if there is a failure - run: | - cat hadolint-results.txt >> "$GITHUB_STEP_SUMMARY" + run: cat hadolint-results.txt >> "$GITHUB_STEP_SUMMARY" - build-and-cache: + trivy-scan: runs-on: ubuntu-latest - outputs: - image: ${{ steps.shared-output.outputs.image }} steps: - uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@master - - - name: Cache Docker layers - id: cache-buildx - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ inputs.app_name }}-buildx-${{ github.sha }} - restore-keys: | - ${{ inputs.app_name }}-buildx- - - - name: Ensure Buildx cache exists - run: | - mkdir -p /tmp/.buildx-cache - - - name: Set shared outputs - id: shared-output + - name: Build and tag Docker image for scanning + id: build-image run: | + make APP_NAME=${{ inputs.app_name }} release-build IMAGE_NAME=$(make APP_NAME=${{ inputs.app_name }} release-image-name) IMAGE_TAG=$(make release-image-tag) echo "image=$IMAGE_NAME:$IMAGE_TAG" >> "$GITHUB_OUTPUT" - - name: Build and tag Docker image for scanning - # If there's an exact match in cache, skip build entirely - if: steps.cache-buildx.outputs.cache-hit != 'true' - run: | - make release-build \ - APP_NAME=${{ inputs.app_name }} \ - OPTIONAL_BUILD_FLAGS=" \ - --cache-from=type=local,src=/tmp/.buildx-cache \ - --cache-to=type=local,dest=/tmp/.buildx-cache" - - - name: Save Docker image - if: steps.cache-buildx.outputs.cache-hit != 'true' - run: | - docker save ${{ steps.shared-output.outputs.image }} > /tmp/docker-image.tar - - - name: Cache Docker image - if: steps.cache-buildx.outputs.cache-hit != 'true' - uses: actions/cache/save@v3 - with: - path: /tmp/docker-image.tar - key: ${{ inputs.app_name }}-docker-image-${{ github.sha }} - - trivy-scan: - name: Trivy Scan - runs-on: ubuntu-latest - needs: build-and-cache - - steps: - - uses: actions/checkout@v3 - - - name: Restore cached Docker image - uses: actions/cache/restore@v3 - with: - path: /tmp/docker-image.tar - key: ${{ inputs.app_name }}-docker-image-${{ github.sha }} - restore-keys: | - ${{ inputs.app_name }}-docker-image- - - - name: Load cached Docker image - run: | - docker load < /tmp/docker-image.tar - - name: Run Trivy vulnerability scan uses: aquasecurity/trivy-action@master with: scan-type: image - image-ref: ${{ needs.build-and-cache.outputs.image }} + image-ref: ${{ steps.build-image.outputs.image }} format: table exit-code: 1 ignore-unfixed: true @@ -123,55 +64,42 @@ jobs: echo "View results in GitHub Action logs" >> "$GITHUB_STEP_SUMMARY" anchore-scan: - name: Anchore Scan runs-on: ubuntu-latest - needs: build-and-cache steps: - uses: actions/checkout@v3 - - name: Restore cached Docker image - uses: actions/cache/restore@v3 - with: - path: /tmp/docker-image.tar - key: ${{ inputs.app_name }}-docker-image-${{ github.sha }} - restore-keys: | - ${{ inputs.app_name }}-docker-image- - - - name: Load cached Docker image + - name: Build and tag Docker image for scanning + id: build-image run: | - docker load < /tmp/docker-image.tar + make APP_NAME=${{ inputs.app_name }} release-build + IMAGE_NAME=$(make APP_NAME=${{ inputs.app_name }} release-image-name) + IMAGE_TAG=$(make release-image-tag) + echo "image=$IMAGE_NAME:$IMAGE_TAG" >> "$GITHUB_OUTPUT" - name: Run Anchore vulnerability scan uses: anchore/scan-action@v3 with: - image: ${{ needs.build-and-cache.outputs.image }} + image: ${{ steps.build-image.outputs.image }} output-format: table - name: Save output to workflow summary if: always() # Runs even if there is a failure - run: | - echo "View results in GitHub Action logs" >> "$GITHUB_STEP_SUMMARY" + run: echo "View results in GitHub Action logs" >> "$GITHUB_STEP_SUMMARY" dockle-scan: - name: Dockle Scan runs-on: ubuntu-latest - needs: build-and-cache steps: - uses: actions/checkout@v3 - - name: Restore cached Docker image - uses: actions/cache/restore@v3 - with: - path: /tmp/docker-image.tar - key: ${{ inputs.app_name }}-docker-image-${{ github.sha }} - restore-keys: | - ${{ inputs.app_name }}-docker-image- - - - name: Load cached Docker image + - name: Build and tag Docker image for scanning + id: build-image run: | - docker load < /tmp/docker-image.tar + make APP_NAME=${{ inputs.app_name }} release-build + IMAGE_NAME=$(make APP_NAME=${{ inputs.app_name }} release-image-name) + IMAGE_TAG=$(make release-image-tag) + echo "image=$IMAGE_NAME:$IMAGE_TAG" >> "$GITHUB_OUTPUT" # Dockle doesn't allow you to have an ignore file for the DOCKLE_ACCEPT_FILES # variable, this will save the variable in this file to env for Dockle @@ -184,7 +112,7 @@ jobs: - name: Run Dockle container linter uses: erzz/dockle-action@v1.3.1 with: - image: ${{ needs.build-and-cache.outputs.image }} + image: ${{ steps.build-image.outputs.image }} exit-code: "1" failure-threshold: WARN accept-filenames: ${{ env.DOCKLE_ACCEPT_FILES }} diff --git a/.grype.yml b/.grype.yml index a6c539b00..9fe419a08 100644 --- a/.grype.yml +++ b/.grype.yml @@ -18,13 +18,3 @@ ignore: - fix-state: unknown # https://github.com/anchore/grype/issues/1172 - vulnerability: GHSA-xqr8-7jwr-rhp7 - - vulnerability: GHSA-7fh5-64p2-3v2j - # pip vulnerability, need to wait for the Python image to update to 23.x - # https://github.com/docker-library/python/blob/402b993af9ca7a5ee22d8ecccaa6197bfb957bc5/3.12/slim-bookworm/Dockerfile#L137 - - vulnerability: GHSA-mq26-g339-26xf - # 11/14/2023 - Postgres vulnerabilities in the Debian image - - vulnerability: CVE-2023-39417 - - vulnerability: CVE-2023-5869 - - vulnerability: CVE-2023-39418 - - vulnerability: CVE-2023-5868 - - vulnerability: CVE-2023-5870 diff --git a/.hadolint.yaml b/.hadolint.yaml index 00f76a4a3..d552e3548 100644 --- a/.hadolint.yaml +++ b/.hadolint.yaml @@ -4,9 +4,3 @@ # https://github.com/hadolint/hadolint#configure failure-threshold: warning ignored: [] -override: - info: - # Casts the apt-get install = finding as info - # We have this set since there is no way to specify version for - # build-essentials in the Dockerfile - - DL3008 diff --git a/.template-version b/.template-version index 84dfada2b..d4be0fab8 100644 --- a/.template-version +++ b/.template-version @@ -1 +1 @@ -fe5c7cd24d3c2c9f15c342826cda0a20af4cd0a5 +649f17792033fd7cf917a1931d23ab7041213c9c diff --git a/.trivyignore b/.trivyignore index 1fa436529..707b3e986 100644 --- a/.trivyignore +++ b/.trivyignore @@ -6,11 +6,4 @@ # Link to the dependencies for ease of checking for updates # Issue: Why there is a finding and why this is here or not been removed # Last checked: Date last checked in scans -#The-CVE-or-vuln-id # Remove comment at start of line -CVE-2023-5363 -# 11/14/2023 - Postgres vulnerabilities in the Debian image -CVE-2023-39417 -CVE-2023-5869 -CVE-2023-39418 -CVE-2023-5868 -CVE-2023-5870 \ No newline at end of file +#The-CVE-or-vuln-id # Remove comment at start of line \ No newline at end of file diff --git a/Makefile b/Makefile index 4dbe93cc1..90f240c7a 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,7 @@ __check_defined = \ infra-update-current-account \ infra-update-network \ infra-validate-modules \ + lint-markdown \ release-build \ release-deploy \ release-image-name \ @@ -64,8 +65,9 @@ infra-set-up-account: ## Configure and create resources for current AWS profile @:$(call check_defined, ACCOUNT_NAME, human readable name for account e.g. "prod" or the AWS account alias) ./bin/set-up-current-account.sh $(ACCOUNT_NAME) -infra-configure-network: ## Configure default network - ./bin/create-tfbackend.sh infra/networks default +infra-configure-network: ## Configure network $NETWORK_NAME + @:$(call check_defined, NETWORK_NAME, the name of the network in /infra/networks) + ./bin/create-tfbackend.sh infra/networks $(NETWORK_NAME) infra-configure-app-build-repository: ## Configure infra/$APP_NAME/build-repository tfbackend and tfvars files @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) @@ -90,8 +92,10 @@ infra-configure-app-service: ## Configure infra/$APP_NAME/service module's tfbac infra-update-current-account: ## Update infra resources for current AWS profile ./bin/terraform-init-and-apply.sh infra/accounts `./bin/current-account-config-name.sh` -infra-update-network: ## Update default network - ./bin/terraform-init-and-apply.sh infra/networks default +infra-update-network: ## Update network + @:$(call check_defined, NETWORK_NAME, the name of the network in /infra/networks) + terraform -chdir="infra/networks" init -input=false -reconfigure -backend-config="$(NETWORK_NAME).s3.tfbackend" + terraform -chdir="infra/networks" apply -var="network_name=$(NETWORK_NAME)" infra-update-app-build-repository: ## Create or update $APP_NAME's build repository @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) @@ -109,7 +113,6 @@ infra-update-app-database-roles: ## Create or update database roles and schemas ./bin/create-or-update-database-roles.sh $(APP_NAME) $(ENVIRONMENT) infra-update-app-service: ## Create or update $APP_NAME's web service module - # APP_NAME has a default value defined above, but check anyways in case the default is ever removed @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) @:$(call check_defined, ENVIRONMENT, the name of the application environment e.g. "prod" or "staging") terraform -chdir="infra/$(APP_NAME)/service" init -input=false -reconfigure -backend-config="$(ENVIRONMENT).s3.tfbackend" @@ -137,7 +140,7 @@ infra-check-compliance-checkov: ## Run checkov compliance checks infra-check-compliance-tfsec: ## Run tfsec compliance checks tfsec infra -infra-lint: infra-lint-scripts infra-lint-terraform infra-lint-workflows ## Lint infra code +infra-lint: lint-markdown infra-lint-scripts infra-lint-terraform infra-lint-workflows ## Lint infra code infra-lint-scripts: ## Lint shell scripts shellcheck bin/** @@ -152,8 +155,10 @@ infra-format: ## Format infra code terraform fmt -recursive infra infra-test-service: ## Run service layer infra test suite - @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) - cd infra/test && go test -run TestService -v -timeout 30m -app_name=$(APP_NAME) + cd infra/test && go test -run TestService -v -timeout 30m + +lint-markdown: ## Lint Markdown docs for broken links + ./bin/lint-markdown.sh ######################## ## Release Management ## @@ -180,7 +185,7 @@ INFO_TAG := $(DATE).$(USER) release-build: ## Build release for $APP_NAME and tag it with current git hash @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) cd $(APP_NAME) && $(MAKE) release-build \ - OPTS="--tag $(IMAGE_NAME):latest --tag $(IMAGE_NAME):$(IMAGE_TAG) --load -t $(IMAGE_NAME):$(IMAGE_TAG) $(OPTIONAL_BUILD_FLAGS)" + OPTS="--tag $(IMAGE_NAME):latest --tag $(IMAGE_NAME):$(IMAGE_TAG)" release-publish: ## Publish release to $APP_NAME's build repository @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) diff --git a/bin/account-ids-by-name.sh b/bin/account-ids-by-name.sh index b5da95dd5..a12df150d 100755 --- a/bin/account-ids-by-name.sh +++ b/bin/account-ids-by-name.sh @@ -13,7 +13,7 @@ SCRIPT_DIR=$(dirname "$0") KEY_VALUE_PAIRS=() BACKEND_CONFIG_FILE_PATHS=$(ls -1 "$SCRIPT_DIR"/../infra/accounts/*.*.s3.tfbackend) -for BACKEND_CONFIG_FILE_PATH in $BACKEND_CONFIG_FILE_PATHS; do +for BACKEND_CONFIG_FILE_PATH in $BACKEND_CONFIG_FILE_PATHS; do BACKEND_CONFIG_FILE=$(basename "$BACKEND_CONFIG_FILE_PATH") BACKEND_CONFIG_NAME="${BACKEND_CONFIG_FILE/.s3.tfbackend/}" IFS='.' read -r ACCOUNT_NAME ACCOUNT_ID <<< "$BACKEND_CONFIG_NAME" diff --git a/bin/check-github-actions-auth.sh b/bin/check-github-actions-auth.sh index e9ce94f31..2c5ec6953 100755 --- a/bin/check-github-actions-auth.sh +++ b/bin/check-github-actions-auth.sh @@ -23,7 +23,7 @@ echo "Get workflow run id" # The current implementation involves getting the create time of the previous # run. Then continuously checking the list of workflow runs until we see a # newly created run. Then we get the id of this new run. -# +# # References: # * This stack overflow article suggests a complicated overengineered approach: # https://stackoverflow.com/questions/69479400/get-run-id-after-triggering-a-github-workflow-dispatch-event @@ -33,7 +33,7 @@ echo "Get workflow run id" echo "Previous workflow run created at $PREV_RUN_CREATE_TIME" echo "Check workflow run create time until we find a newer workflow run" while : ; do - echo -n "." + echo -n "." RUN_CREATE_TIME=$(gh run list --workflow check-infra-auth.yml --limit 1 --json createdAt --jq ".[].createdAt") [[ $RUN_CREATE_TIME > $PREV_RUN_CREATE_TIME ]] && break done diff --git a/bin/configure-monitoring-secret.sh b/bin/configure-monitoring-secret.sh index 8df5431fc..b91a84f79 100755 --- a/bin/configure-monitoring-secret.sh +++ b/bin/configure-monitoring-secret.sh @@ -1,13 +1,13 @@ #!/bin/bash # ----------------------------------------------------------------------------- # This script creates SSM parameter for storing integration URL for incident management -# services. Script creates new SSM attribute or updates existing. +# services. Script creates new SSM attribute or updates existing. # # Positional parameters: # APP_NAME (required) – the name of subdirectory of /infra that holds the # application's infrastructure code. # ENVIRONMENT is the name of the application environment (e.g. dev, staging, prod) -# INTEGRATION_ENDPOINT_URL is the url for the integration endpoint for external +# INTEGRATION_ENDPOINT_URL is the url for the integration endpoint for external # incident management services (e.g. Pagerduty, Splunk-On-Call) # ----------------------------------------------------------------------------- set -euo pipefail @@ -32,7 +32,7 @@ echo "Setting up SSM secret" echo "=====================" echo "APPLICATION_NAME=$APP_NAME" echo "ENVIRONMENT=$ENVIRONMENT" -echo "INTEGRATION_URL=$INTEGRATION_ENDPOINT_URL" +echo "INTEGRATION_URL=$INTEGRATION_ENDPOINT_URL" echo echo "Creating SSM secret: $SECRET_NAME" @@ -41,3 +41,4 @@ aws ssm put-parameter \ --value "$INTEGRATION_ENDPOINT_URL" \ --type SecureString \ --overwrite + diff --git a/bin/lint-markdown.sh b/bin/lint-markdown.sh new file mode 100755 index 000000000..d73fc08d4 --- /dev/null +++ b/bin/lint-markdown.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# To make things simpler, ensure we're in the repo's root directory (one directory up) before +# running, regardless where the user is when invoking this script. + +# Grab the full directory name for where this script lives. +SCRIPT_DIR=$(readlink -f "$0" | xargs dirname) + +# Move up to the root since we want to do everything relative to that. Note that this only impacts +# this script, but will leave the user wherever they were when the script exists. +cd "${SCRIPT_DIR}/.." >/dev/null || exit 1 + + +LINK_CHECK_CONFIG=".github/workflows/markdownlint-config.json" + +# Recursively find all markdown files (*.md) in this directory. Pass them in as args to the lint +# command using the handy `xargs` command. +find . -name \*.md -print0 | xargs -0 -n1 npx markdown-link-check --config $LINK_CHECK_CONFIG diff --git a/bin/run-command.sh b/bin/run-command.sh index 6c0161748..304ff505d 100755 --- a/bin/run-command.sh +++ b/bin/run-command.sh @@ -1,7 +1,7 @@ #!/bin/bash # ----------------------------------------------------------------------------- # Run an application command using the application image -# +# # Optional parameters: # --environment-variables - a JSON list of environment variables to add to the # the container. Each environment variable is an object with the "name" key diff --git a/bin/set-up-current-account.sh b/bin/set-up-current-account.sh index bea3469c8..b253c0fe9 100755 --- a/bin/set-up-current-account.sh +++ b/bin/set-up-current-account.sh @@ -52,11 +52,12 @@ echo "Creating bucket: $TF_STATE_BUCKET_NAME" # For creating buckets outside of us-east-1, a LocationConstraint needs to be set # For creating buckets in us-east-1, LocationConstraint cannot be set # See https://docs.aws.amazon.com/cli/latest/reference/s3api/create-bucket.html -CREATE_BUCKET_CONFIGURATION="" +CREATE_BUCKET_CONFIGURATION=("") if [ "$REGION" != "us-east-1" ]; then - CREATE_BUCKET_CONFIGURATION="--create-bucket-configuration LocationConstraint=$REGION" + CREATE_BUCKET_CONFIGURATION=("--create-bucket-configuration" "LocationConstraint=$REGION") fi -aws s3api create-bucket --bucket "$TF_STATE_BUCKET_NAME" --region "$REGION" "$CREATE_BUCKET_CONFIGURATION" > /dev/null + +aws s3api create-bucket --bucket "$TF_STATE_BUCKET_NAME" --region "$REGION" "${CREATE_BUCKET_CONFIGURATION[@]}" > /dev/null echo echo "----------------------------------" echo "Creating rest of account resources" @@ -65,6 +66,21 @@ echo cd infra/accounts +# Create the OpenID Connect provider for GitHub Actions to allow GitHub Actions +# to authenticate with AWS and manage AWS resources. We create the OIDC provider +# via AWS CLI rather than via Terraform because we need to first check if there +# is already an existing OpenID Connect provider for GitHub Actions. This check +# is needed since there can only be one OpenID Connect provider per URL per AWS +# account. +github_arn=$(aws iam list-open-id-connect-providers | jq -r ".[] | .[] | .Arn" | grep github || echo "") + +if [[ -z ${github_arn} ]]; then + aws iam create-open-id-connect-provider \ + --url "https://token.actions.githubusercontent.com" \ + --client-id-list "sts.amazonaws.com" \ + --thumbprint-list "0000000000000000000000000000000000000000" +fi + # Create the infrastructure for the terraform backend such as the S3 bucket # for storing tfstate files and the DynamoDB table for tfstate locks. # -reconfigure is used in case this isn't the first account being set up diff --git a/docs/code-reviews.md b/docs/code-reviews.md new file mode 100644 index 000000000..611135d2d --- /dev/null +++ b/docs/code-reviews.md @@ -0,0 +1,55 @@ +# Code Reviews + +Code reviews are intended to help all of us grow as engineers and improve the quality of what we ship. +These guidelines are meant to reinforce those two goals. + +## For reviewers + +Aim to respond to code reviews within 1 business day. + +Remember to highlight things that you like and appreciate while reading through the changes, +and to make any other feedback clearly actionable by indicating if it is optional preference, an important consideration, or an error. + +Don't be afraid to comment with a question, or to ask for clarification, or provide a suggestion, +whenever you don’t understand what is going on at first glance — or if you think an approach or decision can be improved. +Suggestions on how to split a large PR into smaller chunks can also help move things along. +Code reviews give us a chance to learn from one another, and to reflect, iterate on, and document why certain decisions are made. + +Once you're ready to approve or request changes, err on the side of trust. +Send a vote of approval if the PR looks ready except for small minor changes, +and trust that the recipient will address your comments before merging by replying via comment or code to any asks. +Use "request changes" sparingly, unless there's a blocking issue or major refactors that should be done. + +## For authors or requesters + +Your PR should be small enough that a reviewer can reasonably respond within 1-2 business days. +For larger changes, break them down into a series of PRs. +If refactors are included in your changes, try to split them out into separate PRs. + +As a PR writer, you should consider your description and comments as documentation; +current and future team members will refer to it to understand your design decisions. +Include relevant context and business requirements, and add preemptive comments (in code or PR) +for sections of code that may be confusing or worth debate. + +### Draft PRs + +If your PR is a work-in-progress, or if you are looking for specific feedback on things, +create a [Draft Pull Request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests#draft-pull-requests) +and state what you are looking for in the description. + +### Re-requesting reviews after completing changes + +After you make requested changes in response to code review feedback, please re-request reviews from the reviewers to notify them that the work is ready to be reviewed again. + +## Advantages of code review + +- catch and prevent bugs +- consistent code +- find shared code +- share knowledge + +## Challenges of code reviews +- it can take long +- who to ask +- how do you know when is "enough" review +- what should i be reviewing diff --git a/docs/compliance.md b/docs/compliance.md new file mode 100644 index 000000000..db9a92b87 --- /dev/null +++ b/docs/compliance.md @@ -0,0 +1,29 @@ +# Compliance + +We use [Checkov](https://www.checkov.io/) and [tfsec](https://aquasecurity.github.io/tfsec/) static analysis tools to check for compliance with infrastructure policies. + +## Setup + +To run these tool locally, first install them by running the following commands. + +* Install checkov + + ```bash + brew install checkov + ``` + +* Install tfsec + + ```bash + brew install tfsec + ``` + +## Check compliance + +```bash +make infra-check-compliance +``` + +## Pre-Commit + +If you use [pre-commit](https://www.checkov.io/4.Integrations/pre-commit.html), you can optionally add checkov to your own pre-commit hook by following the instructions [here](https://www.checkov.io/4.Integrations/pre-commit.html). diff --git a/docs/decisions/index.md b/docs/decisions/index.md new file mode 100644 index 000000000..8b8a9ea1b --- /dev/null +++ b/docs/decisions/index.md @@ -0,0 +1,22 @@ +# Architectural Decision Log + +This log lists the architectural decisions for [project name]. + + + +* [ADR-0000](infra/0000-use-markdown-architectural-decision-records.md) - Use Markdown Architectural Decision Records +* [ADR-0001](infra/0001-ci-cd-interface.md) - CI/CD Interface +* [ADR-0002](infra/0002-use-custom-implementation-of-github-oidc.md) - Use custom implementation of GitHub OIDC to authenticate GitHub actions with AWS rather than using module in Terraform Registry +* [ADR-0003](infra/0003-manage-ecr-in-prod-account-module.md) - Manage ECR in prod account module +* [ADR-0004](infra/0004-separate-terraform-backend-configs-into-separate-config-files.md) - Separate tfbackend configs into separate files +* [ADR-0005](infra/0005-separate-database-infrastructure-into-separate-layer.md) - Separate the database infrastructure into a separate layer +* [ADR-0006](infra/0006-provision-database-users-with-serverless-function.md) - Provision database users with serverless function +* [ADR-0007](infra/0007-database-migration-architecture.md) - Database Migration Infrastructure and Deployment +* [ADR-0008](infra/0008-consolidate-infra-config-from-tfvars-files-into-config-module.md) - Consolidate infra configuration from .tfvars files into config module +* [ADR-0009](infra/0009-separate-app-infrastructure-into-layers.md) - Separate app infrastructure into layeres + + + +For new ADRs, please use [template.md](template.md) as basis. +More information on MADR is available at . +General information about architectural decision records is available at . diff --git a/docs/decisions/infra/0000-use-markdown-architectural-decision-records.md b/docs/decisions/infra/0000-use-markdown-architectural-decision-records.md new file mode 100644 index 000000000..1aab9e567 --- /dev/null +++ b/docs/decisions/infra/0000-use-markdown-architectural-decision-records.md @@ -0,0 +1,26 @@ +# Use Markdown Architectural Decision Records + +## Context and Problem Statement + +We want to record architectural decisions made in this project. +Which format and structure should these records follow? + +## Considered Options + +* [MADR](https://adr.github.io/madr/) 2.1.2 – The Markdown Architectural Decision Records +* [Michael Nygard's template](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions) – The first incarnation of the term "ADR" +* [Sustainable Architectural Decisions](https://www.infoq.com/articles/sustainable-architectural-design-decisions) – The Y-Statements +* Other templates listed at +* Formless – No conventions for file format and structure + +## Decision Outcome + +Chosen option: "MADR 2.1.2", because + +* Implicit assumptions should be made explicit. + Design documentation is important to enable people understanding the decisions later on. + See also [A rational design process: How and why to fake it](https://doi.org/10.1109/TSE.1986.6312940). +* The MADR format is lean and fits our development style. +* The MADR structure is comprehensible and facilitates usage & maintenance. +* The MADR project is vivid. +* Version 2.1.2 is the latest one available when starting to document ADRs. diff --git a/docs/decisions/infra/0001-ci-cd-interface.md b/docs/decisions/infra/0001-ci-cd-interface.md new file mode 100644 index 000000000..1a4bc213d --- /dev/null +++ b/docs/decisions/infra/0001-ci-cd-interface.md @@ -0,0 +1,113 @@ +# CI/CD Interface + +* Status: accepted +* Deciders: @lorenyu @kyeah +* Date: 2022-10-04 + +Technical Story: Define Makefile interface between infra and application [#105](https://github.com/navapbc/template-infra/issues/105) + +## Context and Problem Statement + +In order to reuse CI and CD logic for different tech stacks, we need to establish a consistent interface by which different applications can hook into the common CI/CD infrastructure. + +## Decision Drivers + +* We want to define most of the release management logic in `template-infra` but allow application specific methods for building the release. +* The build needs to be able to be run from the CD workflow defined in `template-infra`, but it also needs to be able to be run from the application as part of the CI workflow as one of the CI checks. + +## Proposal + +### CD interface + +Create a `Makefile` in `template-infra` repo that defines the following make targets: + +```makefile +################### +# Building and deploying +################## + +# Generate an informational tag so we can see where every image comes from. +release-build: # assumes there is a Dockerfile in `app` folder + ... code that builds image from app/Dockerfile + +release-publish: + ... code that publishes to ecr + +release-deploy: + ... code that restarts ecs service with new image +``` + +Each of the template applications (template-application-nextjs, template-application-flask) needs to have a `Makefile` in `app/` e.g. `template-application-flask/app/Makefile` with a `release-build` target that builds the release image. The `release-build` target should take an `OPTS` argument to pass into the build command to allow the parent Makefile to pass in arguments like `--tag IMAGE_NAME:IMAGE_TAG` which can facilitate release management. + +```makefile +# template-application-flask/app/Makefile + +release-build: + docker build $(OPTS) --target release . +``` + +By convention, the application's Dockerfile should have a named stage called `release` e.g. + +```Dockerfile +# template-application-flask/app/Dockerfile +... +FROM scratch AS release +... +``` + +### CI interface + +Each application will have their own CI workflow that gets copied into the project's workflows folder as part of installation. `template-application-nextjs` and `template-application-flask` will have `.github/workflows/ci-app.yml`, and `template-infra` will have `.github/workflows/ci-infra.yml`. + +Installation would look something like: + +```bash +cp template-infra/.github/workflows/* .github/workflows/ +cp template-application-nextjs/.github/workflows/* .github/workflows/ +``` + +CI in `template-application-next` might be something like: + +```yml +# template-application-nextjs/.github/workflows/ci-app.yml + +jobs: + lint: + steps: + - run: npm run lint + type-check: + steps: + - run: npm run type-check + test: + steps: + - run: npm test +``` + +CI in `template-application-flask` might be something like: + +```yml +# template-application-nextjs/.github/workflows/ci-app.yml + +jobs: + lint: + steps: + - run: poetry run black + type-check: + steps: + - run: poetry run mypy + test: + steps: + - run: poetry run pytest +``` + +For now we are assuming there's only one deployable application service per repo, but we could evolve this architecture to have the project rename `app` as part of the installation process to something specific like `api` or `web`, and rename `ci-app.yml` appropriately to `ci-api.yml` or `ci-web.yml`, which would allow for multiple application folders to co-exist. + +## Alternative options considered for CD interface + +1. Application template repos also have their own release-build command (could use Make, but doesn't have to) that is called as part of the application's ci-app.yml. The application's version of release-build doesn't have to tag the release, since the template-infra version will do that: + + * Cons: build command in two places, and while 99% of the build logic is within Dockerfile and code, there's still a small chance that difference in build command line arguments could produce a different build in CI than what is used for release + +2. We can run release-build as part of template-infra's ci-infra.yml, so we still get CI test coverage of build process + + * Cons: things like tests and linting in ci-app.yml can't use the docker image to run the tests, which potentially means CI and production are using slightly different environments diff --git a/docs/decisions/infra/0002-use-custom-implementation-of-github-oidc.md b/docs/decisions/infra/0002-use-custom-implementation-of-github-oidc.md new file mode 100644 index 000000000..b9c5a5bfe --- /dev/null +++ b/docs/decisions/infra/0002-use-custom-implementation-of-github-oidc.md @@ -0,0 +1,38 @@ +# Use custom implementation of GitHub OIDC to authenticate GitHub actions with AWS rather than using module in Terraform Registry + +* Status: accepted +* Deciders: @shawnvanderjagt @lorenyu @NavaTim +* Date: 2022-10-05 (Updated 2023-07-12) + +## Context and Problem Statement + +[GitHub recommends using OpenID Connect to authenticate GitHub actions with AWS](https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect). There are [existing modules in the Terraform Registry](https://registry.terraform.io/search/modules?q=github%20actions%20oidc) that implements these resources. Should we use an existing module or implement our own? + +## Decision Drivers + +* Secure +* Maintainable +* Simple and easily understood + +## Considered Options + +* Use [unfunco/oidc-github](https://registry.terraform.io/modules/unfunco/oidc-github/aws/latest) module from Terraform registry +* Use a fork of [unfunco/oidc-github](https://registry.terraform.io/modules/unfunco/oidc-github/aws/latest) in [NavaPBC GitHub org](https://github.com/navapbc) +* Use a custom implementation + +## Decision Outcome + +We chose to use a custom implementation because it allowed for the simplest implementation that was easiest to understand while still being in our full control and therefore avoids security issues with external dependencies. It is also easy to upgrade to use the external module if circumstances change. + +## Pros and Cons of the Options + +The [unfunco/oidc-github](https://registry.terraform.io/modules/unfunco/oidc-github/aws/latest) module from Terraform registry is effectively what we need, but there are a few disadvantages to using it: + +Cons of unfunco/oidc-github: + +* Dependency on an external module in the Terraform registry has negative security implications. Furthermore, the module isn't published by an "official" organization. It is maintained by a single developer, further increasing the security risk. +* The module includes extra unnecessary options that make the code more difficult to read and understand +* In particular, the module includes the option to attach the `AdminstratorAccess` policy to the GitHub actions IAM role, which isn't necessary and could raise concerns in an audit. +* ~~The module hardcodes the GitHub OIDC Provider thumbprint, which isn't as elegant as the method in the [Initial setup for CD draft PR #43](https://github.com/navapbc/template-infra/pull/43) from @shawnvanderjagt which simply pulls the thumbprint via:~~ (Update: July 12, 2023) Starting July 6, 2023, AWS began securing communication with GitHub’s OIDC identity provider (IdP) using GitHub's library of trusted root Certificate Authorities instead of using a certificate thumbprint to verify the IdP’s server certificate. This approach ensures that the GitHub OIDC configuration behaves correctly without disruption during future certificate rotations and changes. With this new validation approach in place, your legacy thumbprint(s) are longer be needed for validation purposes. + +Forking the module to the navapbc organization gets rid of the security issue, but the other issues remain. diff --git a/docs/decisions/infra/0003-manage-ecr-in-prod-account-module.md b/docs/decisions/infra/0003-manage-ecr-in-prod-account-module.md new file mode 100644 index 000000000..4af6dd105 --- /dev/null +++ b/docs/decisions/infra/0003-manage-ecr-in-prod-account-module.md @@ -0,0 +1,33 @@ +# Manage ECR in prod account module + +* Status: accepted +* Deciders: @lorenyu @shawnvanderjagt @farrcraft @kyeah +* Date: 2022-10-07 + +## Context and Problem Statement + +In a multi-account setup where there is one account per environment, where should the ECR repository live? + +## Decision Drivers + +* Minimize risk that the approach isn't acceptable with the agency given uncertainty around ability to provision accounts with the agency +* Desire an approach that can adapt equally well to a multi-account setup (with an account per environment) as well as to a single-account setup (with one account across all environments) or a two-account setup (with one account for prod and an account for non-prod) +* Desire an approach that can adapt to situations where there is more than one ECR repository i.e. a project with multiple deployable applications +* Simplicity + +## Considered Options + +* Separate `dist`/`build` account to contain the ECR repository and build artifacts +* Manage the ECR repository as part of the `prod` account +* Manage the ECR repository as part of the `dev` or `stage` account + +## Decision Outcome + +Manage the ECR repository(ies) as part of the prod account module, or for single-account setups, the single account module. Since there will always be a prod account, this approach should have minimal risk for not working for the agency, and will also work for projects that only have or need a single account. + +## Discussion of alternative approach + +However, if account management and creation was not an issue, it could be more elegant to have the production candidate build artifacts be managed in a separate `build` account that all environment accounts reference. This approach is described in the following references: + +* [Medium article: Cross-Account Amazon Elastic Container Registry (ECR) Access for ECS](https://garystafford.medium.com/amazon-elastic-container-registry-ecr-cross-account-access-for-ecs-2f90fcb02c80) +* [AWS whitepaper - Recommended Accounts - Deployments](https://docs.aws.amazon.com/whitepapers/latest/organizing-your-aws-environment/deployments-ou.html) diff --git a/docs/decisions/infra/0004-separate-terraform-backend-configs-into-separate-config-files.md b/docs/decisions/infra/0004-separate-terraform-backend-configs-into-separate-config-files.md new file mode 100644 index 000000000..2923c42c3 --- /dev/null +++ b/docs/decisions/infra/0004-separate-terraform-backend-configs-into-separate-config-files.md @@ -0,0 +1,36 @@ +# Separate tfbackend configs into separate files + +* Status: accepted +* Deciders: @lorenyu @shawnvanderjagt @kyeah @bneutra +* Date: 2023-05-09 + +## Context + +Up until now, most projects adopted an infrastructure module architecture that is structured as follows: Each application environment (prod, staging, etc) is a separate root module that calls a template module. The template module defines all the application infra resources needed for an environment. Things that could be different per environment (e.g. desired ECS task count) are template variables, and each environment can have local vars (or somewhat equivalently, a tfvars file) that customizes those variables. Importantly, each environment has it’s own backend tfstate file, and the backend config is stored in the environment module’s `main.tf`. + +An alternative approach exists to managing the backend configs. Rather than saving the backend config directly in main.tf, `main.tf` could contain a [partial configuration](https://developer.hashicorp.com/terraform/language/settings/backends/configuration#partial-configuration), and the rest of the backend config would be passed in during terraform init with a command like `terraform init --backend-config=prod.s3.tfbackend`. There would no longer be a need for separate root modules for each environment. What was previously the template module would instead act as the root module, and engineers would work with different environments solely through separate tfbackend files and tfvar files. Doing this would greatly simplify the module architecture at the cost of some complexity when executing terraform commands due to the extra command line parameters. To manage the extra complexity of running terraform commands, a wrapper script (such as with Makefile commands) can be introduced. + +The approach can be further extended to per-environment variable configurations via an analogous approach with [variable definitions files](https://developer.hashicorp.com/terraform/language/values/variables#variable-definitions-tfvars-files) which can be passed in with the `-var-file` command line option to terraform commands. + +## Notes + +For creating accounts, can't use the .tfbackend backend config file approach because the main.tf file can only have one backend configuration, so if we have the backend configuration as a partial configuration of `backend "s3" {}`, then we can't use that same module to configure a new account, since the process for configuring a new account +requires setting the backend configuration to `backend "local" {}`. We could have a separate duplicate module that's has backend set to local. or we could also temporarily update the backend from `"s3"` to `"local"`, but both of those approaches seem confusing. + +Another alternative is to go back to the old way of bootstrapping an account i.e. to do it via a script that creates an S3 bucket via AWS CLI. The bootstrap script would only do the minimal configuration for the S3 bucket, and let terraform handle the remainder of the configuration, such as creating the dynamodb tables. At this point, there is no risk of not having state locking in place since the account infrastructure has not yet been checked into the repository. This might be the cleanest way to have accounts follow the same pattern of using tfbackend config files. + +## Benefits of separate tfvars and tfbackend files + +* **Reduce risk of differences between environments** – When different environments have their own root modules, development teams have historically sometimes added one-off resources to specific environments without adding those resources to the template module and without realizing that they're violating an important goal of having multiple environments – that environments are isolated from each other but function identically. This creates differences between environments that are more than just configuration differences. By forcing the differences to be limited to the `.tfvars` (-var) file, it limits how badly someone can get an environment out of skew. +* **DRY backend configuration** – With only a single module, there is less duplication of infrastructure code in the `main.tf` file. In particular, provider configurations, shared partial backend configuration, and certain other top level local variables and data resources no longer need to be duplicated across environments, and provider versions can also be forced to be consistent. +* **Make receiving updates from template-infra more robust** – Previously, in order for a project to receive updates from the template-infra repo, the project would copy over template files but then revert files that the project has changed. Currently, the many `main.tf` root module files in the template are expected to be changed by the project since they define project specific backend configurations. With the separation of config files, projects are no longer expected to change the `main.tf` files, so the `main.tf` files in `infra/app/build-repository/`, `infra/project-config/`, `infra/app/app-config/`, etc. can be safely copied over from template-infra without needing to be reverted. +* **Reduce the cost of introducing additional infrastructure layers** – In the future we may want too add new infrastructure layers that are created and updated independently of the application layer. Examples include a network layer or a database layer. We may want to keep them separate so that changes to the application infrastructure are isolated from changes to the database infrastructure, which should occur much less frequently. Previously, to add a new layer such as the database layer, we would need two additional folders: a `db-env-template` module and a `db-envs` folder with separate root modules for each environment. This mirrors the same structure that we have for the application. With separate backend config and tfvar files we would only need a single `db` module with separate `.tfbackend` and `.tfvars` files for each environment. + +## Cons of separate tfvars and tfbackend files + +* **Extra layer of abstraction** – The modules themselves aren't as simple to understand since the configuration is spread out across multiple files, the `main.tf` file and the corresponding `.tfvars` and `.tfbackend` file, rather than all in one `main.tf` file. +* **Requires additional parameters when running terraform** – Due to configuration being separated into `.tfvars` and `.tfbackend` files, terraform commands now require a `-var-file` and `-backend-config` command line options. The added complexity may require a wrapper script, introducing yet another layer of abstraction. + +## Links + +* Refined by [ADR-0008](./0008-consolidate-infra-config-from-tfvars-files-into-config-module.md) diff --git a/docs/decisions/infra/0005-separate-database-infrastructure-into-separate-layer.md b/docs/decisions/infra/0005-separate-database-infrastructure-into-separate-layer.md new file mode 100644 index 000000000..9157b75bc --- /dev/null +++ b/docs/decisions/infra/0005-separate-database-infrastructure-into-separate-layer.md @@ -0,0 +1,66 @@ +# Separate the database infrastructure into a separate layer + +* Status: proposed +* Deciders: @lorenyu @kyeah @shawnvanderjagt @rocketnova +* Date: 2023-05-25 + +## Context and Problem Statement + +On many projects, setting up the application and database is a multiple-step iterative process. The infrastructure team will first set up an application service without a database, with a simple application health check. The infrastructure team will then work on setting up the database, configuring the application service to have network access to the database cluster, configuring a the database user that the application will authenticate as and a database user that will run migrations, and providing a way for the application to authenticate. Then the application team will update the healthcheck to call the database. + +We want to design the template infrastructure so that each infrastructure layer can be configured and created once rather than needing to revisit prior layers. In other words, we'd like to be able to create the database layer, configure the database users, then create the application layer, without having to go back to make changes to database layer again. + +There are some dependencies to keep in mind: + +1. The creation of the application service layer depends on the creation of database layer, since a proper application healthcheck will need to hit the database. +2. The database layer includes the creation and configuring of the database users (i.e. PostgreSQL users) that will be used by the application and migration processes in addition to the database cluster infrastructure resources. +3. The network rule that allows inbound traffic to the database from the application depends on both the database and application service. + +## Decision Drivers + +* Avoid circular dependencies +* Avoid the need to revisit a layer (e.g. database layer, application layer) more than one time during setup of the application environment +* Keep things simple to understand and customize +* Minimize number of steps to set up an environment + +## Module Architecture Options + +* Option A: Put the database infrastructure in the same root module as the application service +* Option B: Separate the database infrastructure into a separate layer + +### Decision Outcome: Separate the database infrastructure into a separate layer + +Changes to database infrastructure are infrequent and therefore do not need to be incorporated as part of the continuous delivery process of deploying the application as it would needlessly slow down application deploys and also increase the risk of accidental changes to the database layer. When database changes are needed, they are sometimes complex due to the stateful nature of databases and can require multiple steps to make those changes gracefully. For these changes, it is beneficial to separate them from application resources so that application deploys can remain unaffected. Finally, breaking down the environment setup process into smaller, more linear steps – creating the database first before creating the application service – makes the environment setup process easier to understand and troubleshoot than trying to do create everything at once. + +The biggest disadvantage to this approach is ~~the fact that dependencies between root modules cannot be directly expressed in terraform. To mitigate this problem, we should carefully design the interface between root modules to minimize breaking changes in that interface.~~ (Update: 2023-07-07) that dependencies between root modules become more indirect and difficult to express. See [module dependencies](/docs/infra/module-dependencies.md) + +## Pros and Cons of the Options + +### Option A: Put the database infrastructure in the same root module as the application service + +Pros: + +* This is what we've typically done in the past. All the infrastructure necessary for the application environment would live in a single root module, with the exception of shared resources like the ECR image repository. + +Cons: + +* The application service's healthcheck depends on the database cluster to be created and the database user to be provisioned. This cannot easily be done in a single terraform apply. +* Changes to the database infrastructure are often more complex than changes to application infrastructure. Unlike application infrastructure, database changes cannot take the approach of spinning up new infrastructure in desired configuration, redirecting traffic to new infrastructure, then destroying old infrastructure. This is because application infrastructure can be designed to be stateless while databases are inherently stateful. In such cases, making database changes may require careful coordination and block changes to the application infrastructure, potentially including blocking deploys, while the database changes are made. + +### Option B: Separate the database infrastructure into a separate layer + +Pros: + +* Separating the database layer makes explicit the dependency between the database and the application service, and enables an environment setup process that involves only creating resources when all dependencies have been created first. +* Application deploys do not require making requests to the database infrastructure. +* Complex database changes that require multiple steps can be made without negatively impacting application deploys. +* Not all applications require a database. Having the database layer separate reduces the amount of customization needed at the application layer for different systems. + +Cons: + +* Application resources for a single environment are split across multiple root modules +* Dependencies between root modules cannot be expressed directly in terraform to use terraform's built-in dependency graph. ~~Instead, dependencies between root modules need to be configured from one module's outputs to another module's variable definitions file~~ (Update: 2023-07-07) Instead, dependencies between root modules need to leverage terraform data sources to reference resources across root modules and need to use a shared config module to reference the parameters that can uniquely identify the resource. See [module dependencies](/docs/infra/module-dependencies.md) + +## Links + +* Refined by [ADR-0009](./0009-separate-app-infrastructure-into-layers.md) diff --git a/docs/decisions/infra/0006-provision-database-users-with-serverless-function.md b/docs/decisions/infra/0006-provision-database-users-with-serverless-function.md new file mode 100644 index 000000000..c885f6889 --- /dev/null +++ b/docs/decisions/infra/0006-provision-database-users-with-serverless-function.md @@ -0,0 +1,87 @@ +# Provision database users with serverless function + +* Status: proposed +* Deciders: @lorenyu @kyeah @shawnvanderjagt @rocketnova +* Date: 2023-05-25 + +## Context and Problem Statement + +What is the best method for setting up database users and permissions for the application service and the migrations task? + +## Decision Drivers + +* Minimize number of steps +* Security and compliance + +## Considered Options + +* **Terraform** – Define users and permissions declaratively in Terraform using the [PostgreSQL provider](https://registry.terraform.io/providers/cyrilgdn/postgresql/latest/docs). Apply changes from infrastructure engineer's local machine or from the CI/CD workflow. When initially creating database cluster, make database cluster publicly accessible and define security group rules to allow traffic from local machine or GitHub actions. After creating database users, reconfigure database cluster to make database private. +* **Shell scripts** – Define users and permissions through a shell script. This could use tools like psql or not. It could also define the permissions in a `.sql` file that gets executed. Similar to the terraform option, the database would need to be made accessible to the machine running the script. One way to do this is for the script itself to temporarily enable access to the database using AWS CLI. +* **Jump host using EC2** – Run terraform or a shell script but from an EC2 instance within the VPC. Create the EC2 instance and set up network connectivity between the EC2 instance and the database cluster as part of creating the database infrastructure resources. +* **Container task using ECS** – Build a Docker image that has the code and logic to provision users and permissions and run the code as an ECS task. +* **Serverless function using Lambda** – Write code to provision database users and permissions and run it as a Lambda function. + +### Decision Outcome: AWS Lambda function + +A Lambda function is the simplest tool that can operate within the VPC and therefore get around the obstacle of needing network access to the database cluster. EC2 instances are too expensive to maintain for rarely used operations like database user provisioning, and ECS tasks add complexity to the infrastructure by requiring an additional ECR image repository and image build step. + +## Pros and Cons of the Options + +### Terraform + +Pros + +* Declarative +* Could create database cluster and database users in a single terraform apply + +Cons + +* The database needs to be publicly accessible to the machine that is running the script – either the infrastructure engineer's local machine or the continuous integration service (e.g. GitHub Actions). First, this causes the database setup process to take a minimum of three steps: (1) create the database cluster with publicly accessible configuration, (2) provision the database users, (3) make the database cluster private. Second, even if it is an acceptable risk to make the database publicly accessible when it is first created and before it has any data, it may not be an acceptable risk to do so once the system is in production. Therefore, after the system is in production, there would no longer be a way to reconfigure the database users or otherwise maintain the system using this approach. +* Need to modify the database cluster configuration after creating it in order to make it private. Modifications requires an additional step, and may also require manual changes to the terraform configuration. + +### Shell scripts + +Pros + +* Simple +* Can represent user configuration as a `.sql` script which could simplify database management by keeping it all within SQL + +Cons + +* Same as the cons for Terraform – the database needs to be accessible to the machine running the script + +### Jump host using EC2 + +Pros + +* Can leverage the Terraform and Shell scripts approaches +* Can access the database securely from within the VPC without making the database cluster publicly accessible + +Cons + +* Added infrastructure complexity due to the need to maintain an EC2 instance + +### Container task using ECS + +Pros + +* Flexible: can build everything needed in a Docker container, including installing necessary binaries and bundling required libraries and code +* Can access the database securely from within the VPC without making the database cluster publicly accessible + +Cons + +* Increases complexity of terraform module architecture. There needs to be an ECR repository to store the Docker images. The ECR repository could be in a separate root module, which adds another layer to the module architecture. The ECR repository could be put in the `build-repository` root module, which would would clutter the `build-repository` since it's not related to application builds. Or it could be put in the `database` root module and be manually created using terraform's `-target` flag, but that adds complexity to the setup process. +* Increases number of steps needed to set up the database by at least two, one to create the ECR repository and one to build and publish the Docker image to the ECR repository, before creating the database cluster resources. + +### Serverless function using Lambda + +Pros + +* Flexible: can build many things in a Lambda function +* Can access the database securely from within the VPC without making the database cluster publicly accessible +* Relatively simple + +Cons + +* Adds a new dependency to the application setup process. The setup process will now rely on the programming language used by the Lambda function (Python in this case). +* Can't easily use custom external binaries in AWS Lambda. So will rely mostly on code rather than lower level scripts like psql. diff --git a/docs/decisions/infra/0007-database-migration-architecture.md b/docs/decisions/infra/0007-database-migration-architecture.md new file mode 100644 index 000000000..7a6f60327 --- /dev/null +++ b/docs/decisions/infra/0007-database-migration-architecture.md @@ -0,0 +1,92 @@ +# Database Migration Infrastructure and Deployment + +* Status: proposed +* Deciders: @lorenyu, @daphnegold, @chouinar, @Nava-JoshLong, @addywolf-nava, @sawyerh, @acouch, @SammySteiner +* Date: 2023-06-05 + +## Context and Problem Statement + +What is the most optimal setup for database migrations infrastructure and deployment? +This will break down the different options for how the migration is run, but not the +tools or languages the migration will be run with, that will be dependent on the framework the application is using. + +Questions that need to be addressed: + + 1. How will the method get the latest migration code to run? + 2. What infrastructure is required to use this method? + 3. How is the migration deployment re-run in case of errors? + +## Decision Drivers + +* Security +* Simplicity +* Flexibility + +## Considered Options + +* Run migrations from GitHub Actions +* Run migrations from a Lambda function +* Run migrations from an ECS task +* Run migrations from self-hosted GitHub Actions runners + +## Decision Outcome + +Run migrations from an ECS task using the same container image that is used for running the web service. Require a `db-migrate` script in the application container image that performs the migration. When running the migration task using [AWS CLI's run-task command](https://docs.aws.amazon.com/cli/latest/reference/ecs/run-task.html), use the `--overrides` option to override the command to the `db-migrate` command. + +Default to rolling forward instead of rolling back when issues arise (See [Pitfalls with SQL rollbacks and automated database deployments](https://octopus.com/blog/database-rollbacks-pitfalls)). Do not support rolling back out of the box, but still project teams to easily implement database rollbacks through the mechanism of running an application-specific database rollback script through a general purpose `run-command.sh` script. + +Pros + +* No changes to the database network configuration are needed. The database can remain inaccessible from the public internet. +* Database migrations are agnostic to the migration framework that the application uses as long as the application is able to provide a `db-migrate` script that is accessible from the container's PATH and is able to use IAM authentication for connecting to the database. Applications can use [alembic](https://alembic.sqlalchemy.org/), [flyway](https://flywaydb.org/), [prisma](https://www.prisma.io/), another migration framework, or custom built migrations. +* Database migrations use the same application image and task definition as the base application. + +Cons + +* Running migrations require doing a [targeted terraform apply](https://developer.hashicorp.com/terraform/tutorials/state/resource-targeting) to update the task definition without updating the service. Terraform recommends against targeting individual resources as part of a normal workflow. However, this is done to ensure migrations are run before the service is updated. + +## Other options considered + +### Run migrations from GitHub Actions using a direct database connection + +Temporarily update the database to be accessible from the internet and allow incoming network traffic from the GitHub Action runner's IP address. Then run the migrations directly from the GitHub Action runner. At the end, revert the database configuration changes. + +Pros: + +* Simple. Requires no additional infrastructure + +Cons: + +* This method requires temporarily exposing the database to incoming connections from the internet, which may not comply with agency security policies. + +### Run migrations from a Lambda function + +Run migrations from an AWS Lambda function that uses the application's container image. The application container image needs to [implement the lambda runtime api](https://aws.amazon.com/blogs/aws/new-for-aws-lambda-container-image-support/) either by using an AWS base image for Lambda or by implementing the Lambda runtime (see [Working with Lambda container images](https://docs.aws.amazon.com/lambda/latest/dg/images-create.html)). + +Pros: + +* Relatively simple. Lambdas are already used for managing database roles. +* The Lambda function can run from within the VPC, avoiding the need to expose the database to the public internet. +* The Lambda function is separate from the application service, so we avoid the need to modify the service's task definition. + +Cons: + +* Lambda function container images need to [implement the lambda runtime api](https://aws.amazon.com/blogs/aws/new-for-aws-lambda-container-image-support/). This is a complex application requirement that would significantly limit the ease of use of the infrastructure. +* Lambda functions have a maximum runtime of 15 minutes, which can limit certain kinds of migrations. + +### Run migrations from self-hosted GitHub Actions runners + +Then run the migrations directly from a [self-hosted GitHub Action runner](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners). Configure the runner to have network access to the database. + +Pros + +* If a project already uses self-hosted runners, this can be the simplest option as it provides all the benefits running migrations directly from GitHub Actions without the security impact. + +Cons + +* The main downside is that this requires maintaining self-hosted GitHub Action runners, which is too costly to implement and maintain for projects that don't already have it set up. + +## Related ADRS + +* [Separate the database infrastructure into a separate layer](./0005-separate-database-infrastructure-into-separate-layer.md) +* [Provision database users with serverless function](./0006-provision-database-users-with-serverless-function.md) diff --git a/docs/decisions/infra/0008-consolidate-infra-config-from-tfvars-files-into-config-module.md b/docs/decisions/infra/0008-consolidate-infra-config-from-tfvars-files-into-config-module.md new file mode 100644 index 000000000..1ae204d1b --- /dev/null +++ b/docs/decisions/infra/0008-consolidate-infra-config-from-tfvars-files-into-config-module.md @@ -0,0 +1,30 @@ +# Consolidate infra configuration from .tfvars files into config module + +* Status: accepted +* Deciders: @lorenyu @rocketnova @kyeah @acouch +* Date: 2023-09-07 + +Technical Story: [Replace configure scripts with project/app config variables #312](https://github.com/navapbc/template-infra/issues/312) + +## Context + +Currently, application infrastructure configuration is split across config modules (see [app-config](/infra/app/app-config/)) as well as .tfvars files in each of the application's infra layers - infra/app/build-repository, infra/app/database, and infra/app/service. As @kyeah pointed out, it’s easy to make mistakes when configuration is spread across multiple files, and expressed a desire to manage manage tfvars across environments all in a single file the way that some applications do for application configuration. And as @acouch [pointed out](https://github.com/navapbc/template-infra/pull/282#discussion_r1219930653), there is a lot of duplicate code with the configure scripts (setup-current-account.sh, configure-app-build-repository.sh, configure-app-database.sh, configure-app-service.sh) that configure the backend config and variable files for each infrastructure layer, which increases the burden of maintaining the configuration scripts. + +## Overview + +This ADR proposes the following: + +* Move all environment configuration into [app-config](/infra/app/app-config/) modules +* Remove the need for .tfvars files +* Remove the configuration scripts that are currently used for configuring each infrastructure layer + +Benefits: + +* All configuration can now be managed in the [app-config](/infra/app/app-config/) module. +* All dependencies between root modules can be managed explicitly via the [app-config](/infra/app/app-config/) module. +* Custom configuration scripts no longer need to be maintained +* Eliminates the need to specify -var-file option when running terraform apply, which reduces the need for terraform wrapper scripts + +## Links + +* Builds on [ADR-0004](./0004-separate-terraform-backend-configs-into-separate-config-files.md) diff --git a/docs/decisions/infra/0009-separate-app-infrastructure-into-layers.md b/docs/decisions/infra/0009-separate-app-infrastructure-into-layers.md new file mode 100644 index 000000000..cbc882153 --- /dev/null +++ b/docs/decisions/infra/0009-separate-app-infrastructure-into-layers.md @@ -0,0 +1,40 @@ +# Separate app infrastructure into layeres + +* Status: accepted +* Deciders: @lorenyu @rocketnova @jamesbursa +* Date: 2023-09-11 + +Technical Story: [Document rationale for splitting up infra layers across multiple root modules](https://github.com/navapbc/template-infra/issues/431) + +## Context and Problem Statement + +This document builds on the database module design [ADR: Separate the database infrastructure into a separate layer](./0005-separate-database-infrastructure-into-separate-layer.md) to describe the general rationale for separating application environment infrastructure into separate root modules that are managed in separate terraform state files and updated separately rather than all in one single root module. It restates and summarizes the rationale from the previous ADR and includes additional motivating examples. + +## Overview + +Based on the factors in the section below, the infrastructure has been groups into the following separate layers: + +* Account layer +* Network layer +* Build repository layer +* Database layer +* Service layer + +### Factors + +* **Variations in number and types of environments in each layer:** Not all layers of infrastructure have the same concept of "environment" as the application layer. The AWS account layer might have one account for all applications, two accounts, one for non-production environments and one for the production environment, or one account per environment. The network (VPC) layer can have similar variations (one VPC for non-prod and one for prod, or one per environment). The build repository layer only has one instance that's shared across all environments. And the database layer may or may not even be needed depending on the application. Putting all resources in one root module would disallow these variations between layers unless you introduce special case logic that behaves differently based on the environment (e.g. logic like "if environment = prod then create build-repository"), which increases complexity and reduces consistency between environments. + +* **AWS uniqueness constraints on resources:** This is a special case of the previous bullet, but some resources have uniqueness constraints in AWS. For example, there can only be one OIDC provider for GitHub actions per AWS account (see [Creating OIDC identity providers](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_oidc.html)). As another example, there can only be one VPC endpoint per VPC per AWS service (see [Fix conflicting DNS domain errors for interface VPC endpoints](https://repost.aws/knowledge-center/vpc-interface-endpoint-domain-conflict)). Therefore, if multiple application environments share a VPC, they can't each create a VPC endpoint for the same AWS service. As such, the VPC endpoint logically belongs to the network layer and VPC endpoints should be created and managed per network environment rather than per application environment. + +* **Policy constraints on what resources project team is authorized to manage:** Some projects have restrictions on who can create or modify certain categories of resources. For example, on some projects, VPCs have to be created by a central cloud operations team upon request and provided to the project team. Separating the infrastructure into modular layers allows project teams to manage downstream layers like the database and service even if upstream layers are managed externally. In our example, if the VPC layer is provided by another department, the project team can skip using the network layer, or modify the network layer to build upon the externally provided VPC, and the project team need not refactor the rest of the infrastructure. + +* **Out of band dependencies:** Some infrastructure resources depend on steps that take place outside of AWS and terraform in order to complete the creation for those layers, which makes it infeasible to rely on terraform's built-in resource dependency graphy to manage the creation of downstream resources. For example, creating an SSL/TLS certificate relies on an external step to verify ownership of the domain before it can be used by a downstream load balancer. As another example, after creating a database cluster, the database schemas, roles, and privileges need to be configured before they can be used by a downstream service. Separating infrastructure layers allows upstream dependencies to be fully created before attempting to create downstream dependent resources. + +* **Mitigate risk of accidental changes:** Some layers, such as the network and database layers, aren't expected to change frequently, whereas the service layer is expected to change on every deploy in order to update the image tag in the task definition. Separating the layers reduces the risk of accidentally making changes to one layer when applying changes to another layer. + +* **Speed of terraform plans:** The more resources are managed in a terraform state file, the more network calls terraform needs to make to AWS in order to fetch the current state of the infrastructure, which causes terraform plans to take more time. Separating out resources that rarely need to change improves the efficiency of making infrastructure changes. + +## Links + +* Based on [ADR-0005](./0005-separate-database-infrastructure-into-separate-layer.md) +* [Module architecture](/docs/infra/module-architecture.md) diff --git a/docs/decisions/infra/0010-feature-flags-system-design.md b/docs/decisions/infra/0010-feature-flags-system-design.md new file mode 100644 index 000000000..d9bb27039 --- /dev/null +++ b/docs/decisions/infra/0010-feature-flags-system-design.md @@ -0,0 +1,124 @@ +# Feature flags system design + +* Deciders: @aligg @Nava-JoshLong @lorenyu +* Date: 2023-11-28 + +## Context + +All projects should have some sort of feature flag mechanism for controlling the release and activation of features. This accelerates product development by unblocking developers from being able to deploy continuously while still providing business owners with control over when features are visible to end users. More advanced feature flag systems can also provide the ability to do gradual rollouts to increasing percentages of end users and to do split tests (also known as A/B tests) to evaluate the impact of different feature variations on user behavior and outcomes, which provide greater flexibility on how to reduce the risk of launching features. As an example, when working on a project to migrate off of legacy systems, having the ability to slowly throttle traffic to the new system while monitoring for issues in production is critical to managing risk. + +## Requirements + +1. The project team can define feature flags, or feature toggles, that enable/disable a set of functionality in an environment, depending on whether the flag is enabled or disabled. +2. The feature flagging system should support gradual rollouts, the ability to roll out a feature incrementally to a percentage of users. +3. Separate feature flag configuration from implementation of the feature flags, so that feature flags can be changed frequently through configuration without touching the underlying feature flag infrastructure code. + +## Approach + +This tech spec explores the use of [AWS CloudWatch Evidently](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Evidently.html), a service that provides functionality for feature flags, gradual rollouts, and conducting split testing (A/B testing) experiments. + +## Feature management + +One key design question is how features should be managed once defined. How should a team go about enabling and disabling the feature flags and adjusting the percentage of traffic to send to a new feature during a feature launch? + +### Option 1. Manage features using app-config module as part of service layer + +Define features in [app-config](/infra/app/app-config/), and use that configuration in the [service layer](/infra/app/service/) to create and configure the features in AWS Evidently. + +* Everything is defined in code and in one place. +* Feature and feature configurations are updated automatically as part of service deploys or can be done manually via a terraform apply. + +The configuration in the app-config module might look something like the following: + +```terraform +features = { + some_disabled_feature = {} // defaults to enabled = false + + some_enabled_feature = { + enabled = true + } + + partially_rolled_out_feature = { + throttle_percentage = 0.2 + } +} +``` + +### Option 2. Manage features using app-config as part of a separate infrastructure layer + +Define features in [app-config](/infra/app/app-config/main.tf). Create the features in the [service layer](/infra/app/service/) but set things like throttle percentages (for gradual rollouts) in a separate infrastructure layer. + +* Allows for separation of permissions. For example, individuals can have permission to update feature launch throttle percentages without having permission to create, edit, or delete the features themselves. + +### Option 3. Manage features in AWS Console outside of terraform + +Define features in [app-config](/infra/app/app-config/main.tf) and create them in the [service layer](/infra/app/service), but set things like throttle percentages (for gradual rollouts) outside of terraform (e.g. via AWS Console). Use `lifecycle { ignore_changes = [entity_overrides] }` in the terraform configuration for the `aws_evidently_feature` resources to ignore settings that are managed via AWS Console. + +* Empowers non-technical roles like business owners and product managers to enable and disable feature flags and adjust feature launch throttle percentages without needing to depend on the development team. +* A no-code approach using the AWS Console GUI means that it's possible to leverage the full set of functionality offered by AWS CloudWatch Evidently, including things like scheduled launches, with minimal training and without needing to learn how to do it in code. + +A reduced configuration in the app-config module that just defines the features might look something like the following: + +```terraform +feature_flags = [ + "some_new_feature_1", "some_new_feature_2" +] +``` + +## Decision Outcome + +Chosen option: "Option 3: Manage features in AWS Console outside of terraform". The ability to empower business and product roles to control launches and experiments without depending on engineering team maximizes autonomy and allows for the fastest delivery. + +## Notes on application layer design + +The scope of this tech spec is focused on the infrastructure layer, but we'll include some notes on the elements of feature flag management that will need to be handled at the application layer. + + +### Application layer requirements + +1. Client interface with feature flag service — Applications need a client module that captures the feature flag service abstraction. The application code will interface with this module rather than directly with the underlying feature flag service. +2. Local development — Project team developers need a way to create and manage feature flags while developing locally, ideally without dependencies on an external service. + +### Application layer design + +#### Feature flag module interface + +At it's core, the feature flag module needs a function `isFeatureEnabled` that determines whether a feature has been enabled. It needs to accept a feature name, and for gradual rollouts it will also need a user identifier. This is so that the system can remember which variation was assigned to a given user, so that any individual user will have a consistent experience. + +```ts +interface FeatureFlagService { + isFeatureEnabled(featureName: string, userId?: string): boolean +} +``` + +#### Adapter pattern + +The feature flag module should use the adapter pattern to provide different mechanisms for managing feature flags depending on the environment. Deployed cloud environments should use the Amazon CloudWatch Evidently service. Local development environments could use a mechanism available locally, such as environment variables, config files, cookies, or a combination. + +```ts +import { EvidentlyClient, EvaluateFeatureCommand } from "@aws-sdk/client-evidently"; + +class EvidentlyFeatureFlagService implements FeatureFlagService { + client: EvidentlyClient + + isFeatureEnabled(feature: string, userId?: string): boolean { + const command = new EvaluateFeatureCommand({ + ... + feature, + entityId: userId, + ... + }); + const response = await this.client.send(command) + ... + } +} +``` + +```ts +class LocalFeatureFlagService implements FeatureFlagService { + + isFeatureEnabled(feature: string, userId?: string): boolean { + // check config files, environment variables, and/or cookies + } +} +``` diff --git a/docs/decisions/infra/0011-network-layer-design.md b/docs/decisions/infra/0011-network-layer-design.md new file mode 100644 index 000000000..5f48c930b --- /dev/null +++ b/docs/decisions/infra/0011-network-layer-design.md @@ -0,0 +1,135 @@ +# Design of network layer + +* Deciders: @lorenyu @shawnvanderjagt +* Date: 2023-12-01 + +## Context and Problem Statement + +Most projects will need to deploy their applications into custom VPCs. The policies around VPCs can vary. For example, some projects might require each application environment to be in its own VPC, while other projects might have all lower environments share a VPC. Some projects might have all resources live in one AWS account, while others might isolate resources into separate accounts. Some projects might have permission to create and configure the VPCs (according to agency security policies), while other projects might have the VPC created by the agency's shared infrastructure team before it's provided to the project team to use. This technical specification proposes a design of the network layer that accommodates these various configurations in a simple modular manner. + +## Requirements + +1. The project team can create any number of networks, or VPCs, independently of the number of AWS accounts or the number of applications or application environments. +2. Created VPCs can be mapped arbitrarily to AWS accounts. They can all be created in a single AWS account or separated across multiple AWS accounts. +3. An application environment can map to any of the created VPCs, or to a VPC that is created outside of the project. + +We aim to achieve these requirements without adding complexity to the other layers. The network layer should be decoupled from the other layers. + +## Approach + +### Network configuration + +Define the configuration for networks in a new property `network_configs` in the [project-config module](/infra/project-config/main.tf). `network_configs` is a map from the network name to the network configuration. The network name is a name the project team choose to serve as a human-readable identifier to reference the network. To keep network configuration DRY and reuse common configurations between networks, create a sub-module `network-config` under the project-config module, analogous to the [env-config module](/infra/app/app-config/env-config/) under the [app-config module](/infra/app/app-config/). The `project-config` module might look something like this: + +```terraform +# project-config/main.tf + +network_configs = { + dev = module.dev_network_config + ... + [NETWORK_NAME] = module.[NETWORK_NAME]_network_config +} + +# project-config/dev-network.tf + +module "dev_network_config" { + source "./network-config" + ... +} + +... + +# project-config/[NETWORK_NAME]-network.tf + +module "[NETWORK_NAME]_network_config" { + source "./network-config" + ... +} +``` + +Each network config will have the following properties: + +* **account_name** — Name of AWS account that the VPC should be created in. Used to document which AWS account the network lives in and to determine which AWS account to authenticate into when making modifications to the network in scripts such as CI/CD +* Each network will have three subnets, (1) a public subnet, (2) a private subnet for the application layer, and (3) private subnet for the data layer +* The network will also have different properties depending on the applications that are using the network (see [Application-specific network configuration](#application-specific-network-configuration)) + +### Add network_name tag to VPC + +Add a "network_name" name tag to the VPC with the name of the network. The VPC tag will be used by the service layer to identify the VPC in an [aws_vpc data source](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc). Tags are used because at this time AWS VPCs do not have any user-provided identifiers such as a VPC name. Generated identifiers like `vpc_id` cannot be used because `vpc_id` is not known statically at configuration time, and we are following the pattern of [using configuration and data sources to manage dependencies between different infrastructure layers](/docs/infra/module-dependencies.md#use-config-modules-and-data-resources-to-manage-dependencies-between-root-modules). + +## Application-specific network configuration + +In order to determine which VPC to use for each application environment, add a `network_name` property to the [environment config](/infra/app/app-config/env-config/). The network name will be used in database and service layers by the `aws_vpc` data source: + +```terraform +data "aws_vpc" "network" { + tags = { + network_name = local.environment_config.network_name + } +} +``` + +Networks associated with applications using the `network_name` property will have the following properties based on the application configuration. + +1. The `has_database` setting determines whether or not to create VPC endpoints needed by the database layer. +2. The `has_external_non_aws_service` setting determines whether or not to create NAT gateways, which allows the service in the private subnet to make requests to the internet. + +### Example configurations + +Example project with a multi-account setup + +```mermaid +graph RL; + subgraph accounts + dev_account[dev] + staging_account[staging] + prod_account[prod] + end + + subgraph networks + dev_network[dev] + staging_network[staging] + prod_network[prod] + end + + subgraph environments + dev_environment[dev] + staging_environment[staging] + prod_environment[prod] + end + + dev_network --> dev_account + staging_network --> staging_account + prod_network --> prod_account + + dev_environment --> dev_network + staging_environment --> staging_network + prod_environment --> prod_network +``` + +Example project with a single account and a shared VPC "lowers" for lower environments + +```mermaid +graph RL; + subgraph accounts + shared_account[shared] + end + + subgraph networks + lowers_network[lowers] + prod_network[prod] + end + + subgraph environments + dev_environment[dev] + staging_environment[staging] + prod_environment[prod] + end + + lowers_network --> shared_account + prod_network --> shared_account + + dev_environment --> lowers_network + staging_environment --> lowers_network + prod_environment --> prod_network +``` diff --git a/docs/decisions/template.md b/docs/decisions/template.md new file mode 100644 index 000000000..25696bbe7 --- /dev/null +++ b/docs/decisions/template.md @@ -0,0 +1,72 @@ +# [short title of solved problem and solution] + +* Status: [proposed | rejected | accepted | deprecated | … | superseded by [ADR-0005](0005-example.md)] +* Deciders: [list everyone involved in the decision] +* Date: [YYYY-MM-DD when the decision was last updated] + +Technical Story: [description | ticket/issue URL] + +## Context and Problem Statement + +[Describe the context and problem statement, e.g., in free form using two to three sentences. You may want to articulate the problem in form of a question.] + +## Decision Drivers + +* [driver 1, e.g., a force, facing concern, …] +* [driver 2, e.g., a force, facing concern, …] +* … + +## Considered Options + +* [option 1] +* [option 2] +* [option 3] +* … + +## Decision Outcome + +Chosen option: "[option 1]", because [justification. e.g., only option, which meets k.o. criterion decision driver | which resolves force force | … | comes out best (see below)]. + +### Positive Consequences + +* [e.g., improvement of quality attribute satisfaction, follow-up decisions required, …] +* … + +### Negative Consequences + +* [e.g., compromising quality attribute, follow-up decisions required, …] +* … + +## Pros and Cons of the Options + +### [option 1] + +[example | description | pointer to more information | …] + +* Good, because [argument a] +* Good, because [argument b] +* Bad, because [argument c] +* … + +### [option 2] + +[example | description | pointer to more information | …] + +* Good, because [argument a] +* Good, because [argument b] +* Bad, because [argument c] +* … + +### [option 3] + +[example | description | pointer to more information | …] + +* Good, because [argument a] +* Good, because [argument b] +* Bad, because [argument c] +* … + +## Links + +* [Link type] [Link to ADR] +* … diff --git a/docs/feature-flags.md b/docs/feature-flags.md new file mode 100644 index 000000000..c9c40280a --- /dev/null +++ b/docs/feature-flags.md @@ -0,0 +1,31 @@ +# Feature flags and partial releases + +Feature flags are an important tool that enables [trunk-based development](https://trunkbaseddevelopment.com/). They allow in-progress features to be merged into the main branch while still allowing that branch to be deployed to production at any time, thus decoupling application deploys from feature releases. For a deeper introduction, [Martin Fowler's article on Feature Toggles](https://martinfowler.com/articles/feature-toggles.html) and [LaunchDarkly's blog post on feature flags](https://launchdarkly.com/blog/what-are-feature-flags/) are both great articles that explain the what and why of feature flags. + +## How it works + +This project leverages [Amazon CloudWatch Evidently](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Evidently.html) to create and manage feature flags. + +## Creating feature flags + +The list of feature flags for an application is defined in the `feature_flags` property in its app-config module (in `/infra/[app_name]/app-config/main.tf`). To create a new feature flag, add a new string to that list. To remove a feature flag, remove the feature flag from the list. The set of feature flags will be updated on the next terraform apply of the service layer, or during the next deploy of the application. + +## Querying feature flags in the application + +To determine whether a particular feature should be enabled or disabled for a given user, the application code calls an "is feature enabled" function in the feature flags module. Under the hood, the module will call AWS Evidently's [EvaluateFeature](https://docs.aws.amazon.com/cloudwatchevidently/latest/APIReference/API_EvaluateFeature.html) API to determine whether a feature is enabled or disabled. For partial rollouts, it will remember which variation of the application a particular user saw and keep the user experience consistent for that user. For more information about the feature flags module, look in the application code and docs. + +## Managing feature releases and partial rollouts via AWS Console + +The system is designed to allow the managing of feature releases and partial rollouts outside of terraform, which empowers business owners and product managers to control enable and disable feature flags and adjust feature launch traffic percentages without needing to depend on the development team. + +### To enable or disable a feature + +1. Navigate to the Evidently service in AWS Console, select the appropriate Evidently feature flags project for the relevant application environment, and select the feature you want to manage. +2. In the actions menu, select "Edit feature". +3. Under "Feature variations", select either "FeatureOn" (to enable a feature) or "FeatureOff" (to disable a feature) to be the "Default" variation, then submit. **Warning: Do not modify the variation values. "FeatureOn" should always have a value of "True" and "FeatureOff" should always have a value of "False".** + +### To manage a partial rollout + +1. Navigate to the Evidently service in AWS Console, and select the appropriate Evidently feature flags project for the relevant application environment +2. Select "Create launch" to create a new partial rollout plan, or select an existing launch to manage an existing rollout +3. Under "Launch configuration", choose the traffic percentage you want to send to each variation, and choose whether you want the launch to begin immediately or on a schedule. diff --git a/docs/infra/database-access-control.md b/docs/infra/database-access-control.md new file mode 100644 index 000000000..5032d16c9 --- /dev/null +++ b/docs/infra/database-access-control.md @@ -0,0 +1,24 @@ +# Database Access Control + +## Manage `postgres` master user password with AWS Secrets Manager + +The master user password is managed by Amazon RDS and Secrets Manager. Managing RDS master user passwords with Secrets Manager provides the following security benefits: + +* RDS rotates database credentials regularly, without requiring application changes. +* Secrets Manager secures database credentials from human access and plain text view. The master password is not even in the terraform state file. + +For more information about the benefits, see [Benefits of managing master user passwords with Secrets Manager](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/rds-secrets-manager.html#rds-secrets-manager-benefits). + +## Database roles and permissions + +The database roles are created by the master user `postgres` when the Role Manager lambda function runs. The following roles are created: + +* **migrator** — The `migrator` role is the role the database migration task assumes. Database migrations are run as part of the deploy workflow before the new container image is deployed to the service. The `migrator` role has permissions to create tables in the `app` schema. +* **app** — The `app` role is the role the application service assumes. The `app` role has read/write permissions in the `app` schema. + +## Database connections + +The database authenticates connections with [IAM database authentication](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.html) (except when connecting as the `postgres` master user). The security benefits of this approach include: + +* The system leverages IAM to centrally manage access to the database +* There are no long lived user database credentials that need to be stored as database authentication tokens are generated by IAM and have a lifetime of 15 minutes diff --git a/docs/infra/destroy-infrastructure.md b/docs/infra/destroy-infrastructure.md new file mode 100644 index 000000000..eea655d30 --- /dev/null +++ b/docs/infra/destroy-infrastructure.md @@ -0,0 +1,59 @@ +# Destroy infrastructure + +To destroy everything you'll need to undeploy all the infrastructure in reverse order that they were created. In particular, the account root module(s) need to be destroyed last. + +## Instructions + +1. First destroy all your environments. Within `/infra/app/service` run the following, replacing `dev` with the environment you're destroying. + + ```bash + $ terraform init --backend-config=dev.s3.tfbackend + $ terraform destroy -var-file=dev.tfvars + ``` + +2. Then to destroy the backends, first you'll need to add `force_destroy = true` to the S3 buckets, and update the lifecycle block to set `prevent_destroy = false`. Then run `terraform apply` from within the `infra/accounts` directory. The reason we need to do this is because S3 buckets by default are protected from destruction to avoid loss of data. See [Terraform: Destroy/Replace Buckets](https://medium.com/interleap/terraform-destroy-replace-buckets-cf9d63d0029d) for a more in depth explanation. + + ```terraform + # infra/modules/modules/terraform-backend-s3/main.tf + + resource "aws_s3_bucket" "tf_state" { + bucket = var.state_bucket_name + + force_destroy = true + + # Prevent accidental destruction a developer executing terraform destory in the wrong directory. Contains terraform state files. + lifecycle { + prevent_destroy = false + } + } + + ... + + resource "aws_s3_bucket" "tf_log" { + bucket = var.tf_logging_bucket_name + force_destroy = true + } + ``` + +3. Then since we're going to be destroying the tfstate buckets, you'll want to move the tfstate file out of S3 and back to your local system. Comment out or delete the s3 backend configuration: + + ```terraform + # infra/accounts/main.tf + + # Comment out or delete the backend block + backend "s3" { + ... + }2 + ``` + +4. Then run the following from within the `infra/accounts` directory to copy the tfstate back to a local tfstate file: + + ```bash + terraform init -force-copy + ``` + +5. Finally, you can run `terraform destroy` within the `infra/accounts` directory. + + ```bash + terraform destroy + ``` diff --git a/docs/infra/intro-to-terraform-workspaces.md b/docs/infra/intro-to-terraform-workspaces.md new file mode 100644 index 000000000..f692e1e82 --- /dev/null +++ b/docs/infra/intro-to-terraform-workspaces.md @@ -0,0 +1,59 @@ +# Workspaces + +Terraform workspaces are created by default, the default workspace is named "default." Workspaces are used to allow multiple engineers to deploy their own stacks for development and testing. This allows multiple engineers to develop new features in parallel using a single environment without destroying each others infrastructure. Separate resources will be created for each engineer when using the prefix variable. + +## Terraform workspace commands + +`terraform workspace show [Name]` - This command will show the workspace you working in. + +`terraform workspace list [Name]` - This command will list all workspaces. + +`terraform workspace new [Name]` - This command will create a new workspace. + +`terraform workspace select [Name]` - This command will switch your workspace to the workspace you select. + +`terraform workspace delete [Name]` - This command will delete the specified workspace. (does not delete infrastructure, that step will done first) + +## Workspaces and prefix - A How To + + Workspaces are used to allow multiple developers to deploy their own stacks for development and testing. By default "prefix~ is set to `terraform.workspace` in the envs/dev environment, it is `staging` and `prod` in those respective environments. + +### envs/dev/main.tf + +``` tf +locals { + prefix = terraform.workspace +} + +module "example" { + source = "../../modules/example" + prefix = local.prefix +} + +``` + +### modules/example/variables.tf - When creating a new module create the variable "prefix" in your variables.tf + +``` tf + +variable "prefix" { + type = string + description = "prefix used to uniquely identify resources, allows parallel development" + +} + +``` + +### modules/example/main.tf - Use var.prefix to uniquely name resources for parallel development + +``` tf + +# Create the S3 bucket with a unique prefix from terraform.workspace. +resource "aws_s3_bucket" "example" { + bucket = "${var.prefix}-bucket" + +} + +``` + +When in the workspace "shawn", the resulting bucket name created in the aws account will be `shawn-bucket`. This prevents the following undesirable situation: If resources are not actively prefixed and two developers deploy the same resource, the developer who runs their deployment second will overwrite the deployment of the first. diff --git a/docs/infra/intro-to-terraform.md b/docs/infra/intro-to-terraform.md new file mode 100644 index 000000000..6736248c5 --- /dev/null +++ b/docs/infra/intro-to-terraform.md @@ -0,0 +1,33 @@ +# Introduction to Terraform + +## Basic Terraform Commands + +The `terraform init` command is used to initialize a working directory containing Terraform configuration files. This is the first command that should be run after writing a new Terraform configuration or cloning an existing one from version control. + +The `terraform plan` command creates an execution plan, which lets you preview the changes that Terraform plans to make to your infrastructure. By default, when Terraform creates a plan it: + +- Reads the current state of any already-existing remote objects to make sure that the Terraform state is up-to-date. +- Compares the current configuration to the prior state and noting any differences. +- Proposes a set of change actions that should, if applied, make the remote objects match the configuration. + +The `terraform apply` command executes the actions proposed in a Terraform plan deploying the infrastructure specified in the configuration. Use with caution. The configuration becomes idempotent once a subsequent apply returns 0 changes. + +The `terraform destroy` command is a convenient way to destroy all remote objects managed by a particular Terraform configuration. Use `terraform plan -destroy` to preview what remote objects will be destroyed if you run `terraform destroy`. + +⚠️ WARNING! ⚠️ This is a destructive command! As a best practice, it's recommended that you comment out resources in non-development environments rather than running this command. `terraform destroy` should only be used as a way to cleanup a development environment. e.g. a developers workspace after they are done with it. + +For more information about terraform commands follow the link below: + +- [Basic CLI Features](https://www.terraform.io/cli/commands) + +## Terraform Dependency Lock File + +The [dependency lock file](https://www.terraform.io/language/files/dependency-lock) tracks provider dependencies. It belongs to the configuration as a whole and is created when running `terraform ini`. The lock file is always named `.terraform.lock.hcl`, and this name is intended to signify that it is a lock file for various items that Terraform caches in the `.terraform` subdirectory of your working directory. You should include this file in your version control repository so that you can discuss potential changes to your external dependencies via code review, just as you would discuss potential changes to your configuration itself. + +## Modules + +A module is a container for multiple resources that are used together. Modules can be used to create lightweight abstractions, so that you can describe your infrastructure in terms of its architecture, rather than directly in terms of physical objects. The .tf files in your working directory when you run `terraform plan` or `terraform apply` together form the root module. In this root module you will call modules that you create from the module directory to build the infrastructure required to provide any functionality needed for the application. + +## Terraform Workspaces + +Workspaces are used to allow multiple engineers to deploy their own stacks for development and testing. Read more about it in [Terraform Workspaces](./intro-to-terraform-workspaces.md) diff --git a/docs/infra/making-infra-changes.md b/docs/infra/making-infra-changes.md new file mode 100644 index 000000000..9998569e3 --- /dev/null +++ b/docs/infra/making-infra-changes.md @@ -0,0 +1,56 @@ +# Making and applying infrastructure changes + +## Requirements + +First read [Module Architecture](./module-architecture.md) to understand how the terraform modules are structured. + +## Using make targets (recommended) + +For most changes you can use the Make targets provided in the root level Makefile, and can all be run from the project root. + +Make changes to the account: + +```bash +make infra-update-current-account +``` + +Make changes to the application service in the dev environment: + +```bash +make infra-update-app-service APP_NAME=app ENVIRONMENT=dev +``` + +Make changes to the application build repository (Note that the build repository is shared across environments, so there is no ENVIRONMENT parameter): + +```bash +make infra-update-app-build-repository APP_NAME=app +``` + +You can also pass in extra arguments to `terraform apply` by using the `TF_CLI_ARGS` or `TF_CLI_ARGS_apply` parameter (see [Terraform's docs on TF_CLI_ARGS and TF_CLI_ARGS_name](https://developer.hashicorp.com/terraform/cli/config/environment-variables#tf_cli_args-and-tf_cli_args_name)): + +```bash +# Example +TF_CLI_ARGS_apply='-input=false -auto-approve' make infra-update-app-service APP_NAME=app ENVIRONMENT=dev +TF_CLI_ARGS_apply='-var=image_tag=abcdef1' make infra-update-app-service APP_NAME=app ENVIRONMENT=dev +``` + +## Using terraform CLI wrapper scripts + +An alternative to using the Makefile is to directly use the terraform wrapper scripts that the Makefile uses: + +```bash +project-root$ ./bin/terraform-init.sh app/service dev +project-root$ ./bin/terraform-apply.sh app/service dev +project-root$ ./bin/terraform-init-and-apply.sh app/service dev # calls init and apply in the same script +``` + +Look in the script files for more details on usage. + +## Using terraform CLI directly + +Finally, if the wrapper scripts don't meet your needs, you can always run terraform directly from the root module directory. You may need to do this if you are running terraform commands other than `terraform plan` and `terraform apply`, such as `terraform import`, `terraform taint`, etc. To do this, you'll need to pass in the appropriate `tfvars` and `tfbackend` files to `terraform init` and `terraform apply`. For example, to make changes to the application's service resources in the dev environment, cd to the `infra/app/service` directory and run: + +```bash +infra/app/service$ terraform init -backend-config=dev.s3.tfbackend +infra/app/service$ terraform apply -var-file=dev.tfvars +``` diff --git a/docs/infra/module-architecture.md b/docs/infra/module-architecture.md new file mode 100644 index 000000000..66aa8aefc --- /dev/null +++ b/docs/infra/module-architecture.md @@ -0,0 +1,92 @@ +# Terraform module architecture + +This doc describes how the Terraform modules are structured. Directory structure and layers are documented in the [infrastructure README](/infra/README.md). + +## Approach + +The infrastructure code is organized into: + +- root modules +- child modules + +[Root modules](https://www.terraform.io/language/modules#the-root-module) are modules that are deployed separately from each other, whereas child modules are reusable modules that are called from root modules. To deploy all the resources necessary for a given environment, all the root modules must be deployed independently in the correct order. + +For a full list of rationale and factors, see [ADR: Separate app infrastructure into layers](/docs/decisions/infra/0009-separate-app-infrastructure-into-layers.md). + +## Module calling structure + +The following diagram describes the relationship between modules and their child modules. Arrows go from the caller module to the child module. + +```mermaid +flowchart TB + + classDef default fill:#FFF,stroke:#000 + classDef root-module fill:#F37100,stroke-width:3,font-family:Arial + classDef child-module fill:#F8E21A,font-family:Arial + + subgraph infra + account:::root-module + + subgraph app + app/build-repository[build-repository]:::root-module + app/network[network]:::root-module + app/database[database]:::root-module + app/service[service]:::root-module + end + + subgraph modules + terraform-backend-s3:::child-module + auth-github-actions:::child-module + container-image-repository:::child-module + network:::child-module + database:::child-module + web-app:::child-module + end + + account --> terraform-backend-s3 + account --> auth-github-actions + app/build-repository --> container-image-repository + app/network --> network + app/database --> database + app/service --> web-app + + end +``` + +## Module dependencies + +The following diagram illustrates the dependency structure of the root modules. + +1. Account root modules need to be deployed first to create the S3 bucket and DynamoDB tables needed to configure the Terraform backends in the rest of the root modules. +2. The application's build repository needs to be deployed next to create the resources needed to store the built release candidates that are deployed to the application environments. +3. The individual application environment root modules are deployed last once everything else is set up. These root modules are the ones that are deployed regularly as part of application deployments. + +```mermaid +flowchart RL + +classDef default fill:#F8E21A,stroke:#000,font-family:Arial + +app/service --> app/build-repository --> accounts +app/service --> accounts +app/service --> app/network +app/service --> app/database --> app/network --> accounts +app/database --> accounts +``` + +### Guidelines for layers + +When deciding which layer to put an infrastructure resource in, follow the following guidelines. + +* **Default to the service layer** By default, consider putting application resources as part of the service layer. This way the resource is managed together with everything else in the environment, and spinning up new application environments automatically spin up the resource. + +* **Consider variations in number and types of environments of each layer:** If the resource does not or might not map one-to-one with application environments, consider putting the resource in a different layer. For example, the number of AWS accounts may or may not match the number of VPCs, which may or may not match the number of application environments. As another example, each application only has one instance of a build repository, which is shared across all environments. As a final example, an application may or may not need a database layer at all, so by putting database related resources in the database layer, and application can skip those resources by skipping the entire layer rather than by needing to change the behavior of an existing layer. Choose the layer for the resource that maps most closely with that resource's purpose. + +* **Consider AWS uniqueness constraints on resources:** This is a special case of the previous consideration: resources that AWS requires to be unique should be managed by a layer that creates only one of that resource per instance of that layer. For example, there can only be one OIDC provider for GitHub actions per AWS account (see [Creating OIDC identity providers](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_oidc.html)), so the OIDC provider should go in the account layer. As another example, there can only be one VPC endpoint per VPC per AWS service (see [Fix conflicting DNS domain errors for interface VPC endpoints](https://repost.aws/knowledge-center/vpc-interface-endpoint-domain-conflict)). Therefore, if multiple application environments share a VPC, they can't each create a VPC endpoint for the same AWS service. As such, the VPC endpoint logically belongs to the network layer and VPC endpoints should be created and managed per network instance rather than per application environment. + +* **Consider policy constraints on what resources the project team is authorized to manage:** Different categories of resources may have different requirements on who is allowed to create and manage those resources. Resources that the project team are not allowed to manage directly should not be mixed with resources that the project team needs to manage directly. + +* **Consider out of band dependencies:** Put infrastructure resources that require steps outside of terraform to be completed configured in layers that are upstream to resources that depend on those completed resources. For example, after creating a database cluster, the database schemas, roles, and privileges need to be configured before they can be used by a downstream service. Therefore database resources should be separate from the service layer so that the database can be configured fully before attempting to create the service layer resources. + +## Making changes to infrastructure + +Now that you understand how the modules are structured, see [making changes to infrastructure](./making-infra-changes.md). diff --git a/docs/infra/module-dependencies.md b/docs/infra/module-dependencies.md new file mode 100644 index 000000000..b79038674 --- /dev/null +++ b/docs/infra/module-dependencies.md @@ -0,0 +1,92 @@ +# Managing module dependencies + +These are the principles that guide the design of the infrastructure template. + +## Use explicit outputs and variables to connect resources across child modules in the same root module + +If a resource in module B depends on a resource in module A, and both modules are called from the same root module, then create an output in module A with the information that is needed by module B, and pass that into module B as an input variable. + +```terraform +# root-module/main.tf + +module "a" { + ... +} + +module "b" { + input = module.a.output +} +``` + +This makes the dependencies between the resources explicit: + +```mermaid +flowchart LR + +subgraph A[module A] + output +end + +subgraph B[module B] + input +end + +input -- depends on --> output +``` + +**Do not** use [data sources](https://developer.hashicorp.com/terraform/language/data-sources) to reference resource dependencies in the same root module. A data source does not represent a dependency in [terraform's dependency graph](https://developer.hashicorp.com/terraform/internals/graph), and therefore there will potentially be a race condition, as Terraform will not know that it needs to create/update the resource in module A before it creates/updates the resource in module B that depends on it. + +## Use config modules and data resources to manage dependencies between root modules + +If a resource in root module S depends on a resource in root module R, it is not possible to specify the dependency directly since the resources are managed in separate state files. In this situation, use a [data source](https://developer.hashicorp.com/terraform/language/data-sources) in module S to reference the resource in module R, and use a shared configuration module that specifies identifying information that is used both to create the resource in R and to query for the resource in S. + +```terraform +# root module R + +module "config" { + ... +} + +resource "parent" "p" { + identifier = module.config.parent_identifier +} +``` + +```terraform +# root module S + +module "config" { + ... +} + +data "parent" "p" { + identifier = module.config.parent_identifier +} + +resource "child" "c" { + input = data.parent.p.some_attribute +} +``` + +This makes the dependency explicit, but indirect. Instead of one resource directly depending on the other, both resources depend on a shared config value(s) that uniquely identifies the parent resource. If the parent resource changes, the data source will also change, triggering the appropriate change in the child resource. If identifying information about the parent resource changes, it must be done through the shared configuration module so that the data source's query remains in sync. + +```mermaid +flowchart RL + +subgraph config[config module] + config_value[config value] +end + +subgraph R[root module R] + parent[parent resource] +end + +subgraph S[root module S] + data.parent[parent data source] + child[child resource] +end + +parent -- depends on --> config_value +data.parent -- depends on --> config_value +child -- depends on --> data.parent +``` diff --git a/docs/infra/set-up-app-build-repository.md b/docs/infra/set-up-app-build-repository.md new file mode 100644 index 000000000..2f91ec211 --- /dev/null +++ b/docs/infra/set-up-app-build-repository.md @@ -0,0 +1,32 @@ +# Set up application build repository + +The application build repository setup process will create infrastructure resources needed to store built release candidate artifacts used to deploy the application to an environment. + +## Requirements + +Before setting up the application's build repository you'll need to have: + +1. [Set up the AWS account](./set-up-aws-account.md) +2. [Configure the app](/infra/app/app-config/main.tf) + +## 1. Configure backend + +To create the tfbackend file for the build repository using the backend configuration values from your current AWS account, run + +```bash +make infra-configure-app-build-repository APP_NAME=app +``` + +Pass in the name of the app folder within `infra`. By default this is `app`. + +## 2. Create build repository resources + +Now run the following commands to create the resources, making sure to verify the plan before confirming the apply. + +```bash +make infra-update-app-build-repository APP_NAME=app +``` + +## Set up application environments + +Once you set up the deployment process, you can proceed to [set up application environments](./set-up-app-env.md) diff --git a/docs/infra/set-up-app-env.md b/docs/infra/set-up-app-env.md new file mode 100644 index 000000000..1773a860f --- /dev/null +++ b/docs/infra/set-up-app-env.md @@ -0,0 +1,60 @@ +# Set up application environment + +The application environment setup process will: + +1. Configure a new application environment and create the infrastructure resources for the application in that environment + +## Requirements + +Before setting up the application's environments you'll need to have: + +1. [A compatible application in the app folder](https://github.com/navapbc/template-infra/blob/main/template-only-docs/application-requirements.md) +2. [Configure the app](/infra/app/app-config/main.tf). Make sure you update `has_database` to `true` or `false` depending on whether or not your application has a database to integrate with. + 1. If you're configuring your production environment, make sure to update the `service_cpu`, `service_memory`, and `service_desired_instance_count` settings based on the project's needs. If your application is sensitive to performance, consider doing a load test. +3. [Create a nondefault VPC to be used by the application](./set-up-network.md) +4. (If the application has a database) [Set up the database for the application](./set-up-database.md) +5. (If you have an incident management service) [Set up monitoring](./set-up-monitoring-alerts.md) +6. [Set up the application build repository](./set-up-app-build-repository.md) + +## 1. Configure backend + +To create the tfbackend and tfvars files for the new application environment, run + +```bash +make infra-configure-app-service APP_NAME=app ENVIRONMENT= +``` + +`APP_NAME` needs to be the name of the application folder within the `infra` folder. It defaults to `app`. +`ENVIRONMENT` needs to be the name of the environment you are creating. This will create a file called `.s3.tfbackend` in the `infra/app/service` module directory. + +Depending on the value of `has_database` in the [app-config module](/infra/app/app-config/main.tf), the application will be configured with or without database access. + +## 2. Build and publish the application to the application build repository + +Before creating the application resources, you'll need to first build and publish at least one image to the application build repository. + +There are two ways to do this: + +1. Trigger the "Build and Publish" workflow from your repo's GitHub Actions tab. This option requires that the `role-to-assume` GitHub workflow variable has already been setup as part of the overall infra account setup process. +1. Alternatively, run the following from the root directory. This option can take much longer than the GitHub workflow, depending on your machine's architecture. + + ```bash + make release-build APP_NAME=app + make release-publish APP_NAME=app + ``` + +Copy the image tag name that was published. You'll need this in the next step. + +## 3. Create application resources with the image tag that was published + +Now run the following commands to create the resources, using the image tag that was published from the previous step. Review the terraform before confirming "yes" to apply the changes. + +```bash +TF_CLI_ARGS_apply="-var=image_tag=" make infra-update-app-service APP_NAME=app ENVIRONMENT= +``` + +## 4. Configure monitoring alerts + +Configure email alerts, external incident management service integration and additional Cloudwatch Alerts. +[Configure monitoring module](./set-up-monitoring-alerts.md) + diff --git a/docs/infra/set-up-aws-account.md b/docs/infra/set-up-aws-account.md new file mode 100644 index 000000000..a65abcf40 --- /dev/null +++ b/docs/infra/set-up-aws-account.md @@ -0,0 +1,57 @@ +# Set up AWS account + +The AWS account setup process will: + +1. Create the [Terraform backend](https://www.terraform.io/language/settings/backends/configuration) resources needed to store Terraform's infrastructure state files. The project uses an [S3 backend](https://www.terraform.io/language/settings/backends/s3). +2. Create the [OpenID connect provider in AWS](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_oidc.html) to allow GitHub actions to access AWS account resources. + +## Prerequisites + +* You'll need to have [set up infrastructure tools](./set-up-infrastructure-tools.md), like Terraform, AWS CLI, and AWS authentication. +* You'll also need to make sure the [project is configured](/infra/project-config/main.tf). + +## Overview of Terraform backend management + +The approach to backend management allows Terraform to both create the resources needed for a remote backend as well as allow terraform to store that configuration state in that newly created backend. This also allows us to seperate infrastructure required to support terraform from infrastructure required to support the application. Because each backend, bootstrap or environment, stores their own terraform.tfstate in these buckets, ensure that any backends that are shared use a unique key. When using a non-default workspace, the state path will be `/workspace_key_prefix/workspace_name/key`, `workspace_key_prefix` default is `env:` + +## Instructions + +### 1. Make sure you're authenticated into the AWS account you want to configure + +The account set up sets up whatever account you're authenticated into. To see which account that is, run + +```bash +aws sts get-caller-identity +``` + +To see a more human readable account alias instead of the account, run + +```bash +aws iam list-account-aliases +``` + +### 2. Create backend resources and tfbackend config file + +Run the following command, replacing `` with a human readable name for the AWS account that you're authenticated into. The account name will be used to prefix the created tfbackend file so that it's easier to visually identify as opposed to identifying the file using the account id. For example, you have an account per environment, the account name can be the name of the environment (e.g. "prod" or "staging"). Or if you are setting up an account for all lower environments, account name can be "lowers". If your AWS account has an account alias, you can also use that. + +```bash +make infra-set-up-account ACCOUNT_NAME= +``` + +This command will create the S3 tfstate bucket and the GitHub OIDC provider. It will also create a `[account name].[account id].s3.tfbackend` file in the `infra/accounts` directory. + +### 3. Update the account names map in app-config + +In [app-config/main.tf](/infra/app/app-config/main.tf), update the `account_names_by_environment` config to reflect the account name you chose. + +## Making changes to the account + +If you make changes to the account terraform and want to apply those changes, run + +```bash +make infra-update-current-account +``` + +## Destroying infrastructure + +To undeploy and destroy infrastructure, see [instructions on destroying infrastructure](./destroy-infrastructure.md). diff --git a/docs/infra/set-up-database.md b/docs/infra/set-up-database.md new file mode 100644 index 000000000..d5cb6c66e --- /dev/null +++ b/docs/infra/set-up-database.md @@ -0,0 +1,88 @@ +# Set up database + +The database setup process will: + +1. Configure and deploy an application database cluster using [Amazon Aurora Serverless V2](https://aws.amazon.com/rds/aurora/serverless/) +2. Create a [PostgreSQL schema](https://www.postgresql.org/docs/current/ddl-schemas.html) `app` to contain tables used by the application. +3. Create an IAM policy that allows IAM roles with that policy attached to [connect to the database using IAM authentication](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.Connecting.html) +4. Create an [AWS Lambda function](https://docs.aws.amazon.com/lambda/latest/dg/welcome.html), the "role manager", for provisioning the [PostgreSQL database users](https://www.postgresql.org/docs/8.0/user-manag.html) that will be used by the application service and by the migrations task. +5. Invoke the role manager function to create the `app` and `migrator` Postgres users. + +## Requirements + +Before setting up the database you'll need to have: + +1. [Set up the AWS account](./set-up-aws-account.md) +2. pip installed (pip is needed to download dependencies for the role manager Lambda function) + +## 1. Configure backend + +To create the tfbackend file for the new application environment, run + +```bash +make infra-configure-app-database APP_NAME= ENVIRONMENT= +``` + +`APP_NAME` needs to be the name of the application folder within the `infra` folder. By default, this is `app`. +`ENVIRONMENT` needs to be the name of the environment you are creating. This will create a file called `.s3.tfbackend` in the `infra/app/service` module directory. + +## 2. Create database resources + +Now run the following commands to create the resources. Review the terraform before confirming "yes" to apply the changes. This can take over 5 minutes. + +```bash +make infra-update-app-database APP_NAME=app ENVIRONMENT= +``` + +## 3. Create Postgres users + +Trigger the role manager Lambda function that was created in the previous step in order to create the application and migrator Postgres users. + +```bash +make infra-update-app-database-roles APP_NAME=app ENVIRONMENT= +``` + +The Lambda function's response should describe the resulting PostgreSQL roles and groups that are configured in the database. It should look like a minified version of the following: + +```json +{ + "roles": [ + "postgres", + "migrator", + "app" + ], + "roles_with_groups": { + "rds_superuser": "rds_password", + "pg_monitor": "pg_read_all_settings,pg_read_all_stats,pg_stat_scan_tables", + "postgres": "rds_superuser", + "app": "rds_iam", + "migrator": "rds_iam" + }, + "schema_privileges": { + "public": "{postgres=UC/postgres,=UC/postgres}", + "app": "{migrator=UC/migrator,app=U/migrator}" + } +} +``` + +### Important note on Postgres table permissions + +Before creating migrations that create tables, first create a migration that includes the following SQL command (or equivalent if your migrations are written in a general purpose programming language): + +```sql +ALTER DEFAULT PRIVILEGES GRANT ALL ON TABLES TO app +``` + +This will cause all future tables created by the `migrator` user to automatically be accessible by the `app` user. See the [Postgres docs on ALTER DEFAULT PRIVILEGES](https://www.postgresql.org/docs/current/sql-alterdefaultprivileges.html) for more info. As an example see the example app's migrations file [migrations.sql](https://github.com/navapbc/template-infra/blob/main/app/migrations.sql). + +Why is this needed? The reason is because the `migrator` role will be used by the migration task to run database migrations (creating tables, altering tables, etc.), while the `app` role will be used by the web service to access the database. Moreover, in Postgres, new tables won't automatically be accessible by roles other than the creator unless specifically granted, even if those other roles have usage access to the schema that the tables are created in. In other words if the `migrator` user created a new table `foo` in the `app` schema, the `app` user will not have automatically be able to access it by default. + +## 4. Check that database roles have been configured properly + +```bash +make infra-check-app-database-roles APP_NAME=app ENVIRONMENT= +``` + +## Set up application environments + +Once you set up the deployment process, you can proceed to [set up the application service](./set-up-app-env.md) diff --git a/docs/infra/set-up-infrastructure-tools.md b/docs/infra/set-up-infrastructure-tools.md new file mode 100644 index 000000000..e27477f36 --- /dev/null +++ b/docs/infra/set-up-infrastructure-tools.md @@ -0,0 +1,102 @@ +# Set up infrastructure developer tools + +If you are contributing to infrastructure, you will need to complete these setup steps. + +## Prerequisites + +### Install Terraform + +[Terraform](https://www.terraform.io/) is an infrastructure as code (IaC) tool that allows you to build, change, and version infrastructure safely and efficiently. This includes both low-level components like compute instances, storage, and networking, as well as high-level components like DNS entries and SaaS features. + +You may need different versions of Terraform since different projects may require different versions of Terraform. The best way to manage Terraform versions is with [Terraform Version Manager](https://github.com/tfutils/tfenv). + +To install via [Homebrew](https://brew.sh/) + +```bash +brew install tfenv +``` + +Then install the version of Terraform you need. + +```bash +tfenv install 1.4.6 +``` + +If you are unfamiliar with Terraform, check out this [basic introduction to Terraform](./intro-to-terraform.md). + +### Install AWS CLI + +The [AWS Command Line Interface (AWS CLI)](https://aws.amazon.com/cli/) is a unified tool to manage your AWS services. With just one tool to download and configure, you can control multiple AWS services from the command line and automate them through scripts. Install the AWS commmand line tool by following the instructions found here: + +- [Install AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) + +### Install Go + +The [Go programming language](https://go.dev/dl/) is required to run [Terratest](https://terratest.gruntwork.io/), the unit test framework for Terraform. + +### Install GitHub CLI + +The [GitHub CLI](https://cli.github.com/) is useful for automating certain operations for GitHub such as with GitHub actions. This is needed to run [check-github-actions-auth.sh](/bin/check-github-actions-auth.sh) + +```bash +brew install gh +``` + +### Install linters + +We have several optional utilities for running infrastructure linters locally. These are run as part of the CI pipeline, therefore, it is often simpler to test them locally first. + +* [Shellcheck](https://github.com/koalaman/shellcheck) +* [actionlint](https://github.com/rhysd/actionlint) +* [markdown-link-check](https://github.com/tcort/markdown-link-check) + +```bash +brew install shellcheck +brew install actionlint +make infra-lint +``` + +## AWS Authentication + +In order for Terraform to authenticate with your accounts you will need to configure your aws credentials using the AWS CLI or manually create your config and credentials file. If you need to manage multiple credentials or create named profiles for use with different environments you can add the `--profile` option. + +There are multiple ways to authenticate, but we recommend creating a separate profile for your project in your AWS credentials file, and setting your local environment variable `AWS_PROFILE` to the profile name. We recommend using [direnv](https://direnv.net/) to manage local environment variables. +**Credentials should be located in ~/.aws/credentials** (Linux & Mac) or **%USERPROFILE%\.aws\credentials** (Windows) + +### Examples + +```bash +$ aws configure +AWS Access Key ID [None]: AKIAIOSFODNN7EXAMPLE +AWS Secret Access Key [None]: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +Default region name [None]: us-east-2 +Default output format [None]: json +``` + +**Using the above command will create a [default] profile.** + +```bash +$ aws configure --profile dev +AWS Access Key ID [None]: AKIAIOSFODNN7EXAMPLE +AWS Secret Access Key [None]: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +Default region name [None]: us-east-2 +Default output format [None]: json +``` + +**Using the above command will create a [dev] profile.** + +Once you're done, verify access by running the following command to print out information about the AWS IAM user you authenticated as. + +```bash +aws sts get-caller-identity +``` + +### References + +- [Configuration basics][1] +- [Named profiles for the AWS CLI][2] +- [Configuration and credential file settings][3] + +[1]: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html +[2]: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html +[3]: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html diff --git a/docs/infra/set-up-monitoring-alerts.md b/docs/infra/set-up-monitoring-alerts.md new file mode 100644 index 000000000..0619d529e --- /dev/null +++ b/docs/infra/set-up-monitoring-alerts.md @@ -0,0 +1,29 @@ +# Set up monitoring notifications + +## Overview + +The monitoring module defines metric-based alerting policies that provides awareness into issues with the cloud application. The module supports integration with external incident management tools like Splunk-On-Call or Pagerduty. It also supports email alerts. + +### Set up email alerts. + +1. Add the `email_alerts_subscription_list` variable to the monitoring module call in the service layer + +For example: +``` +module "monitoring" { + source = "../../modules/monitoring" + email_alerts_subscription_list = ["email1@email.com", "email2@email.com"] + ... +} +``` +2. Run `make infra-update-app-service APP_NAME= ENVIRONMENT=` to apply the changes to each environment. +When any of the alerts described by the module are triggered notification will be send to all email specified in the `email_alerts_subscription_list` + +### Set up External incident management service integration. + +1. Set setting `has_incident_management_service = true` in app-config/main.tf +2. Get the integration URL for the incident management service and store it in AWS SSM Parameter Store by running the following command for each environment: +``` +make infra-configure-monitoring-secrets APP_NAME= ENVIRONMENT= URL= +``` +3. Run `make infra-update-app-service APP_NAME= ENVIRONMENT=` to apply the changes to each environment. diff --git a/docs/infra/set-up-network.md b/docs/infra/set-up-network.md new file mode 100644 index 000000000..0e90a07da --- /dev/null +++ b/docs/infra/set-up-network.md @@ -0,0 +1,38 @@ +# Set up network + +The network setup process will configure and deploy network resources needed by other modules. In particular, it will: + +1. Create a nondefault VPC +2. Create public subnets for publicly accessible resources such as the application load balancer, private subnets for the application service, and private subnets for the database. +3. Create VPC endpoints for the AWS services needed by ECS Fargate to fetch the container image and log to AWS CloudWatch. If your application has a database, it will also create VPC endpoints for the AWS services needed by the database layer and a security group to contain those VPC endpoints. + +## Requirements + +Before setting up the network you'll need to have: + +1. [Set up the AWS account](./set-up-aws-account.md) +2. Optionally adjust the configuration for the networks you want to have on your project in the [project-config module](/infra/project-config/main.tf). By default there are three networks defined, one for each application environment. If you have multiple apps and want your applications in separate networks, you may want to give the networks differentiating names (e.g. "foo-dev", "foo-prod", "bar-dev", "bar-prod", instead of just "dev", "prod"). +3. [Configure the app](/infra/app/app-config/main.tf). + 1. Update `has_database` to `true` or `false` depending on whether or not your application has a database to integrate with. This setting determines whether or not to create VPC endpoints needed by the database layer. + 2. Update `has_external_non_aws_service` to `true` or `false` depending on whether or not your application makes calls to an external non-AWS service. This setting determines whether or not to create NAT gateways, which allows the service in the private subnet to make requests to the internet. + 3. Update `network_name` for your application environments. This mapping ensures that each network is configured appropriately based on the application(s) in that network (see `local.apps_in_network` in [/infra/networks/main.tf](/infra/networks/main.tf)) Failure to set the network name properly means that the network layer may not receive the correct application configurations for `has_database` and `has_external_non_aws_service`. + +## 1. Configure backend + +To create the tfbackend file for the new network, run + +```bash +make infra-configure-network NETWORK_NAME= +``` + +## 2. Create network resources + +Now run the following commands to create the resources. Review the terraform before confirming "yes" to apply the changes. + +```bash +make infra-update-network NETWORK_NAME= +``` + +## Updating the network + +If you make changes to your application's configuration that impacts the network (such as `has_database` and `has_external_non_aws_service`), make sure to update the network before you update or deploy subsequent infrastructure layers. diff --git a/docs/infra/vulnerability-management.md b/docs/infra/vulnerability-management.md new file mode 100644 index 000000000..ac5693449 --- /dev/null +++ b/docs/infra/vulnerability-management.md @@ -0,0 +1,35 @@ +# Vulnerability Management for Docker Images +This repository contains a GitHub workflow that allows you to scan Docker images for vulnerabilities. The workflow, named `ci-vulnerability-scans` is located in the directory `.github/workflows`. The goal in scanning the image before pushing it to the repository is so that you can catch any vulnerabilities before deploying the image, ECR scanning takes time and the image can still be used even with vulnerabilities found by Inspector. Also, if you use `scratch` as a base image, ECR is unable to scan the image when it is pushed, which is a known issue. + +A way to ensure that there are smaller surface areas for vulnerabilities, follow this method of building images +- Build base image with required packages, name it something like `build` +- Configure app build from the image in the previous step, name it something like `app-build` +- Create a final image from `scratch` named `release` (ie `from scratch as release`), and copy any needed directories from the `app-build` image + +``` +FROM ... AS build +# Do base installs for dev and app-build here +FROM build AS dev +# Local dev installs only +FROM build AS app-build +# All installs for the release image +# Any tweaks needed for the release image +FROM scratch AS release +# Copy over the files from app-build +# COPY --from=app-build /app-build/paths/to/files /release/paths/to/files +``` + +By following this method, your deployment image will have the minimum required directories and files, it will shrink the overall image size, and reduce findings + +## How to use Workflow +The workflow will run whenever there is a push to a PR or when merged to main if there are changes in the `app` direcotry. It is scanning in both cases to ensure there is no issues if a PR is approved on a Friday, but isn't merged till Monday - a CVE could have been found in the time between the last run and the merge. + +## Notes about Scanners +### Hadolint +The hadolint scanner allows you to ignore or safelist certain findings, which can be specified in the [.hadolint.yaml](../../.hadolint.yaml) file. There is a template file here that you can use in your repo. +### Trivy +The trivy scanner allows you to ignore or safelist certain findings, which can be specified in the [.trivyignore](../../.trivyignore) file. There is a template file here that you can use in your repo. +### Anchore +The anchore scanner allows you to ignore or safelist certain findings, which can be specified in the [.grype.yml](../../.grype.yml) file. There is a template file here that you can use in your repo. There are flags set to ignore findings that are in the state `not-fixed`, `wont-fix`, and `unknown`. +### Dockle +The dockle scanner action does not have the ability to use an ignore or safelist findings file, but is able to by specifying an allow file, or `DOCKLE_ACCEPT_FILES`, environmental variable. To get around this, there is a step before the dockle scan is ran to check for a file named [.dockleconfig](../../.dockleconfig), and pipe it to the environmental variable if it exists. Note that this will not ignore finding types like the other scanner's ignore file, but ignore the file specified in the list \ No newline at end of file diff --git a/docs/releases.md b/docs/releases.md new file mode 100644 index 000000000..68a05e62b --- /dev/null +++ b/docs/releases.md @@ -0,0 +1,21 @@ +# Release Management + +## Building a release + +To build a release, run + +```bash +make release-build +``` + +This builds the release from [app/Dockerfile](../app/Dockerfile). The Dockerfile +needs to have a build stage called `release` to act as the build target. +(See [Name your build stages](https://docs.docker.com/build/building/multi-stage/#name-your-build-stages)) + +## Publishing a release + +TODO + +## Deploying a release + +TODO diff --git a/docs/system-architecture.md b/docs/system-architecture.md new file mode 100644 index 000000000..ba22b5f55 --- /dev/null +++ b/docs/system-architecture.md @@ -0,0 +1,21 @@ +# System Architecture + +This diagram shows the system architecture. [🔒 Make a copy of this Lucid template for your own application](https://lucid.app/lucidchart/8851888e-1292-4228-8fef-60a61c6b57e7/edit). + +![System architecture](https://lucid.app/publicSegments/view/e5a36152-200d-4d95-888e-4cdbdab80d1b/image.png) + +* **Access Logs** — Amazon S3 bucket storing the application service's access logs. +* **Alarms SNS Topic** — SNS topic that notifies the incident management service when an alarm triggers. +* **Application Load Balancer** — Amazon application load balancer. +* **Aurora PostgreSQL Database** — Amazon Aurora Serverless PostgreSQL database used by the application. +* **Build Repository ECR Registry** — Amazon ECR registry that acts as the build repository of application container images. +* **CloudWatch Alarms** — Amazon CloudWatch Alarms that trigger on errors and latency. +* **CloudWatch Evidently Feature Flags** — Amazon CloudWatch Evidently service that manages feature flags used by the application to manage feature launches. +* **Database Role Manager** — AWS Lambda serverless function that provisions the database roles needed by the application. +* **GitHub** — Source code repository. Also responsible for Continuous Integration (CI) and Continuous Delivery (CD) workflows. GitHub Actions builds and deploys releases to an Amazon ECR registry that stores Docker container images for the application service. +* **Incident Management Service** — Incident management service (e.g. PagerDuty or Splunk On-Call) for managing on-call schedules and paging engineers for urgent production issues. +* **Service** — Amazon ECS service running the application. +* **Terraform Backend Bucket** — Amazon S3 bucket used to store terraform state files. +* **Terraform Locks DynamoDB Table** — Amazon DynamoDB table used to manage concurrent access to terraform state files. +* **VPC Endpoints** — VPC endpoints are used by the Database Role Manager to access Amazon Services without traffic leaving the VPC. +* **VPC Network** — Amazon VPC network. diff --git a/infra/README.md b/infra/README.md index c83a298ee..30005fc0e 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,6 +1,81 @@ -# Simpler Grants Infrastructure +# Overview -The infrastructure for this project is stored as Terraform files and hosted on Amazon Web Services. +This project practices infrastructure-as-code and uses the [Terraform framework](https://www.terraform.io). This directory contains the infrastructure code for this project, including infrastructure for all application resources. This terraform project uses the [AWS provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs). It is based on the [Nava platform infrastructure template](https://github.com/navapbc/template-infra). -- See the [infra documentation](../documentation/infra/module-architecture.md) for details about the Terraform modules and architecture. -- See the [Cloud Hosting](../documentation/decisions/adr/2023-08-21-cloud-platform.md) and [Infrastructure as Code](../documentation/decisions/adr/2023-08-21-infrastructure-as-code-tool.md) ADRs for information on the decisions that informed this architecture. +## 📂 Directory structure + +The structure for the infrastructure code looks like this: + +```text +infra/ Infrastructure code + accounts/ [Root module] IaC and IAM resources + [app_name]/ Application directory: infrastructure for the main application + modules/ Reusable child modules + networks/ [Root module] Account level network config (shared across all apps, environments, and terraform workspaces) +``` + +Each application directory contains the following: + +```text + app-config/ Application-level configuration for the application resources (different config for different environments) + build-repository/ [Root module] Docker image repository for the application (shared across environments and terraform workspaces) + database/ [Root module] Configuration for database (different config for different environments) + service/ [Root module] Configuration for containers, such as load balancer, application service (different config for different environments) +``` + +Details about terraform root modules and child modules are documented in [module-architecture](/docs/infra/module-architecture.md). + +## 🏗️ Project architecture + +### 🧅 Infrastructure layers + +The infrastructure template is designed to operate on different layers: + +- Account layer +- Network layer +- Build repository layer (per application) +- Database layer (per application) +- Service layer (per application) + +### 🏜️ Application environments + +This project has the following AWS environments: + +- `dev` +- `staging` +- `prod` + +The environments share the same root modules but will have different configurations. Backend configuration is saved as [`.tfbackend`](https://developer.hashicorp.com/terraform/language/settings/backends/configuration#file) files. Most `.tfbackend` files are named after the environment. For example, the `[app_name]/service` infrastructure resources for the `dev` environment are configured via `dev.s3.tfbackend`. Resources for a module that are shared across environments, such as the build-repository, use `shared.s3.tfbackend`. Resources that are shared across the entire account (e.g. /infra/accounts) use `..s3.tfbackend`. + +### 🔀 Project workflow + +This project relies on Make targets in the [root Makefile](/Makefile), which in turn call shell scripts in [./bin](/bin). The shell scripts call terraform commands. Many of the shell scripts are also called by the [Github Actions CI/CD](/.github/workflows). + +Generally you should use the Make targets or the underlying bin scripts, but you can call the underlying terraform commands if needed. See [making-infra-changes](/docs/infra/making-infra-changes.md) for more details. + +## 💻 Development + +### 1️⃣ First time initialization + +To set up this project for the first time (aka it has never been deployed to the target AWS account): + +1. [Configure the project](/infra/project-config/main.tf) (These values will be used in subsequent infra setup steps to namespace resources and add infrastructure tags.) +2. [Set up infrastructure developer tools](/docs/infra/set-up-infrastructure-tools.md) +3. [Set up AWS account](/docs/infra/set-up-aws-account.md) +4. [Set up the virtual network (VPC)](/docs/infra/set-up-network.md) +5. For each application: + 1. [Set up application build repository](/docs/infra/set-up-app-build-repository.md) + 2. [Set up application database](/docs/infra/set-up-database.md) + 3. [Set up application environment](/docs/infra/set-up-app-env.md) + +### 🆕 New developer + +To get set up as a new developer to a project that has already been deployed to the target AWS account: + +1. [Set up infrastructure developer tools](/docs/infra/set-up-infrastructure-tools.md) +2. [Review how to make changes to infrastructure](/docs/infra/making-infra-changes.md) +3. (Optional) Set up a [terraform workspace](/docs/infra/intro-to-terraform-workspaces.md) + +## 📇 Additional reading + +Additional documentation can be found in the [documentation directory](/docs/infra). diff --git a/infra/accounts/main.tf b/infra/accounts/main.tf index 1ed35c5df..247ba1433 100644 --- a/infra/accounts/main.tf +++ b/infra/accounts/main.tf @@ -8,7 +8,7 @@ locals { # Choose the region where this infrastructure should be deployed. region = module.project_config.default_region - # Set project tags that will be used to tag all resources. + # Set project tags that will be used to tag all resources. tags = merge(module.project_config.default_tags, { description = "Backend resources required for terraform state management and GitHub authentication with AWS." }) diff --git a/infra/app/app-config/dev.tf b/infra/app/app-config/dev.tf new file mode 100644 index 000000000..22a5ceac9 --- /dev/null +++ b/infra/app/app-config/dev.tf @@ -0,0 +1,9 @@ +module "dev_config" { + source = "./env-config" + app_name = local.app_name + default_region = module.project_config.default_region + environment = "dev" + network_name = "dev" + has_database = local.has_database + has_incident_management_service = local.has_incident_management_service +} diff --git a/infra/app/app-config/env-config/outputs.tf b/infra/app/app-config/env-config/outputs.tf new file mode 100644 index 000000000..09005a1f9 --- /dev/null +++ b/infra/app/app-config/env-config/outputs.tf @@ -0,0 +1,30 @@ +output "database_config" { + value = var.has_database ? { + region = var.default_region + cluster_name = "${var.app_name}-${var.environment}" + app_username = "app" + migrator_username = "migrator" + schema_name = var.app_name + app_access_policy_name = "${var.app_name}-${var.environment}-app-access" + migrator_access_policy_name = "${var.app_name}-${var.environment}-migrator-access" + } : null +} + +output "network_name" { + value = var.network_name +} + +output "service_config" { + value = { + region = var.default_region + cpu = var.service_cpu + memory = var.service_memory + desired_instance_count = var.service_desired_instance_count + } +} + +output "incident_management_service_integration" { + value = var.has_incident_management_service ? { + integration_url_param_name = "/monitoring/${var.app_name}/${var.environment}/incident-management-integration-url" + } : null +} diff --git a/infra/app/app-config/env-config/variables.tf b/infra/app/app-config/env-config/variables.tf new file mode 100644 index 000000000..712cea1bb --- /dev/null +++ b/infra/app/app-config/env-config/variables.tf @@ -0,0 +1,41 @@ +variable "app_name" { + type = string +} + +variable "environment" { + description = "name of the application environment (e.g. dev, staging, prod)" + type = string +} + +variable "network_name" { + description = "Human readable identifier of the network / VPC" + type = string +} + +variable "default_region" { + description = "default region for the project" + type = string +} + +variable "has_database" { + type = bool +} + +variable "has_incident_management_service" { + type = bool +} + +variable "service_cpu" { + type = number + default = 256 +} + +variable "service_memory" { + type = number + default = 512 +} + +variable "service_desired_instance_count" { + type = number + default = 1 +} diff --git a/infra/app/app-config/main.tf b/infra/app/app-config/main.tf new file mode 100644 index 000000000..93fd896d5 --- /dev/null +++ b/infra/app/app-config/main.tf @@ -0,0 +1,75 @@ +locals { + app_name = "app" + environments = ["dev", "staging", "prod"] + project_name = module.project_config.project_name + image_repository_name = "${local.project_name}-${local.app_name}" + + # Whether or not the application has a database + # If enabled: + # 1. The networks associated with this application's environments will have + # VPC endpoints needed by the database layer + # 2. Each environment's config will have a database_config property that is used to + # pass db_vars into the infra/modules/service module, which provides the necessary + # configuration for the service to access the database + has_database = false + + # Whether or not the application depends on external non-AWS services. + # If enabled, the networks associated with this application's environments + # will have NAT gateways, which allows the service in the private subnet to + # make calls to the internet. + has_external_non_aws_service = false + + has_incident_management_service = false + + feature_flags = ["foo", "bar"] + + environment_configs = { + dev = module.dev_config + staging = module.staging_config + prod = module.prod_config + } + + build_repository_config = { + region = module.project_config.default_region + } + + # Map from environment name to the account name for the AWS account that + # contains the resources for that environment. Resources that are shared + # across environments use the key "shared". + # The list of configured AWS accounts can be found in /infra/account + # by looking for the backend config files of the form: + # ..s3.tfbackend + # + # Projects/applications that use the same AWS account for all environments + # will refer to the same account for all environments. For example, if the + # project has a single account named "myaccount", then infra/accounts will + # have one tfbackend file myaccount.XXXXX.s3.tfbackend, and the + # account_names_by_environment map will look like: + # + # account_names_by_environment = { + # shared = "myaccount" + # dev = "myaccount" + # staging = "myaccount" + # prod = "myaccount" + # } + # + # Projects/applications that have separate AWS accounts for each environment + # might have a map that looks more like this: + # + # account_names_by_environment = { + # shared = "dev" + # dev = "dev" + # staging = "staging" + # prod = "prod" + # } + account_names_by_environment = { + shared = "dev" + dev = "dev" + staging = "dev" + prod = "prod" + } +} + +module "project_config" { + source = "../../project-config" +} diff --git a/infra/app/app-config/outputs.tf b/infra/app/app-config/outputs.tf new file mode 100644 index 000000000..4da2bc30d --- /dev/null +++ b/infra/app/app-config/outputs.tf @@ -0,0 +1,39 @@ +output "app_name" { + value = local.app_name +} + +output "account_names_by_environment" { + value = local.account_names_by_environment +} + +output "environments" { + value = local.environments +} + +output "feature_flags" { + value = local.feature_flags +} + +output "has_database" { + value = local.has_database +} + +output "has_external_non_aws_service" { + value = local.has_external_non_aws_service +} + +output "has_incident_management_service" { + value = local.has_incident_management_service +} + +output "image_repository_name" { + value = local.image_repository_name +} + +output "build_repository_config" { + value = local.build_repository_config +} + +output "environment_configs" { + value = local.environment_configs +} diff --git a/infra/app/app-config/prod.tf b/infra/app/app-config/prod.tf new file mode 100644 index 000000000..4a85e9349 --- /dev/null +++ b/infra/app/app-config/prod.tf @@ -0,0 +1,16 @@ +module "prod_config" { + source = "./env-config" + app_name = local.app_name + default_region = module.project_config.default_region + environment = "prod" + network_name = "prod" + has_database = local.has_database + has_incident_management_service = local.has_incident_management_service + + # These numbers are a starting point based on this article + # Update the desired instance size and counts based on the project's specific needs + # https://conchchow.medium.com/aws-ecs-fargate-compute-capacity-planning-a5025cb40bd0 + service_cpu = 1024 + service_memory = 4096 + service_desired_instance_count = 3 +} diff --git a/infra/app/app-config/staging.tf b/infra/app/app-config/staging.tf new file mode 100644 index 000000000..59e6dde0b --- /dev/null +++ b/infra/app/app-config/staging.tf @@ -0,0 +1,9 @@ +module "staging_config" { + source = "./env-config" + app_name = local.app_name + default_region = module.project_config.default_region + environment = "staging" + network_name = "staging" + has_database = local.has_database + has_incident_management_service = local.has_incident_management_service +} diff --git a/infra/app/build-repository/.terraform.lock.hcl b/infra/app/build-repository/.terraform.lock.hcl new file mode 100644 index 000000000..43cfe95a2 --- /dev/null +++ b/infra/app/build-repository/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "4.20.1" + constraints = "~> 4.20.1" + hashes = [ + "h1:1JbjdrwUCLTNVVhlE+acEPnJFJ/FqBTHy5Ooll6nwjI=", + "zh:21d064d8fac08376c633e002e2f36e83eb7958535e251831feaf38f51c49dafd", + "zh:3a37912ff43d89ce8d559ec86265d7506801bccb380c7cfb896e8ff24e3fe79d", + "zh:795eb175c85279ec51dbe12e4d1afa0860c2c0b22e5d36a8e8869f60a93b7931", + "zh:8afb61a18b17f8ff249cb23e9d3b5d2530944001ef1d56c1d53f41b0890c7ab8", + "zh:911701040395e0e4da4b7252279e7cf1593cdd26f22835e1a9eddbdb9691a1a7", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a46d54a6a5407f569f8178e916af888b2b268f86448c64cad165dc89759c8399", + "zh:c5f71fd5e3519a24fd6af455ef1c26a559cfdde7f626b0afbd2a73bb79f036b1", + "zh:df3b69d6c9b0cdc7e3f90ee08412b22332c32e97ad8ce6ccad528f89f235a7d3", + "zh:e99d6a64c03549d60c2accf792fa04466cfb317f72e895c8f67eff8a02920887", + "zh:eea7a0df8bcb69925c9ce8e15ef403c8bbf16d46c43e8f5607b116531d1bce4a", + "zh:f6a26ce77f7db1d50ce311e32902fd001fb365e5e45e47a9a5cd59d734c89cb6", + ] +} diff --git a/infra/app/build-repository/main.tf b/infra/app/build-repository/main.tf new file mode 100644 index 000000000..cb2aebb78 --- /dev/null +++ b/infra/app/build-repository/main.tf @@ -0,0 +1,59 @@ +data "aws_iam_role" "github_actions" { + name = module.project_config.github_actions_role_name +} + +locals { + # Set project tags that will be used to tag all resources. + tags = merge(module.project_config.default_tags, { + application = module.app_config.app_name + application_role = "build-repository" + description = "Backend resources required for storing built release candidate artifacts to be used for deploying to environments." + }) + + # Get list of AWS account ids for the application environments that + # will need access to the build repository + app_account_names = values(module.app_config.account_names_by_environment) + account_ids_by_name = data.external.account_ids_by_name.result + app_account_ids = [for account_name in local.app_account_names : local.account_ids_by_name[account_name] if contains(keys(local.account_ids_by_name), account_name)] +} + +terraform { + required_version = ">= 1.2.0, < 2.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~>4.20.1" + } + } + + backend "s3" { + encrypt = "true" + } +} + +provider "aws" { + region = module.app_config.build_repository_config.region + default_tags { + tags = local.tags + } +} + +module "project_config" { + source = "../../project-config" +} + +module "app_config" { + source = "../app-config" +} + +data "external" "account_ids_by_name" { + program = ["../../../bin/account-ids-by-name.sh"] +} + +module "container_image_repository" { + source = "../../modules/container-image-repository" + name = module.app_config.image_repository_name + push_access_role_arn = data.aws_iam_role.github_actions.arn + app_account_ids = local.app_account_ids +} diff --git a/infra/app/database/main.tf b/infra/app/database/main.tf new file mode 100644 index 000000000..d881a35d1 --- /dev/null +++ b/infra/app/database/main.tf @@ -0,0 +1,92 @@ +data "aws_vpc" "network" { + tags = { + project = module.project_config.project_name + network_name = local.environment_config.network_name + } +} + +data "aws_subnets" "database" { + tags = { + project = module.project_config.project_name + network_name = local.environment_config.network_name + subnet_type = "database" + } +} + +locals { + # The prefix key/value pair is used for Terraform Workspaces, which is useful for projects with multiple infrastructure developers. + # By default, Terraform creates a workspace named “default.” If a non-default workspace is not created this prefix will equal “default”, + # if you choose not to use workspaces set this value to "dev" + prefix = terraform.workspace == "default" ? "" : "${terraform.workspace}-" + + # Add environment specific tags + tags = merge(module.project_config.default_tags, { + environment = var.environment_name + description = "Database resources for the ${var.environment_name} environment" + }) + + environment_config = module.app_config.environment_configs[var.environment_name] + database_config = local.environment_config.database_config + network_config = module.project_config.network_configs[local.environment_config.network_name] +} + +terraform { + required_version = ">=1.4.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~>4.67.0" + } + } + + backend "s3" { + encrypt = "true" + } +} + +provider "aws" { + region = local.database_config.region + default_tags { + tags = local.tags + } +} + +module "project_config" { + source = "../../project-config" +} + +module "app_config" { + source = "../app-config" +} + +data "aws_security_groups" "aws_services" { + filter { + name = "group-name" + values = ["${module.project_config.aws_services_security_group_name_prefix}*"] + } + + filter { + name = "vpc-id" + values = [data.aws_vpc.network.id] + } +} + +module "database" { + source = "../../modules/database" + + name = "${local.prefix}${local.database_config.cluster_name}" + app_access_policy_name = "${local.prefix}${local.database_config.app_access_policy_name}" + migrator_access_policy_name = "${local.prefix}${local.database_config.migrator_access_policy_name}" + + # The following are not AWS infra resources and therefore do not need to be + # isolated via the terraform workspace prefix + app_username = local.database_config.app_username + migrator_username = local.database_config.migrator_username + schema_name = local.database_config.schema_name + + vpc_id = data.aws_vpc.network.id + database_subnet_group_name = local.network_config.database_subnet_group_name + private_subnet_ids = data.aws_subnets.database.ids + aws_services_security_group_id = data.aws_security_groups.aws_services.ids[0] +} diff --git a/infra/app/database/outputs.tf b/infra/app/database/outputs.tf new file mode 100644 index 000000000..927b820a9 --- /dev/null +++ b/infra/app/database/outputs.tf @@ -0,0 +1,3 @@ +output "role_manager_function_name" { + value = module.database.role_manager_function_name +} diff --git a/infra/app/database/variables.tf b/infra/app/database/variables.tf new file mode 100644 index 000000000..c142bdf97 --- /dev/null +++ b/infra/app/database/variables.tf @@ -0,0 +1,4 @@ +variable "environment_name" { + type = string + description = "name of the application environment" +} diff --git a/infra/app/service/image_tag.tf b/infra/app/service/image_tag.tf new file mode 100644 index 000000000..ce1bc75c2 --- /dev/null +++ b/infra/app/service/image_tag.tf @@ -0,0 +1,56 @@ +# Make the "image_tag" variable optional so that "terraform plan" +# and "terraform apply" work without any required variables. +# +# This works as follows: + +# 1. Accept an optional variable during a terraform plan/apply. (see "image_tag" variable in variables.tf) + +# 2. Read the output used from the last terraform state using "terraform_remote_state". +# Get the backend config by parsing the backend config file +locals { + backend_config_file_path = "${path.module}/${var.environment_name}.s3.tfbackend" + backend_config_file = file("${path.module}/${var.environment_name}.s3.tfbackend") + + # Use regex to parse backend config file to get a map of variables to their + # defined values since there is no built-in terraform function that does that + # + # The backend config file consists of lines that look like + # = " match[1] } + tfstate_bucket = local.backend_config["bucket"] + tfstate_key = local.backend_config["key"] +} +data "terraform_remote_state" "current_image_tag" { + # Don't do a lookup if image_tag is provided explicitly. + # This saves some time and also allows us to do a first deploy, + # where the tfstate file does not yet exist. + count = var.image_tag == null ? 1 : 0 + backend = "s3" + + config = { + bucket = local.tfstate_bucket + key = local.tfstate_key + region = local.service_config.region + } + + defaults = { + image_tag = null + } +} + +# 3. Prefer the given variable if provided, otherwise default to the value from last time. +locals { + image_tag = (var.image_tag == null + ? data.terraform_remote_state.current_image_tag[0].outputs.image_tag + : var.image_tag) +} + +# 4. Store the final value used as a terraform output for next time. +output "image_tag" { + value = local.image_tag +} diff --git a/infra/app/service/main.tf b/infra/app/service/main.tf new file mode 100644 index 000000000..8dfb8b09a --- /dev/null +++ b/infra/app/service/main.tf @@ -0,0 +1,169 @@ +data "aws_vpc" "network" { + tags = { + project = module.project_config.project_name + network_name = local.environment_config.network_name + } +} + +data "aws_subnets" "public" { + tags = { + project = module.project_config.project_name + network_name = local.environment_config.network_name + subnet_type = "public" + } +} + +data "aws_subnets" "private" { + tags = { + project = module.project_config.project_name + network_name = local.environment_config.network_name + subnet_type = "private" + } +} + +locals { + # The prefix key/value pair is used for Terraform Workspaces, which is useful for projects with multiple infrastructure developers. + # By default, Terraform creates a workspace named “default.” If a non-default workspace is not created this prefix will equal “default”, + # if you choose not to use workspaces set this value to "dev" + prefix = terraform.workspace == "default" ? "" : "${terraform.workspace}-" + + # Add environment specific tags + tags = merge(module.project_config.default_tags, { + environment = var.environment_name + description = "Application resources created in ${var.environment_name} environment" + }) + + service_name = "${local.prefix}${module.app_config.app_name}-${var.environment_name}" + + # Include project name in bucket name since buckets need to be globally unique across AWS + bucket_name = "${local.prefix}${module.project_config.project_name}-${module.app_config.app_name}-${var.environment_name}" + + environment_config = module.app_config.environment_configs[var.environment_name] + service_config = local.environment_config.service_config + database_config = local.environment_config.database_config + incident_management_service_integration_config = local.environment_config.incident_management_service_integration +} + +terraform { + required_version = ">= 1.2.0, < 2.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.56.0, < 5.0.0" + } + } + + backend "s3" { + encrypt = "true" + } +} + +provider "aws" { + region = local.service_config.region + default_tags { + tags = local.tags + } +} + +module "project_config" { + source = "../../project-config" +} + +module "app_config" { + source = "../app-config" +} + +data "aws_rds_cluster" "db_cluster" { + count = module.app_config.has_database ? 1 : 0 + cluster_identifier = local.database_config.cluster_name +} + +data "aws_iam_policy" "app_db_access_policy" { + count = module.app_config.has_database ? 1 : 0 + name = local.database_config.app_access_policy_name +} + +data "aws_iam_policy" "migrator_db_access_policy" { + count = module.app_config.has_database ? 1 : 0 + name = local.database_config.migrator_access_policy_name +} + +# Retrieve url for external incident management tool (e.g. Pagerduty, Splunk-On-Call) + +data "aws_ssm_parameter" "incident_management_service_integration_url" { + count = module.app_config.has_incident_management_service ? 1 : 0 + name = local.incident_management_service_integration_config.integration_url_param_name +} + +data "aws_security_groups" "aws_services" { + filter { + name = "group-name" + values = ["${module.project_config.aws_services_security_group_name_prefix}*"] + } + + filter { + name = "vpc-id" + values = [data.aws_vpc.network.id] + } +} + +module "service" { + source = "../../modules/service" + service_name = local.service_name + image_repository_name = module.app_config.image_repository_name + image_tag = local.image_tag + vpc_id = data.aws_vpc.network.id + public_subnet_ids = data.aws_subnets.public.ids + private_subnet_ids = data.aws_subnets.private.ids + + cpu = local.service_config.cpu + memory = local.service_config.memory + desired_instance_count = local.service_config.desired_instance_count + + aws_services_security_group_id = data.aws_security_groups.aws_services.ids[0] + + db_vars = module.app_config.has_database ? { + security_group_ids = data.aws_rds_cluster.db_cluster[0].vpc_security_group_ids + app_access_policy_arn = data.aws_iam_policy.app_db_access_policy[0].arn + migrator_access_policy_arn = data.aws_iam_policy.migrator_db_access_policy[0].arn + connection_info = { + host = data.aws_rds_cluster.db_cluster[0].endpoint + port = data.aws_rds_cluster.db_cluster[0].port + user = local.database_config.app_username + db_name = data.aws_rds_cluster.db_cluster[0].database_name + schema_name = local.database_config.schema_name + } + } : null + + extra_environment_variables = [ + { name : "FEATURE_FLAGS_PROJECT", value : module.feature_flags.evidently_project_name }, + { name : "BUCKET_NAME", value : local.bucket_name } + ] + extra_policies = { + feature_flags_access = module.feature_flags.access_policy_arn, + storage_access = module.storage.access_policy_arn + } +} + +module "monitoring" { + source = "../../modules/monitoring" + #Email subscription list: + #email_alerts_subscription_list = ["email1@email.com", "email2@email.com"] + + # Module takes service and ALB names to link all alerts with corresponding targets + service_name = local.service_name + load_balancer_arn_suffix = module.service.load_balancer_arn_suffix + incident_management_service_integration_url = module.app_config.has_incident_management_service ? data.aws_ssm_parameter.incident_management_service_integration_url[0].value : null +} + +module "feature_flags" { + source = "../../modules/feature-flags" + service_name = local.service_name + feature_flags = module.app_config.feature_flags +} + +module "storage" { + source = "../../modules/storage" + name = local.bucket_name +} diff --git a/infra/app/service/outputs.tf b/infra/app/service/outputs.tf new file mode 100644 index 000000000..fe319eabf --- /dev/null +++ b/infra/app/service/outputs.tf @@ -0,0 +1,24 @@ +output "service_endpoint" { + description = "The public endpoint for the service." + value = module.service.public_endpoint +} + +output "service_cluster_name" { + value = module.service.cluster_name +} + +output "service_name" { + value = local.service_name +} + +output "application_log_group" { + value = module.service.application_log_group +} + +output "application_log_stream_prefix" { + value = module.service.application_log_stream_prefix +} + +output "migrator_role_arn" { + value = module.service.migrator_role_arn +} diff --git a/infra/app/service/variables.tf b/infra/app/service/variables.tf new file mode 100644 index 000000000..19a5f312f --- /dev/null +++ b/infra/app/service/variables.tf @@ -0,0 +1,10 @@ +variable "environment_name" { + type = string + description = "name of the application environment" +} + +variable "image_tag" { + type = string + description = "image tag to deploy to the environment" + default = null +} diff --git a/infra/modules/auth-github-actions/main.tf b/infra/modules/auth-github-actions/main.tf index 0ef271ea5..5fc1651d0 100644 --- a/infra/modules/auth-github-actions/main.tf +++ b/infra/modules/auth-github-actions/main.tf @@ -1,15 +1,6 @@ # Set up GitHub's OpenID Connect provider in AWS account -resource "aws_iam_openid_connect_provider" "github" { - url = "https://token.actions.githubusercontent.com" - client_id_list = ["sts.amazonaws.com"] - - # AWS already trusts the GitHub OIDC identity provider's library of root certificate authorities - # so no thumbprints from intermediate certificates are needed - # At the time of writing (July 12, 2023), the thumbprint_list parameter - # is required to be a non-empty array, so we are passing an array with a dummy string that passes validation - # TODO(https://github.com/navapbc/template-infra/issues/350) Remove this parameter thumbprint_list is no - # longer required (see https://github.com/hashicorp/terraform-provider-aws/issues/32480) - thumbprint_list = ["0000000000000000000000000000000000000000"] +data "aws_iam_openid_connect_provider" "github" { + url = "https://token.actions.githubusercontent.com" } # Create IAM role for GitHub Actions @@ -40,7 +31,7 @@ data "aws_iam_policy_document" "github_assume_role" { principals { type = "Federated" - identifiers = [aws_iam_openid_connect_provider.github.arn] + identifiers = [data.aws_iam_openid_connect_provider.github.arn] } condition { diff --git a/infra/modules/database/backups.tf b/infra/modules/database/backups.tf index bdf8afe61..9beca0bad 100644 --- a/infra/modules/database/backups.tf +++ b/infra/modules/database/backups.tf @@ -41,7 +41,7 @@ resource "aws_backup_selection" "db_backup" { # Role that AWS Backup uses to authenticate when backing up the target resource resource "aws_iam_role" "db_backup_role" { - name_prefix = "${var.name}-db-backup-role-" + name_prefix = "${var.name}-db-backup-" assume_role_policy = data.aws_iam_policy_document.db_backup_policy.json } diff --git a/infra/modules/database/main.tf b/infra/modules/database/main.tf index b5913d75a..bdd2f60de 100644 --- a/infra/modules/database/main.tf +++ b/infra/modules/database/main.tf @@ -6,10 +6,12 @@ locals { primary_instance_name = "${var.name}-primary" role_manager_name = "${var.name}-role-manager" role_manager_package = "${path.root}/role_manager.zip" - # The ARN that represents the users accessing the database are of the format: "arn:aws:rds-db:::dbuser:/"" # See https://aws.amazon.com/blogs/database/using-iam-authentication-to-connect-with-pgadmin-amazon-aurora-postgresql-or-amazon-rds-for-postgresql/ db_user_arn_prefix = "arn:aws:rds-db:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:dbuser:${aws_rds_cluster.db.cluster_resource_id}" + + engine_version = "14.6" + engine_major_version = regex("^\\d+", local.engine_version) } # Database Configuration @@ -25,6 +27,7 @@ resource "aws_rds_cluster" "db" { engine = "aurora-postgresql" engine_mode = "provisioned" + engine_version = local.engine_version database_name = var.database_name port = var.port master_username = local.master_username @@ -32,6 +35,8 @@ resource "aws_rds_cluster" "db" { storage_encrypted = true kms_key_id = aws_kms_key.db.arn + db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.rds_query_logging.name + # checkov:skip=CKV_AWS_128:Auth decision needs to be ironed out # checkov:skip=CKV_AWS_162:Auth decision needs to be ironed out iam_database_authentication_enabled = true @@ -45,6 +50,7 @@ resource "aws_rds_cluster" "db" { min_capacity = 0.5 } + db_subnet_group_name = var.database_subnet_group_name vpc_security_group_ids = [aws_security_group.db.id] enabled_cloudwatch_logs_exports = ["postgresql"] @@ -71,7 +77,7 @@ resource "aws_kms_key" "db" { resource "aws_rds_cluster_parameter_group" "rds_query_logging" { name = var.name - family = "aurora-postgresql14" + family = "aurora-postgresql${local.engine_major_version}" description = "Default cluster parameter group" parameter { diff --git a/infra/modules/database/monitoring.tf b/infra/modules/database/monitoring.tf index ee5a6ca0b..788af8635 100644 --- a/infra/modules/database/monitoring.tf +++ b/infra/modules/database/monitoring.tf @@ -3,7 +3,7 @@ #----------------------------------# resource "aws_iam_role" "rds_enhanced_monitoring" { - name_prefix = "${var.name}-enhanced-monitoring-" + name_prefix = "${var.name}-db-monitor-" assume_role_policy = data.aws_iam_policy_document.rds_enhanced_monitoring.json } diff --git a/infra/modules/database/networking.tf b/infra/modules/database/networking.tf index 4ffedde89..afc0e46ef 100644 --- a/infra/modules/database/networking.tf +++ b/infra/modules/database/networking.tf @@ -13,14 +13,6 @@ resource "aws_security_group" "role_manager" { vpc_id = var.vpc_id } -# We need to attach this security group to our DMS instance when created -resource "aws_security_group" "dms" { - # checkov:skip= CKV2_AWS_5:DMS instance not created yet - name_prefix = "${var.name}-dms" - description = "Database DMS security group" - vpc_id = var.vpc_id -} - resource "aws_vpc_security_group_egress_rule" "role_manager_egress_to_db" { security_group_id = aws_security_group.role_manager.id description = "Allow role manager to access database" @@ -41,26 +33,6 @@ resource "aws_vpc_security_group_ingress_rule" "db_ingress_from_role_manager" { referenced_security_group_id = aws_security_group.role_manager.id } -resource "aws_vpc_security_group_egress_rule" "db_egress_from_dms" { - security_group_id = aws_security_group.db.id - description = "Allow outbound requests to database from DMS" - - from_port = 5432 - to_port = 5432 - ip_protocol = "tcp" - referenced_security_group_id = aws_security_group.dms.id -} - -resource "aws_vpc_security_group_ingress_rule" "db_ingress_from_dms" { - security_group_id = aws_security_group.db.id - description = "Allow inbound requests to database from DMS" - - from_port = 5432 - to_port = 5432 - ip_protocol = "tcp" - referenced_security_group_id = aws_security_group.dms.id -} - resource "aws_vpc_security_group_egress_rule" "role_manager_egress_to_vpc_endpoints" { security_group_id = aws_security_group.role_manager.id description = "Allow outbound requests from role manager to VPC endpoints" diff --git a/infra/modules/database/role-manager.tf b/infra/modules/database/role-manager.tf index 02e16a996..f4aa16d75 100644 --- a/infra/modules/database/role-manager.tf +++ b/infra/modules/database/role-manager.tf @@ -5,6 +5,10 @@ # This includes creating and granting permissions to roles # as well as viewing existing roles +locals { + db_password_param_name = "/aws/reference/secretsmanager/${data.aws_secretsmanager_secret.db_password.name}" +} + resource "aws_lambda_function" "role_manager" { function_name = local.role_manager_name @@ -29,7 +33,8 @@ resource "aws_lambda_function" "role_manager" { DB_PORT = aws_rds_cluster.db.port DB_USER = local.master_username DB_NAME = aws_rds_cluster.db.database_name - DB_PASSWORD_PARAM_NAME = "/aws/reference/secretsmanager/${data.aws_secretsmanager_secret.db_pass.name}" + DB_PASSWORD_PARAM_NAME = local.db_password_param_name + DB_PASSWORD_SECRET_ARN = aws_rds_cluster.db.master_user_secret[0].secret_arn DB_SCHEMA = var.schema_name APP_USER = var.app_username MIGRATOR_USER = var.migrator_username @@ -42,14 +47,14 @@ resource "aws_lambda_function" "role_manager" { tracing_config { mode = "Active" } - + timeout = 30 # checkov:skip=CKV_AWS_272:TODO(https://github.com/navapbc/template-infra/issues/283) # checkov:skip=CKV_AWS_116:Dead letter queue (DLQ) configuration is only relevant for asynchronous invocations } # Installs python packages needed by the role manager lambda function before -# creating the zip archive. +# creating the zip archive. # Runs pip install on every apply so that the role manager archive file that # is generated locally is guaranteed to have the required dependencies even # when terraform is run by a developer that did not originally create the @@ -58,7 +63,7 @@ resource "aws_lambda_function" "role_manager" { resource "terraform_data" "role_manager_python_vendor_packages" { triggers_replace = timestamp() provisioner "local-exec" { - command = "pip3 install -r ${path.module}/role_manager/requirements.txt -t ${path.module}/role_manager/vendor" + command = "pip3 install -r ${path.module}/role_manager/requirements.txt -t ${path.module}/role_manager/vendor --upgrade" } } @@ -79,6 +84,12 @@ resource "aws_kms_key" "role_manager" { enable_key_rotation = true } +data "aws_secretsmanager_secret" "db_password" { + # master_user_secret is available when aws_rds_cluster.db.manage_master_user_password + # is true (see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/rds_cluster#master_user_secret) + arn = aws_rds_cluster.db.master_user_secret[0].secret_arn +} + # IAM for Role Manager lambda function resource "aws_iam_role" "role_manager" { name = "${var.name}-manager" @@ -94,11 +105,9 @@ resource "aws_iam_role" "role_manager" { ] } -data "aws_secretsmanager_secret" "db_pass" { - arn = aws_rds_cluster.db.master_user_secret[0].secret_arn -} -resource "aws_iam_role_policy" "ssm_access" { + +resource "aws_iam_role_policy" "role_manager_access_to_db_password" { name = "${var.name}-role-manager-ssm-access" role = aws_iam_role.role_manager.id @@ -107,13 +116,20 @@ resource "aws_iam_role_policy" "ssm_access" { Statement = [ { Effect = "Allow" - Action = ["secretsmanager:GetSecretValue"] - Resource = [data.aws_secretsmanager_secret.db_pass.arn] + Action = ["kms:Decrypt"] + Resource = [data.aws_kms_key.default_ssm_key.arn] }, { Effect = "Allow" - Action = ["kms:Decrypt"] - Resource = [data.aws_kms_key.default_ssm_key.arn] + Action = ["secretsmanager:GetSecretValue"] + Resource = [data.aws_secretsmanager_secret.db_password.arn] + }, + { + Effect = "Allow" + Action = ["ssm:GetParameter"] + Resource = [ + "arn:aws:ssm:${data.aws_region.current.name}:${data.aws_caller_identity.current.id}:parameter${local.db_password_param_name}" + ] } ] }) diff --git a/infra/modules/database/role_manager/requirements.txt b/infra/modules/database/role_manager/requirements.txt index 94345bbec..a63f23746 100644 --- a/infra/modules/database/role_manager/requirements.txt +++ b/infra/modules/database/role_manager/requirements.txt @@ -1 +1 @@ -pg8000 +pg8000 \ No newline at end of file diff --git a/infra/modules/database/role_manager/role_manager.py b/infra/modules/database/role_manager/role_manager.py index a435e65bd..910da843e 100644 --- a/infra/modules/database/role_manager/role_manager.py +++ b/infra/modules/database/role_manager/role_manager.py @@ -2,6 +2,7 @@ import itertools from operator import itemgetter import os +import json import logging from pg8000.native import Connection, identifier @@ -11,9 +12,6 @@ def lambda_handler(event, context): if event == "check": return check() - elif event == "password_ts": - connect_as_master_user() - return "Succeeded" else: return manage() @@ -61,7 +59,7 @@ def check(): schema_name = os.environ.get("DB_SCHEMA") app_conn = connect_using_iam(app_username) migrator_conn = connect_using_iam(migrator_username) - + check_search_path(migrator_conn, schema_name) check_migrator_create_table(migrator_conn, app_username) check_app_use_table(app_conn) @@ -79,14 +77,14 @@ def check_migrator_create_table(migrator_conn: Connection, app_username: str): logger.info("Checking that migrator is able to create tables and grant access to app user: %s", app_username) migrator_conn.run("CREATE TABLE IF NOT EXISTS temporary(created_at TIMESTAMP)") migrator_conn.run(f"GRANT ALL PRIVILEGES ON temporary TO {identifier(app_username)}") - + def check_app_use_table(app_conn: Connection): logger.info("Checking that app is able to read and write from the table") app_conn.run("INSERT INTO temporary (created_at) VALUES (NOW())") app_conn.run("SELECT * FROM temporary") - - + + def cleanup_migrator_drop_table(migrator_conn: Connection): logger.info("Cleaning up the table that migrator created") migrator_conn.run("DROP TABLE IF EXISTS temporary") @@ -115,9 +113,9 @@ def connect_using_iam(user: str) -> Connection: return Connection(user=user, host=host, port=port, database=database, password=token, ssl_context=True) def get_password() -> str: - ssm = boto3.client("ssm") + ssm = boto3.client("ssm",region_name=os.environ["AWS_REGION"]) param_name = os.environ["DB_PASSWORD_PARAM_NAME"] - logger.info("Fetching password from parameter store") + logger.info("Fetching password from parameter store:\n%s"%param_name) result = json.loads(ssm.get_parameter( Name=param_name, WithDecryption=True, @@ -138,7 +136,7 @@ def get_roles_with_groups(conn: Connection) -> dict[str, str]: INNER JOIN pg_auth_members a ON u.oid = a.member \ INNER JOIN pg_roles g ON g.oid = a.roleid \ ORDER BY user ASC") - + result = {} for user, groups in itertools.groupby(roles_groups, itemgetter(0)): result[user] = ",".join(map(itemgetter(1), groups)) diff --git a/infra/modules/database/variables.tf b/infra/modules/database/variables.tf index 8ec48dc03..8033fa7e6 100644 --- a/infra/modules/database/variables.tf +++ b/infra/modules/database/variables.tf @@ -7,11 +7,6 @@ variable "name" { } } -variable "access_policy_name" { - description = "name of the IAM policy to create that will be provide the ability to connect to the database as a user that will have read/write access." - type = string -} - variable "app_access_policy_name" { description = "name of the IAM policy to create that will provide the service the ability to connect to the database as a user that will have read/write access." type = string @@ -56,6 +51,11 @@ variable "vpc_id" { description = "Uniquely identifies the VPC." } +variable "database_subnet_group_name" { + type = string + description = "Name of database subnet group" +} + variable "private_subnet_ids" { type = list(any) description = "list of private subnet IDs to put the role provisioner and role checker lambda functions in" diff --git a/infra/modules/feature-flags/access-policy.tf b/infra/modules/feature-flags/access-policy.tf new file mode 100644 index 000000000..67b6c7dac --- /dev/null +++ b/infra/modules/feature-flags/access-policy.tf @@ -0,0 +1,21 @@ +resource "aws_iam_policy" "access_policy" { + name = "${local.evidently_project_name}-access" + path = "/" + description = "Allows calls to EvaluateFeature on the Evidently project ${local.evidently_project_name}" + + policy = data.aws_iam_policy_document.access_policy.json +} + +#tfsec:ignore:aws-iam-no-policy-wildcards +data "aws_iam_policy_document" "access_policy" { + statement { + sid = "AllowEvaluateFeature" + effect = "Allow" + actions = [ + "evidently:EvaluateFeature", + ] + resources = [ + "${aws_evidently_project.feature_flags.arn}/feature/*", + ] + } +} diff --git a/infra/modules/feature-flags/logs.tf b/infra/modules/feature-flags/logs.tf new file mode 100644 index 000000000..dc639b723 --- /dev/null +++ b/infra/modules/feature-flags/logs.tf @@ -0,0 +1,47 @@ +data "aws_caller_identity" "current" {} +data "aws_region" "current" {} + +resource "aws_cloudwatch_log_group" "logs" { + name = "feature-flags/${local.evidently_project_name}" + + # checkov:skip=CKV_AWS_158:Feature flag evaluation logs are not sensitive + + # Conservatively retain logs for 5 years. + # Looser requirements may allow shorter retention periods + retention_in_days = 1827 +} + +# Manually create policy allowing AWS services to deliver logs to this log group +# so that the automatically created one by AWS doesn't exceed the character limit +# see https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/AWS-logs-and-resource-policy.html#AWS-vended-logs-permissions +# see https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html#reference_iam-quotas-entity-length +resource "aws_cloudwatch_log_resource_policy" "logs" { + policy_name = "/log-delivery/feature-flags/${local.evidently_project_name}-logs" + policy_document = data.aws_iam_policy_document.logs.json +} + +data "aws_iam_policy_document" "logs" { + statement { + sid = "AWSLogDeliveryWrite" + effect = "Allow" + principals { + type = "Service" + identifiers = ["delivery.logs.amazonaws.com"] + } + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + resources = ["${aws_cloudwatch_log_group.logs.arn}:log-stream:*"] + condition { + test = "StringEquals" + variable = "aws:SourceAccount" + values = [data.aws_caller_identity.current.account_id] + } + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = ["arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*"] + } + } +} diff --git a/infra/modules/feature-flags/main.tf b/infra/modules/feature-flags/main.tf new file mode 100644 index 000000000..3cd11f3e5 --- /dev/null +++ b/infra/modules/feature-flags/main.tf @@ -0,0 +1,49 @@ +locals { + evidently_project_name = "${var.service_name}-flags" +} + +resource "aws_evidently_project" "feature_flags" { + name = local.evidently_project_name + description = "Feature flags for ${var.service_name}" + data_delivery { + cloudwatch_logs { + log_group = aws_cloudwatch_log_group.logs.name + } + } + # Make sure the resource policy is created first so that AWS doesn't try to + # automatically create one + depends_on = [aws_cloudwatch_log_resource_policy.logs] +} + +resource "aws_evidently_feature" "feature_flag" { + for_each = var.feature_flags + + name = each.key + project = aws_evidently_project.feature_flags.name + description = "Enables the ${each.key} feature" + variations { + name = "FeatureOff" + value { + bool_value = false + } + } + variations { + name = "FeatureOn" + value { + bool_value = true + } + } + + # default_variation specifies the variation to use as the default variation. + # Ignore this in terraform to allow business users to enable a feature for all users. + # + # entity_overrides specifies users that should always be served a specific variation of a feature. + # Ignore this in terraform to allow business users and developers to select feature variations + # for testing or pilot purposes. + lifecycle { + ignore_changes = [ + default_variation, + entity_overrides, + ] + } +} diff --git a/infra/modules/feature-flags/outputs.tf b/infra/modules/feature-flags/outputs.tf new file mode 100644 index 000000000..51bb64f5a --- /dev/null +++ b/infra/modules/feature-flags/outputs.tf @@ -0,0 +1,9 @@ +output "evidently_project_name" { + description = "Name of AWS Evidently feature flags project" + value = local.evidently_project_name +} + +output "access_policy_arn" { + description = "Policy that allows access to query feature flag values" + value = aws_iam_policy.access_policy.arn +} diff --git a/infra/modules/feature-flags/variables.tf b/infra/modules/feature-flags/variables.tf new file mode 100644 index 000000000..a04f471bf --- /dev/null +++ b/infra/modules/feature-flags/variables.tf @@ -0,0 +1,9 @@ +variable "service_name" { + type = string + description = "The name of the service that the feature flagging system will be associated with" +} + +variable "feature_flags" { + type = set(string) + description = "A set of feature flag names" +} diff --git a/infra/modules/monitoring/main.tf b/infra/modules/monitoring/main.tf index 65fa3e99a..cdba90d0f 100644 --- a/infra/modules/monitoring/main.tf +++ b/infra/modules/monitoring/main.tf @@ -4,7 +4,7 @@ resource "aws_sns_topic" "this" { name = "${var.service_name}-monitoring" - # checkov:skip=CKV_AWS_26:SNS encryption for alerts is unnecessary + # checkov:skip=CKV_AWS_26:SNS encryption for alerts is unnecessary } # Create CloudWatch alarms for the service diff --git a/infra/modules/network/main.tf b/infra/modules/network/main.tf new file mode 100644 index 000000000..1bfde2558 --- /dev/null +++ b/infra/modules/network/main.tf @@ -0,0 +1,32 @@ +data "aws_availability_zones" "available" {} + +locals { + vpc_cidr = "10.0.0.0/20" + num_availability_zones = 3 + availability_zones = slice(data.aws_availability_zones.available.names, 0, local.num_availability_zones) +} + +module "aws_vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "5.2.0" + + name = var.name + azs = local.availability_zones + cidr = local.vpc_cidr + + public_subnets = ["10.0.10.0/24", "10.0.11.0/24", "10.0.12.0/24"] + private_subnets = ["10.0.0.0/24", "10.0.1.0/24", "10.0.2.0/24"] + database_subnets = ["10.0.5.0/24", "10.0.6.0/24", "10.0.7.0/24"] + public_subnet_tags = { subnet_type = "public" } + private_subnet_tags = { subnet_type = "private" } + database_subnet_tags = { subnet_type = "database" } + database_subnet_group_name = var.database_subnet_group_name + + # If application needs external services, then create one NAT gateway per availability zone + enable_nat_gateway = var.has_external_non_aws_service + single_nat_gateway = false + one_nat_gateway_per_az = var.has_external_non_aws_service + + enable_dns_hostnames = true + enable_dns_support = true +} diff --git a/infra/modules/network/variables.tf b/infra/modules/network/variables.tf new file mode 100644 index 000000000..d57661b8c --- /dev/null +++ b/infra/modules/network/variables.tf @@ -0,0 +1,26 @@ +variable "name" { + type = string + description = "Name to give the VPC. Will be added to the VPC under the 'network_name' tag." +} + +variable "aws_services_security_group_name_prefix" { + type = string + description = "Prefix for the name of the security group attached to VPC endpoints" +} + +variable "database_subnet_group_name" { + type = string + description = "Name of the database subnet group" +} + +variable "has_database" { + type = bool + description = "Whether the application(s) in this network have a database. Determines whether to create VPC endpoints needed by the database layer." + default = false +} + +variable "has_external_non_aws_service" { + type = bool + description = "Whether the application(s) in this network need to call external non-AWS services. Determines whether or not to create NAT gateways." + default = false +} diff --git a/infra/modules/network/vpc-endpoints.tf b/infra/modules/network/vpc-endpoints.tf new file mode 100644 index 000000000..701d57d37 --- /dev/null +++ b/infra/modules/network/vpc-endpoints.tf @@ -0,0 +1,92 @@ +locals { + # List of AWS services used by this VPC + # This list is used to create VPC endpoints so that the AWS services can + # be accessed without network traffic ever leaving the VPC's private network + # For a list of AWS services that integrate with AWS PrivateLink + # see https://docs.aws.amazon.com/vpc/latest/privatelink/aws-services-privatelink-support.html + # + # The database module requires VPC access from private networks to SSM, KMS, and RDS + aws_service_integrations = setunion( + # AWS services used by ECS Fargate: ECR to fetch images, S3 for image layers, and CloudWatch for logs + ["ecr.api", "ecr.dkr", "s3", "logs"], + + # Feature flags with AWS Evidently + ["evidently", "evidently-dataplane"], + + # AWS services used by the database's role manager + var.has_database ? ["ssm", "kms", "secretsmanager"] : [], + ) + + # S3 and DynamoDB use Gateway VPC endpoints. All other services use Interface VPC endpoints + interface_vpc_endpoints = toset([for aws_service in local.aws_service_integrations : aws_service if !contains(["s3", "dynamodb"], aws_service)]) + gateway_vpc_endpoints = toset([for aws_service in local.aws_service_integrations : aws_service if contains(["s3", "dynamodb"], aws_service)]) +} + +data "aws_region" "current" {} + +# VPC Endpoints for accessing AWS Services +# ---------------------------------------- +# +# Since the role manager Lambda function is in the VPC (which is needed to be +# able to access the database) we need to allow the Lambda function to access +# AWS Systems Manager Parameter Store (to fetch the database password) and +# KMS (to decrypt SecureString parameters from Parameter Store). We can do +# this by either allowing internet access to the Lambda, or by using a VPC +# endpoint. The latter is more secure. +# See https://repost.aws/knowledge-center/lambda-vpc-parameter-store +# See https://docs.aws.amazon.com/vpc/latest/privatelink/create-interface-endpoint.html#create-interface-endpoint + +data "aws_subnet" "private" { + count = length(module.aws_vpc.private_subnets) + id = module.aws_vpc.private_subnets[count.index] +} + +# AWS services may only be available in certain regions and availability zones, +# so we use this data source to get that information and only create +# VPC endpoints in the regions / availability zones where the particular service +# is available. +data "aws_vpc_endpoint_service" "aws_service" { + for_each = local.interface_vpc_endpoints + service = each.key +} + +locals { + # Map from the name of an AWS service to a list of the private subnets that are in availability + # zones where the service is available. Only create this map for AWS services where we are going + # to create an Interface VPC endpoint, which require a list of subnet ids in which to create the + # elastic network interface for the endpoint. + aws_service_subnets = { + for service in local.interface_vpc_endpoints : + service => [ + for subnet in data.aws_subnet.private[*] : + subnet.id + if contains(data.aws_vpc_endpoint_service.aws_service[service].availability_zones, subnet.availability_zone) + ] + } +} + +resource "aws_security_group" "aws_services" { + name_prefix = var.aws_services_security_group_name_prefix + description = "VPC endpoints to access AWS services from the VPCs private subnets" + vpc_id = module.aws_vpc.vpc_id +} + +resource "aws_vpc_endpoint" "interface" { + for_each = local.interface_vpc_endpoints + + vpc_id = module.aws_vpc.vpc_id + service_name = "com.amazonaws.${data.aws_region.current.name}.${each.key}" + vpc_endpoint_type = "Interface" + security_group_ids = [aws_security_group.aws_services.id] + subnet_ids = local.aws_service_subnets[each.key] + private_dns_enabled = true +} + +resource "aws_vpc_endpoint" "gateway" { + for_each = local.gateway_vpc_endpoints + + vpc_id = module.aws_vpc.vpc_id + service_name = "com.amazonaws.${data.aws_region.current.name}.${each.key}" + vpc_endpoint_type = "Gateway" + route_table_ids = module.aws_vpc.private_route_table_ids +} diff --git a/infra/modules/service/access-control.tf b/infra/modules/service/access-control.tf index 0b295088f..090310730 100644 --- a/infra/modules/service/access-control.tf +++ b/infra/modules/service/access-control.tf @@ -69,3 +69,10 @@ resource "aws_iam_role_policy" "task_executor" { role = aws_iam_role.task_executor.id policy = data.aws_iam_policy_document.task_executor.json } + +resource "aws_iam_role_policy_attachment" "extra_policies" { + for_each = var.extra_policies + + role = aws_iam_role.app_service.name + policy_arn = each.value +} diff --git a/infra/modules/service/database-access.tf b/infra/modules/service/database-access.tf index a8a7b9186..77ffa9bf8 100644 --- a/infra/modules/service/database-access.tf +++ b/infra/modules/service/database-access.tf @@ -1,5 +1,5 @@ #----------------- -# Database Access +# Database Access #----------------- resource "aws_vpc_security_group_ingress_rule" "db_ingress_from_service" { diff --git a/infra/modules/service/load-balancer.tf b/infra/modules/service/load-balancer.tf index 631c77db1..1b751b2bb 100644 --- a/infra/modules/service/load-balancer.tf +++ b/infra/modules/service/load-balancer.tf @@ -9,7 +9,7 @@ resource "aws_lb" "alb" { idle_timeout = "120" internal = false security_groups = [aws_security_group.alb.id] - subnets = var.subnet_ids + subnets = var.public_subnet_ids # TODO(https://github.com/navapbc/template-infra/issues/163) Implement HTTPS # checkov:skip=CKV2_AWS_20:Redirect HTTP to HTTPS as part of implementing HTTPS support @@ -56,65 +56,8 @@ resource "aws_lb_listener" "alb_listener_http" { } } -resource "aws_lb_listener" "alb_listener_https" { - count = var.cert_arn != null ? 1 : 0 - load_balancer_arn = aws_lb.alb.arn - port = 443 - protocol = "HTTPS" - ssl_policy = "ELBSecurityPolicy-TLS13-1-2-2021-06" - certificate_arn = var.cert_arn - - default_action { - type = "fixed-response" - - fixed_response { - content_type = "text/plain" - message_body = "Not Found" - status_code = "404" - } - } -} - -resource "aws_lb_listener_rule" "redirect_http_to_https" { - count = var.cert_arn != null ? 1 : 0 - listener_arn = aws_lb_listener.alb_listener_http.arn - priority = 100 - - action { - type = "redirect" - - redirect { - port = "443" - protocol = "HTTPS" - status_code = "HTTP_301" - } - } - - condition { - path_pattern { - values = ["/*"] - } - } -} - resource "aws_lb_listener_rule" "app_http_forward" { listener_arn = aws_lb_listener.alb_listener_http.arn - priority = 110 - - action { - type = "forward" - target_group_arn = aws_lb_target_group.app_tg.arn - } - condition { - path_pattern { - values = ["/*"] - } - } -} - -resource "aws_lb_listener_rule" "app_https_forward" { - count = var.cert_arn != null ? 1 : 0 - listener_arn = aws_lb_listener.alb_listener_https[0].arn priority = 100 action { diff --git a/infra/modules/service/main.tf b/infra/modules/service/main.tf index 07a6f887a..5e7ecf2f3 100644 --- a/infra/modules/service/main.tf +++ b/infra/modules/service/main.tf @@ -11,16 +11,12 @@ locals { log_stream_prefix = var.service_name task_executor_role_name = "${var.service_name}-task-executor" image_url = "${data.aws_ecr_repository.app.repository_url}:${var.image_tag}" - hostname = var.hostname != null ? [{ name = "HOSTNAME", value = var.hostname }] : [] - sendy_api_key = var.sendy_api_key != null ? [{ name = "SENDY_API_KEY", value = var.sendy_api_key }] : [] - sendy_api_url = var.sendy_api_url != null ? [{ name = "SENDY_API_URL", value = var.sendy_api_url }] : [] - sendy_list_id = var.sendy_list_id != null ? [{ name = "SENDY_LIST_ID", value = var.sendy_list_id }] : [] - base_environment_variables = concat([ + base_environment_variables = [ { name : "PORT", value : tostring(var.container_port) }, + { name : "AWS_DEFAULT_REGION", value : data.aws_region.current.name }, { name : "AWS_REGION", value : data.aws_region.current.name }, - { name : "API_AUTH_TOKEN", value : var.api_auth_token }, - ], local.hostname, local.sendy_api_key, local.sendy_api_url, local.sendy_list_id) + ] db_environment_variables = var.db_vars == null ? [] : [ { name : "DB_HOST", value : var.db_vars.connection_info.host }, { name : "DB_PORT", value : var.db_vars.connection_info.port }, @@ -28,7 +24,11 @@ locals { { name : "DB_NAME", value : var.db_vars.connection_info.db_name }, { name : "DB_SCHEMA", value : var.db_vars.connection_info.schema_name }, ] - environment_variables = concat(local.base_environment_variables, local.db_environment_variables) + environment_variables = concat( + local.base_environment_variables, + local.db_environment_variables, + var.extra_environment_variables, + ) } #------------------- @@ -49,10 +49,8 @@ resource "aws_ecs_service" "app" { } network_configuration { - # TODO(https://github.com/navapbc/template-infra/issues/152) set assign_public_ip = false after using private subnets - # checkov:skip=CKV_AWS_333:Switch to using private subnets - assign_public_ip = true - subnets = var.subnet_ids + assign_public_ip = false + subnets = var.private_subnet_ids security_groups = [aws_security_group.app.id] } diff --git a/infra/modules/service/networking.tf b/infra/modules/service/networking.tf index f6ccc6166..89dbc5ec7 100644 --- a/infra/modules/service/networking.tf +++ b/infra/modules/service/networking.tf @@ -19,6 +19,16 @@ resource "aws_security_group" "alb" { vpc_id = var.vpc_id + # TODO(https://github.com/navapbc/template-infra/issues/163) Disallow incoming traffic to port 80 + # checkov:skip=CKV_AWS_260:Disallow ingress from 0.0.0.0:0 to port 80 when implementing HTTPS support in issue #163 + ingress { + description = "Allow HTTP traffic from public internet" + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + egress { description = "Allow all outgoing traffic" from_port = 0 @@ -28,32 +38,6 @@ resource "aws_security_group" "alb" { } } -resource "aws_security_group_rule" "http_ingress" { - # TODO(https://github.com/navapbc/template-infra/issues/163) Disallow incoming traffic to port 80 - # checkov:skip=CKV_AWS_260:Disallow ingress from 0.0.0.0:0 to port 80 when implementing HTTPS support in issue #163 - - security_group_id = aws_security_group.alb.id - - description = "Allow HTTP traffic from public internet" - from_port = 80 - to_port = 80 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - type = "ingress" -} - -resource "aws_security_group_rule" "https_ingress" { - count = var.cert_arn != null ? 1 : 0 - security_group_id = aws_security_group.alb.id - - description = "Allow HTTPS traffic from public internet" - from_port = 443 - to_port = 443 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - type = "ingress" -} - # Security group to allow access to Fargate tasks resource "aws_security_group" "app" { # Specify name_prefix instead of name because when a change requires creating a new @@ -65,20 +49,32 @@ resource "aws_security_group" "app" { lifecycle { create_before_destroy = true } +} - ingress { - description = "Allow HTTP traffic to application container port" - protocol = "tcp" - from_port = var.container_port - to_port = var.container_port - security_groups = [aws_security_group.alb.id] - } +resource "aws_vpc_security_group_egress_rule" "service_egress_to_all" { + security_group_id = aws_security_group.app.id + description = "Allow all outgoing traffic from application" - egress { - description = "Allow all outgoing traffic from application" - protocol = "-1" - from_port = 0 - to_port = 0 - cidr_blocks = ["0.0.0.0/0"] - } + ip_protocol = "-1" + cidr_ipv4 = "0.0.0.0/0" +} + +resource "aws_vpc_security_group_ingress_rule" "service_ingress_from_load_balancer" { + security_group_id = aws_security_group.app.id + description = "Allow HTTP traffic to application container port" + + from_port = var.container_port + to_port = var.container_port + ip_protocol = "tcp" + referenced_security_group_id = aws_security_group.alb.id +} + +resource "aws_vpc_security_group_ingress_rule" "vpc_endpoints_ingress_from_service" { + security_group_id = var.aws_services_security_group_id + description = "Allow inbound requests to VPC endpoints from role manager" + + from_port = 443 + to_port = 443 + ip_protocol = "tcp" + referenced_security_group_id = aws_security_group.app.id } diff --git a/infra/modules/service/variables.tf b/infra/modules/service/variables.tf index c3f5b86dc..a405ef3cf 100644 --- a/infra/modules/service/variables.tf +++ b/infra/modules/service/variables.tf @@ -41,38 +41,30 @@ variable "container_port" { default = 8000 } -variable "hostname" { - type = string - description = "The hostname to override the default AWS configuration" - default = null -} - -variable "sendy_api_key" { - description = "Sendy API key to pass with requests for sendy subscriber endpoints." +variable "vpc_id" { type = string - default = null + description = "Uniquely identifies the VPC." } -variable "sendy_api_url" { - description = "Sendy API base url for requests to manage subscribers." - type = string - default = null +variable "public_subnet_ids" { + type = list(any) + description = "Public subnet ids in VPC" } -variable "sendy_list_id" { - description = "Sendy list ID to for requests to manage subscribers to the Simpler Grants distribution list." - type = string - default = null +variable "private_subnet_ids" { + type = list(any) + description = "Private subnet ids in VPC" } -variable "vpc_id" { +variable "aws_services_security_group_id" { type = string - description = "Uniquely identifies the VPC." + description = "Security group ID for VPC endpoints that access AWS Services" } -variable "subnet_ids" { - type = list(any) - description = "Private subnet id from vpc module" +variable "extra_environment_variables" { + type = list(object({ name = string, value = string })) + description = "Additional environment variables to pass to the service container" + default = [] } variable "db_vars" { @@ -92,32 +84,8 @@ variable "db_vars" { default = null } -variable "cert_arn" { - description = "The ARN for the TLS certificate passed in from the app service layer" - type = string - default = null -} - -variable "enable_autoscaling" { - description = "Flag to enable or disable auto-scaling" - type = bool - default = false -} - -variable "max_capacity" { - description = "Maximum number of tasks for autoscaling" - type = number - default = 4 +variable "extra_policies" { + description = "Map of extra IAM policies to attach to the service's task role. The map's keys define the resource name in terraform." + type = map(string) + default = {} } - -variable "min_capacity" { - description = "Minimum number of tasks for autoscaling" - type = number - default = 2 -} - -variable "api_auth_token" { - type = string - default = null - description = "Auth token for connecting to the API" -} \ No newline at end of file diff --git a/infra/modules/storage/access-control.tf b/infra/modules/storage/access-control.tf new file mode 100644 index 000000000..6cba9bb5f --- /dev/null +++ b/infra/modules/storage/access-control.tf @@ -0,0 +1,58 @@ +# Block public access +resource "aws_s3_bucket_public_access_block" "storage" { + bucket = aws_s3_bucket.storage.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +# Bucket policy that requires HTTPS +resource "aws_s3_bucket_policy" "storage" { + bucket = aws_s3_bucket.storage.id + policy = data.aws_iam_policy_document.storage.json +} + +data "aws_iam_policy_document" "storage" { + statement { + sid = "RestrictToTLSRequestsOnly" + effect = "Deny" + actions = ["s3:*"] + resources = [aws_s3_bucket.storage.arn] + principals { + type = "*" + identifiers = ["*"] + } + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + } +} + +# Create policy for read/write access +# Attach this policy to roles that need access to the bucket +resource "aws_iam_policy" "storage_access" { + name = "${var.name}-access" + policy = data.aws_iam_policy_document.storage_access.json +} + +data "aws_iam_policy_document" "storage_access" { + statement { + actions = [ + "s3:DeleteObject", + "s3:GetObject", + "s3:GetObjectAttributes", + "s3:PutObject", + ] + effect = "Allow" + resources = ["arn:aws:s3:::${var.name}/*"] + } + statement { + actions = ["kms:GenerateDataKey"] + effect = "Allow" + resources = [aws_kms_key.storage.arn] + } +} diff --git a/infra/modules/storage/encryption.tf b/infra/modules/storage/encryption.tf new file mode 100644 index 000000000..71c5b58cf --- /dev/null +++ b/infra/modules/storage/encryption.tf @@ -0,0 +1,18 @@ +resource "aws_kms_key" "storage" { + description = "KMS key for bucket ${var.name}" + # The waiting period, specified in number of days. After the waiting period ends, AWS KMS deletes the KMS key. + deletion_window_in_days = "10" + # Generates new cryptographic material every 365 days, this is used to encrypt your data. The KMS key retains the old material for decryption purposes. + enable_key_rotation = "true" +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "storage" { + bucket = aws_s3_bucket.storage.id + rule { + apply_server_side_encryption_by_default { + kms_master_key_id = aws_kms_key.storage.arn + sse_algorithm = "aws:kms" + } + bucket_key_enabled = true + } +} diff --git a/infra/modules/storage/events.tf b/infra/modules/storage/events.tf new file mode 100644 index 000000000..80536435d --- /dev/null +++ b/infra/modules/storage/events.tf @@ -0,0 +1,7 @@ +# Tell S3 to publish events to EventBridge. Subscribers can then subscribe +# to these events in an event-based architecture. +# See https://docs.aws.amazon.com/AmazonS3/latest/userguide/ev-mapping-troubleshooting.html +resource "aws_s3_bucket_notification" "storage" { + bucket = aws_s3_bucket.storage.id + eventbridge = true +} diff --git a/infra/modules/storage/lifecycle.tf b/infra/modules/storage/lifecycle.tf new file mode 100644 index 000000000..174aa42c9 --- /dev/null +++ b/infra/modules/storage/lifecycle.tf @@ -0,0 +1,11 @@ +resource "aws_s3_bucket_lifecycle_configuration" "storage" { + bucket = aws_s3_bucket.storage.id + + rule { + id = "AbortIncompleteUpload" + status = "Enabled" + abort_incomplete_multipart_upload { + days_after_initiation = 7 + } + } +} diff --git a/infra/modules/storage/main.tf b/infra/modules/storage/main.tf new file mode 100644 index 000000000..48f465af4 --- /dev/null +++ b/infra/modules/storage/main.tf @@ -0,0 +1,10 @@ +resource "aws_s3_bucket" "storage" { + bucket = var.name + force_destroy = false + + # checkov:skip=CKV_AWS_18:TODO(https://github.com/navapbc/template-infra/issues/507) Implement access logging + + # checkov:skip=CKV_AWS_144:Cross region replication not required by default + # checkov:skip=CKV2_AWS_62:S3 bucket does not need notifications enabled + # checkov:skip=CKV_AWS_21:Bucket versioning is not needed +} diff --git a/infra/modules/storage/outputs.tf b/infra/modules/storage/outputs.tf new file mode 100644 index 000000000..8337ee7a5 --- /dev/null +++ b/infra/modules/storage/outputs.tf @@ -0,0 +1,3 @@ +output "access_policy_arn" { + value = aws_iam_policy.storage_access.arn +} diff --git a/infra/modules/storage/variables.tf b/infra/modules/storage/variables.tf new file mode 100644 index 000000000..0a261c969 --- /dev/null +++ b/infra/modules/storage/variables.tf @@ -0,0 +1,4 @@ +variable "name" { + type = string + description = "Name of the AWS S3 bucket. Needs to be globally unique across all regions." +} diff --git a/infra/modules/terraform-backend-s3/main.tf b/infra/modules/terraform-backend-s3/main.tf index 09bf780c8..3b95ca3c7 100644 --- a/infra/modules/terraform-backend-s3/main.tf +++ b/infra/modules/terraform-backend-s3/main.tf @@ -11,7 +11,7 @@ locals { # Create the dynamodb table required for state locking. # Options for encryption are an AWS owned key, which is not unique to your account; AWS managed; or customer managed. The latter two options are more secure, and customer managed gives -# control over the key. This allows for ability to restrict access by key as well as policies attached to roles or users. +# control over the key. This allows for ability to restrict access by key as well as policies attached to roles or users. # https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html resource "aws_kms_key" "tf_backend" { description = "KMS key for DynamoDB table ${local.tf_locks_table_name}" @@ -127,7 +127,7 @@ resource "aws_s3_bucket_policy" "tf_state" { # Create the S3 bucket to provide server access logging. # -# Ignore bucket logging complaince check for this bucket since +# Ignore bucket logging complaince check for this bucket since # the bucket is used for logging only and doesn't need server access logging itself # (see https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerLogs.html) # tfsec:ignore:aws-s3-enable-bucket-logging diff --git a/infra/networks/main.tf b/infra/networks/main.tf index 140573d8c..c64fd4516 100644 --- a/infra/networks/main.tf +++ b/infra/networks/main.tf @@ -1,25 +1,32 @@ -# TODO: This file is is a temporary implementation of the network layer -# that currently just adds resources to the default VPC -# The full network implementation is part of https://github.com/navapbc/template-infra/issues/152 - -data "aws_region" "current" {} - locals { tags = merge(module.project_config.default_tags, { - description = "VPC resources" + network_name = var.network_name + description = "VPC resources" }) region = module.project_config.default_region - # List of AWS services used by this VPC - # This list is used to create VPC endpoints so that the AWS services can - # be accessed without network traffic ever leaving the VPC's private network - # For a list of AWS services that integrate with AWS PrivateLink - # see https://docs.aws.amazon.com/vpc/latest/privatelink/aws-services-privatelink-support.html - # - # The database module requires VPC access from private networks to SSM, KMS, and RDS - aws_service_integrations = toset( - module.app_config.has_database ? ["ssm", "kms"] : [] - ) + network_config = module.project_config.network_configs[var.network_name] + + # List of configuration for all applications, even ones that are not in the current network + # If project has multiple applications, add other app configs to this list + app_configs = [module.app_config] + + # List of configuration for applications that are in the current network + # An application is in the current network if at least one of its environments + # is mapped to the network + apps_in_network = [ + for app in local.app_configs : + app + if anytrue([ + for environment_config in app.environment_configs : true if environment_config.network_name == var.network_name + ]) + ] + + # Whether any of the applications in the network have a database + has_database = anytrue([for app in local.apps_in_network : app.has_database]) + + # Whether any of the applications in the network have dependencies on an external non-AWS service + has_external_non_aws_service = anytrue([for app in local.apps_in_network : app.has_external_non_aws_service]) } terraform { @@ -49,76 +56,14 @@ module "project_config" { } module "app_config" { - source = "../api/app-config" -} - -data "aws_vpc" "default" { - default = true -} - -data "aws_subnets" "default" { - filter { - name = "default-for-az" - values = [true] - } -} - -# VPC Endpoints for accessing AWS Services -# ---------------------------------------- -# -# Since the role manager Lambda function is in the VPC (which is needed to be -# able to access the database) we need to allow the Lambda function to access -# AWS Systems Manager Parameter Store (to fetch the database password) and -# KMS (to decrypt SecureString parameters from Parameter Store). We can do -# this by either allowing internet access to the Lambda, or by using a VPC -# endpoint. The latter is more secure. -# See https://repost.aws/knowledge-center/lambda-vpc-parameter-store -# See https://docs.aws.amazon.com/vpc/latest/privatelink/create-interface-endpoint.html#create-interface-endpoint - -resource "aws_security_group" "aws_services" { - count = length(local.aws_service_integrations) > 0 ? 1 : 0 - - name_prefix = module.project_config.aws_services_security_group_name_prefix - description = "VPC endpoints to access AWS services from the VPCs private subnets" - vpc_id = data.aws_vpc.default.id -} - -resource "aws_vpc_endpoint" "aws_service" { - for_each = local.aws_service_integrations - - vpc_id = data.aws_vpc.default.id - service_name = "com.amazonaws.${data.aws_region.current.name}.${each.key}" - vpc_endpoint_type = "Interface" - security_group_ids = [aws_security_group.aws_services[0].id] - subnet_ids = data.aws_subnets.default.ids - private_dns_enabled = true -} - -# VPC Configuration for DMS -# ---------------------------------------- - -data "aws_ssm_parameter" "dms_peer_owner_id" { - name = "/network/dms/peer-owner-id" -} - -data "aws_ssm_parameter" "dms_peer_vpc_id" { - name = "/network/dms/peer-vpc-id" -} - -resource "aws_vpc_peering_connection" "dms" { - peer_owner_id = data.aws_ssm_parameter.dms_peer_owner_id.value - peer_vpc_id = data.aws_ssm_parameter.dms_peer_vpc_id.value - vpc_id = data.aws_vpc.default.id - peer_region = "us-east-2" - - tags = { - Name = "DMS VPC Peering" - } + source = "../app/app-config" } -resource "aws_route" "dms" { - route_table_id = data.aws_vpc.default.main_route_table_id - # MicroHealth VPC CIDR block - destination_cidr_block = "10.220.0.0/16" - vpc_peering_connection_id = aws_vpc_peering_connection.dms.id +module "network" { + source = "../modules/network" + name = var.network_name + aws_services_security_group_name_prefix = module.project_config.aws_services_security_group_name_prefix + database_subnet_group_name = local.network_config.database_subnet_group_name + has_database = local.has_database + has_external_non_aws_service = local.has_external_non_aws_service } diff --git a/infra/networks/variables.tf b/infra/networks/variables.tf new file mode 100644 index 000000000..0182a92d2 --- /dev/null +++ b/infra/networks/variables.tf @@ -0,0 +1,4 @@ +variable "network_name" { + type = string + description = "Human readable identifier for the VPC" +} diff --git a/infra/project-config/main.tf b/infra/project-config/main.tf index 08755144e..d9d7da183 100644 --- a/infra/project-config/main.tf +++ b/infra/project-config/main.tf @@ -1,19 +1,26 @@ locals { # Machine readable project name (lower case letters, dashes, and underscores) # This will be used in names of AWS resources - project_name = "simpler-grants-gov" + project_name = "" # Project owner (e.g. navapbc). Used for tagging infra resources. - owner = "navapbc" + owner = "" # URL of project source code repository - code_repository_url = "https://github.com/HHS/simpler-grants-gov" + code_repository_url = "" # Default AWS region for project (e.g. us-east-1, us-east-2, us-west-1). # This is dependent on where your project is located (if regional) # otherwise us-east-1 is a good default - default_region = "us-east-1" + default_region = "" + + github_actions_role_name = "${local.project_name}-github-actions" - github_actions_role_name = "${local.project_name}-github-actions" aws_services_security_group_name_prefix = "aws-service-vpc-endpoints" + + network_configs = { + dev = { database_subnet_group_name = "dev" } + staging = { database_subnet_group_name = "staging" } + prod = { database_subnet_group_name = "prod" } + } } diff --git a/infra/project-config/outputs.tf b/infra/project-config/outputs.tf index 690f1eb22..52a66cc52 100644 --- a/infra/project-config/outputs.tf +++ b/infra/project-config/outputs.tf @@ -27,7 +27,7 @@ output "default_tags" { repository = local.code_repository_url terraform = true terraform_workspace = terraform.workspace - # description is set in each environments local use key project_description if required. + # description is set in each environments local use key project_description if required. } } @@ -38,3 +38,7 @@ output "github_actions_role_name" { output "aws_services_security_group_name_prefix" { value = local.aws_services_security_group_name_prefix } + +output "network_configs" { + value = local.network_configs +} diff --git a/infra/test/go.mod b/infra/test/go.mod index ad9bdff48..4719ab068 100644 --- a/infra/test/go.mod +++ b/infra/test/go.mod @@ -5,22 +5,21 @@ go 1.19 require github.com/gruntwork-io/terratest v0.41.0 require ( - cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/compute v1.19.1 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.13.0 // indirect - cloud.google.com/go/storage v1.28.1 // indirect + cloud.google.com/go v0.104.0 // indirect + cloud.google.com/go/compute v1.10.0 // indirect + cloud.google.com/go/iam v0.5.0 // indirect + cloud.google.com/go/storage v1.27.0 // indirect github.com/agext/levenshtein v1.2.3 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/aws/aws-sdk-go v1.44.122 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.7.1 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.6.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-getter v1.7.0 // indirect @@ -37,21 +36,21 @@ require ( github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/stretchr/testify v1.8.1 // indirect + github.com/stretchr/testify v1.7.0 // indirect github.com/tmccombs/hcl2json v0.3.3 // indirect github.com/ulikunitz/xz v0.5.10 // indirect github.com/zclconf/go-cty v1.9.1 // indirect - go.opencensus.io v0.24.0 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.7.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/text v0.14.0 // indirect + go.opencensus.io v0.23.0 // indirect + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect + golang.org/x/net v0.1.0 // indirect + golang.org/x/oauth2 v0.1.0 // indirect + golang.org/x/sys v0.1.0 // indirect + golang.org/x/text v0.4.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/api v0.114.0 // indirect + google.golang.org/api v0.100.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/grpc v1.56.3 // indirect - google.golang.org/protobuf v1.30.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71 // indirect + google.golang.org/grpc v1.50.1 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/infra/test/go.sum b/infra/test/go.sum index e462a559b..ed2b5def8 100644 --- a/infra/test/go.sum +++ b/infra/test/go.sum @@ -31,8 +31,6 @@ cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+ cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= cloud.google.com/go v0.104.0 h1:gSmWO7DY1vOm0MVU6DNXM11BWHHsTUmsC5cv1fuW5X8= cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= -cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= -cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= @@ -70,10 +68,6 @@ cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLq cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0 h1:aoLIYaA1fX3ywihqpBk2APQKOo20nXsp1GEZQbx5Jk4= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= -cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= -cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= @@ -112,8 +106,6 @@ cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0 h1:fz9X5zyTWBmamZsqvqZqD7khbifcZF/q+Z1J8pfhIUg= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= -cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= -cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= @@ -175,8 +167,6 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= -cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= -cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= @@ -277,8 +267,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -304,7 +292,6 @@ github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIG github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -327,8 +314,6 @@ github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99 github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= -github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= -github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -339,8 +324,6 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0 h1:SXk3ABtQYDT/OH8jAyvEOQ58mgawq5C4o/4/89qN2ZU= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= -github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= -github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gruntwork-io/terratest v0.41.0 h1:QKFK6m0EMVnrV7lw2L06TlG+Ha3t0CcOXuBVywpeNRU= @@ -412,8 +395,6 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -421,10 +402,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tmccombs/hcl2json v0.3.3 h1:+DLNYqpWE0CsOQiEZu+OZm5ZBImake3wtITYxQ8uLFQ= github.com/tmccombs/hcl2json v0.3.3/go.mod h1:Y2chtz2x9bAeRTvSibVRVgbLJhLJXKlUeIvjeVdnm4w= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= @@ -452,8 +429,6 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -463,8 +438,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -550,10 +523,6 @@ golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfS golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -580,8 +549,6 @@ golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.1.0 h1:isLCZuhj4v+tYv7eskaN4v/TM+A1begWWgyVJDdl1+Y= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -661,13 +628,10 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -679,8 +643,6 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -794,8 +756,6 @@ google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.100.0 h1:LGUYIrbW9pzYQQ8NWXlaIVkgnfubVBZbMFb9P8TK374= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= -google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= -google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -906,8 +866,6 @@ google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71 h1:GEgb2jF5zxsFJpJfg9RoDDWm7tiwc/DDSTE2BtLUkXU= google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -944,8 +902,6 @@ google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= -google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -963,8 +919,6 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -977,10 +931,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20220521103104-8f96da9f5d5e h1:3i3ny04XV6HbZ2N1oIBw1UBYATHAOpo4tfTF83JM3Z0= -gopkg.in/yaml.v3 v3.0.0-20220521103104-8f96da9f5d5e/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/infra/test/infra_test.go b/infra/test/infra_test.go index 4f68a9bcb..70f7f5ca7 100644 --- a/infra/test/infra_test.go +++ b/infra/test/infra_test.go @@ -1,7 +1,6 @@ package test import ( - "flag" "fmt" "strings" "testing" @@ -11,11 +10,11 @@ import ( "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/shell" "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/stretchr/testify/require" ) var uniqueId = strings.ToLower(random.UniqueId()) var workspaceName = fmt.Sprintf("t-%s", uniqueId) -var appName = flag.String("app_name", "", "name of subdirectory that holds the app's infrastructure code") func TestService(t *testing.T) { BuildAndPublish(t) @@ -27,7 +26,7 @@ func TestService(t *testing.T) { }) terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ Reconfigure: true, - TerraformDir: fmt.Sprintf("../%s/service/", *appName), + TerraformDir: "../app/service/", Vars: map[string]interface{}{ "environment_name": "dev", "image_tag": imageTag, @@ -61,14 +60,14 @@ func BuildAndPublish(t *testing.T) { // after which we add BackendConfig: []string{"dev.s3.tfbackend": terraform.KeyOnly} to terraformOptions // and replace the call to terraform.RunTerraformCommand with terraform.Init TerraformInit(t, &terraform.Options{ - TerraformDir: fmt.Sprintf("../%s/build-repository/", *appName), + TerraformDir: "../app/build-repository/", }, "shared.s3.tfbackend") fmt.Println("::endgroup::") fmt.Println("::group::Build release") shell.RunCommand(t, shell.Command{ Command: "make", - Args: []string{"release-build", fmt.Sprintf("APP_NAME=%s", *appName)}, + Args: []string{"release-build", "APP_NAME=app"}, WorkingDir: "../../", }) fmt.Println("::endgroup::") @@ -76,7 +75,7 @@ func BuildAndPublish(t *testing.T) { fmt.Println("::group::Publish release") shell.RunCommand(t, shell.Command{ Command: "make", - Args: []string{"release-publish", fmt.Sprintf("APP_NAME=%s", *appName)}, + Args: []string{"release-publish", "APP_NAME=app"}, WorkingDir: "../../", }) fmt.Println("::endgroup::") @@ -84,7 +83,7 @@ func BuildAndPublish(t *testing.T) { func WaitForServiceToBeStable(t *testing.T, workspaceName string) { fmt.Println("::group::Wait for service to be stable") - appName := *appName + appName := "app" environmentName := "dev" serviceName := fmt.Sprintf("%s-%s-%s", workspaceName, appName, environmentName) shell.RunCommand(t, shell.Command{ @@ -101,6 +100,11 @@ func RunEndToEndTests(t *testing.T, terraformOptions *terraform.Options) { http_helper.HttpGetWithRetryWithCustomValidation(t, serviceEndpoint, nil, 5, 1*time.Second, func(responseStatus int, responseBody string) bool { return responseStatus == 200 }) + // Hit feature flags endpoint to make sure Evidently integration is working + featureFlagsEndpoint := fmt.Sprintf("%s/feature-flags", serviceEndpoint) + http_helper.HttpGetWithRetryWithCustomValidation(t, featureFlagsEndpoint, nil, 5, 1*time.Second, func(responseStatus int, responseBody string) bool { + return responseStatus == 200 + }) fmt.Println("::endgroup::") } @@ -124,6 +128,23 @@ func EnableDestroyService(t *testing.T, terraformOptions *terraform.Options) { }, WorkingDir: "../../", }) + shell.RunCommand(t, shell.Command{ + Command: "sed", + Args: []string{ + "-i.bak", + "s/force_destroy = false/force_destroy = true/g", + "infra/modules/storage/main.tf", + }, + WorkingDir: "../../", + }) + + // Clone the options and set targets to only apply to the buckets + terraformOptions, err := terraformOptions.Clone() + require.NoError(t, err) + terraformOptions.Targets = []string{ + "module.service.aws_s3_bucket.access_logs", + "module.storage.aws_s3_bucket.storage", + } terraform.Apply(t, terraformOptions) fmt.Println("::endgroup::") }