[Issue 1086] Update to latest version of template-infra

.github/workflows/ci-docs.yml

@@ -0,0 +1,20 @@
+name: CI Documentation Checks
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+
+jobs:
+  lint-markdown:
+    name: Lint markdown
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      # This is the GitHub Actions-friendly port of the linter used in the Makefile.
+      - uses: gaurav-nelson/[email protected]
+        with:
+          use-quiet-mode: 'yes'  # errors only.
+          config-file: '.github/workflows/markdownlint-config.json'

.github/workflows/markdownlint-config.json

@@ -0,0 +1,19 @@
+{
+    "ignorePatterns" : [
+        {
+            "pattern": "0005-example.md"
+        },
+        {
+            "pattern": "localhost"
+        },
+        {
+            "pattern": "127.0.0.1"
+        }
+    ],
+    "replacementPatterns": [
+        {
+            "pattern": "^/",
+            "replacement": "{{BASEURL}}/"
+        }
+  ]
+}

Makefile

@@ -51,6 +51,7 @@ __check_defined = \
 	infra-update-current-account \
 	infra-update-network \
 	infra-validate-modules \
+	lint-markdown \
 	release-build \
 	release-deploy \
 	release-image-name \
@@ -64,8 +65,9 @@ infra-set-up-account: ## Configure and create resources for current AWS profile
 	@:$(call check_defined, ACCOUNT_NAME, human readable name for account e.g. "prod" or the AWS account alias)
 	./bin/set-up-current-account.sh $(ACCOUNT_NAME)
 
-infra-configure-network: ## Configure default network
-	./bin/create-tfbackend.sh infra/networks default
+infra-configure-network: ## Configure network $NETWORK_NAME
+	@:$(call check_defined, NETWORK_NAME, the name of the network in /infra/networks)
+	./bin/create-tfbackend.sh infra/networks $(NETWORK_NAME)
 
 infra-configure-app-build-repository: ## Configure infra/$APP_NAME/build-repository tfbackend and tfvars files
 	@:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code)
@@ -90,8 +92,10 @@ infra-configure-app-service: ## Configure infra/$APP_NAME/service module's tfbac
 infra-update-current-account: ## Update infra resources for current AWS profile
 	./bin/terraform-init-and-apply.sh infra/accounts `./bin/current-account-config-name.sh`
 
-infra-update-network: ## Update default network
-	./bin/terraform-init-and-apply.sh infra/networks default
+infra-update-network: ## Update network
+	@:$(call check_defined, NETWORK_NAME, the name of the network in /infra/networks)
+	terraform -chdir="infra/networks" init -input=false -reconfigure -backend-config="$(NETWORK_NAME).s3.tfbackend"
+	terraform -chdir="infra/networks" apply -var="network_name=$(NETWORK_NAME)"
 
 infra-update-app-build-repository: ## Create or update $APP_NAME's build repository
 	@:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code)
@@ -109,7 +113,6 @@ infra-update-app-database-roles: ## Create or update database roles and schemas
 	./bin/create-or-update-database-roles.sh $(APP_NAME) $(ENVIRONMENT)
 
 infra-update-app-service: ## Create or update $APP_NAME's web service module
-	# APP_NAME has a default value defined above, but check anyways in case the default is ever removed
 	@:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code)
 	@:$(call check_defined, ENVIRONMENT, the name of the application environment e.g. "prod" or "staging")
 	terraform -chdir="infra/$(APP_NAME)/service" init -input=false -reconfigure -backend-config="$(ENVIRONMENT).s3.tfbackend"
@@ -137,7 +140,7 @@ infra-check-compliance-checkov: ## Run checkov compliance checks
 infra-check-compliance-tfsec: ## Run tfsec compliance checks
 	tfsec infra
 
-infra-lint: infra-lint-scripts infra-lint-terraform infra-lint-workflows ## Lint infra code
+infra-lint: lint-markdown infra-lint-scripts infra-lint-terraform infra-lint-workflows ## Lint infra code
 
 infra-lint-scripts: ## Lint shell scripts
 	shellcheck bin/**
@@ -155,6 +158,9 @@ infra-test-service: ## Run service layer infra test suite
 	@:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code)
 	cd infra/test && go test -run TestService -v -timeout 30m -app_name=$(APP_NAME)
 
+lint-markdown: ## Lint Markdown docs for broken links
+	./bin/lint-markdown.sh
+
 ########################
 ## Release Management ##
 ########################

bin/lint-markdown.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# To make things simpler, ensure we're in the repo's root directory (one directory up) before
+# running, regardless where the user is when invoking this script.
+
+# Grab the full directory name for where this script lives.
+SCRIPT_DIR=$(readlink -f "$0" | xargs dirname)
+
+# Move up to the root since we want to do everything relative to that. Note that this only impacts
+# this script, but will leave the user wherever they were when the script exists.
+cd "${SCRIPT_DIR}/.." >/dev/null || exit 1
+
+
+LINK_CHECK_CONFIG=".github/workflows/markdownlint-config.json"
+
+# Recursively find all markdown files (*.md) in the current directory, excluding node_modules and .venv subfolders.
+# Pass them in as args to the lint command using the handy `xargs` command.
+find . -name \*.md -not -path "*/node_modules/*" -not -path "*/.venv/*" -print0 | xargs -0 -n1 npx markdown-link-check --config $LINK_CHECK_CONFIG

bin/set-up-current-account.sh

@@ -52,11 +52,12 @@ echo "Creating bucket: $TF_STATE_BUCKET_NAME"
 # For creating buckets outside of us-east-1, a LocationConstraint needs to be set
 # For creating buckets in us-east-1, LocationConstraint cannot be set
 # See https://docs.aws.amazon.com/cli/latest/reference/s3api/create-bucket.html
-CREATE_BUCKET_CONFIGURATION=""
+CREATE_BUCKET_CONFIGURATION=("")
 if [ "$REGION" != "us-east-1" ]; then
-  CREATE_BUCKET_CONFIGURATION="--create-bucket-configuration LocationConstraint=$REGION"
+  CREATE_BUCKET_CONFIGURATION=("--create-bucket-configuration" "LocationConstraint=$REGION")
 fi
-aws s3api create-bucket --bucket "$TF_STATE_BUCKET_NAME" --region "$REGION" "$CREATE_BUCKET_CONFIGURATION" > /dev/null
+
+aws s3api create-bucket --bucket "$TF_STATE_BUCKET_NAME" --region "$REGION" "${CREATE_BUCKET_CONFIGURATION[@]}" > /dev/null
 echo
 echo "----------------------------------"
 echo "Creating rest of account resources"
@@ -65,6 +66,21 @@ echo
 
 cd infra/accounts
 
+# Create the OpenID Connect provider for GitHub Actions to allow GitHub Actions
+# to authenticate with AWS and manage AWS resources. We create the OIDC provider
+# via AWS CLI rather than via Terraform because we need to first check if there
+# is already an existing OpenID Connect provider for GitHub Actions. This check
+# is needed since there can only be one OpenID Connect provider per URL per AWS
+# account.
+github_arn=$(aws iam list-open-id-connect-providers | jq -r ".[] | .[] | .Arn" | grep github || echo "")
+
+if [[ -z ${github_arn} ]]; then
+  aws iam create-open-id-connect-provider \
+    --url "https://token.actions.githubusercontent.com" \
+    --client-id-list "sts.amazonaws.com" \
+    --thumbprint-list "0000000000000000000000000000000000000000"
+fi
+
 # Create the infrastructure for the terraform backend such as the S3 bucket
 # for storing tfstate files and the DynamoDB table for tfstate locks.
 # -reconfigure is used in case this isn't the first account being set up

docs/feature-flags.md

@@ -0,0 +1,31 @@
+# Feature flags and partial releases
+
+Feature flags are an important tool that enables [trunk-based development](https://trunkbaseddevelopment.com/). They allow in-progress features to be merged into the main branch while still allowing that branch to be deployed to production at any time, thus decoupling application deploys from feature releases. For a deeper introduction, [Martin Fowler's article on Feature Toggles](https://martinfowler.com/articles/feature-toggles.html) and [LaunchDarkly's blog post on feature flags](https://launchdarkly.com/blog/what-are-feature-flags/) are both great articles that explain the what and why of feature flags.
+
+## How it works
+
+This project leverages [Amazon CloudWatch Evidently](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Evidently.html) to create and manage feature flags.
+
+## Creating feature flags
+
+The list of feature flags for an application is defined in the `feature_flags` property in its app-config module (in `/infra/[app_name]/app-config/main.tf`). To create a new feature flag, add a new string to that list. To remove a feature flag, remove the feature flag from the list. The set of feature flags will be updated on the next terraform apply of the service layer, or during the next deploy of the application.
+
+## Querying feature flags in the application
+
+To determine whether a particular feature should be enabled or disabled for a given user, the application code calls an "is feature enabled" function in the feature flags module. Under the hood, the module will call AWS Evidently's [EvaluateFeature](https://docs.aws.amazon.com/cloudwatchevidently/latest/APIReference/API_EvaluateFeature.html) API to determine whether a feature is enabled or disabled. For partial rollouts, it will remember which variation of the application a particular user saw and keep the user experience consistent for that user. For more information about the feature flags module, look in the application code and docs.
+
+## Managing feature releases and partial rollouts via AWS Console
+
+The system is designed to allow the managing of feature releases and partial rollouts outside of terraform, which empowers business owners and product managers to control enable and disable feature flags and adjust feature launch traffic percentages without needing to depend on the development team.
+
+### To enable or disable a feature
+
+1. Navigate to the Evidently service in AWS Console, select the appropriate Evidently feature flags project for the relevant application environment, and select the feature you want to manage.
+2. In the actions menu, select "Edit feature".
+3. Under "Feature variations", select either "FeatureOn" (to enable a feature) or "FeatureOff" (to disable a feature) to be the "Default" variation, then submit. **Warning: Do not modify the variation values. "FeatureOn" should always have a value of "True" and "FeatureOff" should always have a value of "False".**
+
+### To manage a partial rollout
+
+1. Navigate to the Evidently service in AWS Console, and select the appropriate Evidently feature flags project for the relevant application environment
+2. Select "Create launch" to create a new partial rollout plan, or select an existing launch to manage an existing rollout
+3. Under "Launch configuration", choose the traffic percentage you want to send to each variation, and choose whether you want the launch to begin immediately or on a schedule.

docs/system-architecture.md

@@ -0,0 +1,21 @@
+# System Architecture
+
+This diagram shows the system architecture. [🔒 Make a copy of this Lucid template for your own application](https://lucid.app/lucidchart/8851888e-1292-4228-8fef-60a61c6b57e7/edit).
+
+![System architecture](https://lucid.app/publicSegments/view/e5a36152-200d-4d95-888e-4cdbdab80d1b/image.png)
+
+* **Access Logs** — Amazon S3 bucket storing the application service's access logs.
+* **Alarms SNS Topic** — SNS topic that notifies the incident management service when an alarm triggers.
+* **Application Load Balancer** — Amazon application load balancer.
+* **Aurora PostgreSQL Database** — Amazon Aurora Serverless PostgreSQL database used by the application.
+* **Build Repository ECR Registry** — Amazon ECR registry that acts as the build repository of application container images.
+* **CloudWatch Alarms** — Amazon CloudWatch Alarms that trigger on errors and latency.
+* **CloudWatch Evidently Feature Flags** — Amazon CloudWatch Evidently service that manages feature flags used by the application to manage feature launches.
+* **Database Role Manager** — AWS Lambda serverless function that provisions the database roles needed by the application.
+* **GitHub** — Source code repository. Also responsible for Continuous Integration (CI) and Continuous Delivery (CD) workflows. GitHub Actions builds and deploys releases to an Amazon ECR registry that stores Docker container images for the application service.
+* **Incident Management Service** — Incident management service (e.g. PagerDuty or Splunk On-Call) for managing on-call schedules and paging engineers for urgent production issues.
+* **Service** — Amazon ECS service running the application.
+* **Terraform Backend Bucket** — Amazon S3 bucket used to store terraform state files.
+* **Terraform Locks DynamoDB Table** — Amazon DynamoDB table used to manage concurrent access to terraform state files.
+* **VPC Endpoints** — VPC endpoints are used by the Database Role Manager to access Amazon Services without traffic leaving the VPC.
+* **VPC Network** — Amazon VPC network.

infra/README.md

@@ -1,6 +1,81 @@
-# Simpler Grants Infrastructure
+# Overview
 
-The infrastructure for this project is stored as Terraform files and hosted on Amazon Web Services.
+This project practices infrastructure-as-code and uses the [Terraform framework](https://www.terraform.io). This directory contains the infrastructure code for this project, including infrastructure for all application resources. This terraform project uses the [AWS provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs). It is based on the [Nava platform infrastructure template](https://github.com/navapbc/template-infra).
 
-- See the [infra documentation](../documentation/infra/module-architecture.md) for details about the Terraform modules and architecture.
-- See the [Cloud Hosting](../documentation/decisions/adr/2023-08-21-cloud-platform.md) and [Infrastructure as Code](../documentation/decisions/adr/2023-08-21-infrastructure-as-code-tool.md) ADRs for information on the decisions that informed this architecture.
+## 📂 Directory structure
+
+The structure for the infrastructure code looks like this:
+
+```text
+infra/                  Infrastructure code
+  accounts/             [Root module] IaC and IAM resources
+  [app_name]/           Application directory: infrastructure for the main application
+  modules/              Reusable child modules
+  networks/             [Root module] Account level network config (shared across all apps, environments, and terraform workspaces)
+```
+
+Each application directory contains the following:
+
+```text
+  app-config/         Application-level configuration for the application resources (different config for different environments)
+  build-repository/   [Root module] Docker image repository for the application (shared across environments and terraform workspaces)
+  database/           [Root module] Configuration for database (different config for different environments)
+  service/            [Root module] Configuration for containers, such as load balancer, application service (different config for different environments)
+```
+
+Details about terraform root modules and child modules are documented in [module-architecture](/docs/infra/module-architecture.md).
+
+## 🏗️ Project architecture
+
+### 🧅 Infrastructure layers
+
+The infrastructure template is designed to operate on different layers:
+
+- Account layer
+- Network layer
+- Build repository layer (per application)
+- Database layer (per application)
+- Service layer (per application)
+
+### 🏜️ Application environments
+
+This project has the following AWS environments:
+
+- `dev`
+- `staging`
+- `prod`
+
+The environments share the same root modules but will have different configurations. Backend configuration is saved as [`.tfbackend`](https://developer.hashicorp.com/terraform/language/settings/backends/configuration#file) files. Most `.tfbackend` files are named after the environment. For example, the `[app_name]/service` infrastructure resources for the `dev` environment are configured via `dev.s3.tfbackend`. Resources for a module that are shared across environments, such as the build-repository, use `shared.s3.tfbackend`. Resources that are shared across the entire account (e.g. /infra/accounts) use `<account name>.<account id>.s3.tfbackend`.
+
+### 🔀 Project workflow
+
+This project relies on Make targets in the [root Makefile](/Makefile), which in turn call shell scripts in [./bin](/bin). The shell scripts call terraform commands. Many of the shell scripts are also called by the [Github Actions CI/CD](/.github/workflows).
+
+Generally you should use the Make targets or the underlying bin scripts, but you can call the underlying terraform commands if needed. See [making-infra-changes](/docs/infra/making-infra-changes.md) for more details.
+
+## 💻 Development
+
+### 1️⃣ First time initialization
+
+To set up this project for the first time (aka it has never been deployed to the target AWS account):
+
+1. [Configure the project](/infra/project-config/main.tf) (These values will be used in subsequent infra setup steps to namespace resources and add infrastructure tags.)
+2. [Set up infrastructure developer tools](/docs/infra/set-up-infrastructure-tools.md)
+3. [Set up AWS account](/docs/infra/set-up-aws-account.md)
+4. [Set up the virtual network (VPC)](/docs/infra/set-up-network.md)
+5. For each application:
+    1. [Set up application build repository](/docs/infra/set-up-app-build-repository.md)
+    2. [Set up application database](/docs/infra/set-up-database.md)
+    3. [Set up application environment](/docs/infra/set-up-app-env.md)
+
+### 🆕 New developer
+
+To get set up as a new developer to a project that has already been deployed to the target AWS account:
+
+1. [Set up infrastructure developer tools](/docs/infra/set-up-infrastructure-tools.md)
+2. [Review how to make changes to infrastructure](/docs/infra/making-infra-changes.md)
+3. (Optional) Set up a [terraform workspace](/docs/infra/intro-to-terraform-workspaces.md)
+
+## 📇 Additional reading
+
+Additional documentation can be found in the [documentation directory](/docs/infra).

infra/accounts/main.tf

@@ -50,4 +50,5 @@ module "auth_github_actions" {
   source                   = "../modules/auth-github-actions"
   github_actions_role_name = module.project_config.github_actions_role_name
   github_repository        = module.project_config.code_repository
+  allowed_actions          = [for aws_service in module.project_config.aws_services : "${aws_service}:*"]
 }