Migrate AWS Blake env to its own TF directory (#321)

infrastructure/maskfile.md

@@ -28,15 +28,9 @@ Both `act` commands (for container version updates) and `terraform` commands
 Usually this means setting a file at `~/.aws/credentials` and `~/.aws/config`:
 see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html
 
+## aws:tf:plan
 
-## aws:tf:apply
-
-> Runs the named infrastructure script interactively using the environment specified.
-
-REQUIRES an environment variable setting:
- TF_VAR_HONEYCOMB_API_KEY
-for secrets that don't exist in this repository
-
+> Runs the plan (diff showing) command interactively using the environment specified.
 
 **OPTIONS**
 
@@ -50,36 +44,53 @@ for secrets that don't exist in this repository
 
 ```bash
 (
-    cd terraform/aws
-
-    echo "checking for environment configuration files..."
+    cd terraform/aws/environments/${proper_name}
 
-    tf_var_file="./environments/${proper_name}/variables.tfvars"
-
-    if [ ! -f ${tf_var_file} ]; then
-        echo "REQUIRED var file missing: ${tf_var_file}"
+    export AWS_PAGER=""
+    if aws sts get-caller-identity; then
+        echo "AWS identity check succeeded."
+    else
+        echo "AWS CLI misconfigured; see maskfile.md for info"
         exit 1
     fi
 
-    echo "checking environment setup..."
-    if [ -z "${TF_VAR_HONEYCOMB_API_KEY}" ]; then
-        echo "REQURED env secret TF_VAR_HONEYCOMB_API_KEY missing"
-        exit 1
-    fi
+    echo "showing env diff for $proper_name from $(pwd)"
+
+    terraform plan \
+        -input=false
+)
+```
+
+## aws:tf:apply
+
+> Runs the apply command interactively, still asking for confirmation, using the environment specified.
+
+**OPTIONS**
+
+-   proper_name
+    -   flags: -n --proper-name
+    -   type: string
+    -   desc: proper name of AWS environment (see `./aws` module); e.g. blake
+    -   required
+
+<!-- A code block defines the script to be executed -->
+
+```bash
+(
+    cd terraform/aws/environments/${proper_name}
 
     export AWS_PAGER=""
     if aws sts get-caller-identity; then
         echo "AWS identity check succeeded."
     else
-        echo "AWS CLI misconfigured; see Maskfile.md for info"
+        echo "AWS CLI misconfigured; see maskfile.md for info"
         exit 1
     fi
 
     echo "applying $proper_name from $(pwd)"
 
     terraform apply \
-        -input=false \
-        -var-file=${tf_var_file}
+        -input=false
 )
 ```
 
@@ -98,18 +109,9 @@ for secrets that don't exist in this repository
 ```bash
 
 (
-    cd terraform/aws
-    echo "checking for environment configuration files..."
-
-    tf_backend_file="./environments/${proper_name}/${proper_name}.s3.tfbackend"
-
-    if [ ! -f ${tf_backend_file} ]; then
-        echo "REQUIRED backend file missing: ${tf_backend_file}"
-        exit 1
-    fi
+    cd terraform/aws/environments/${proper_name}
 
-    terraform init \
-        -backend-config ${tf_backend_file}
+    terraform init
 )
 ```
 

infrastructure/terraform/aws/environments/blake/main.tf

@@ -0,0 +1,68 @@
+######
+##
+##  Terraform-meta configurations
+##
+######
+
+terraform {
+  required_version = ">= 1.5.0"
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+
+    honeycombio = {
+      source  = "honeycombio/honeycombio"
+      version = ">= 0.22.0"
+    }
+  }
+  backend "s3" {
+    bucket = "pubpub-tfstates"
+    key    = "ecs-blake.tfstate"
+    region = "us-east-1"
+  }
+}
+
+provider "aws" {
+  region = local.region
+}
+
+######
+##
+##  Environment-specific configuration
+##
+######
+
+locals {
+  name = "blake"
+  environment = "staging"
+  region = "us-east-1"
+
+  # TODO: Resume using this once we also Terraform the Route53
+  # pubpub_url = "https://v7.pubpub.org"
+
+  MAILGUN_SMTP_USERNAME = "[email protected]"
+  NEXT_PUBLIC_SUPABASE_URL = "https://dsleqjuvzuoycpeotdws.supabase.co"
+  NEXT_PUBLIC_SUPABASE_PUBLIC_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImRzbGVxanV2enVveWNwZW90ZHdzIiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODIzNTE0MjEsImV4cCI6MTk5NzkyNzQyMX0.3HHC0f7zlFXP77N0U8cS3blr7n6hhjqdYI6_ciQJams"
+  ASSETS_BUCKET_NAME = "assets.blake.pubpub.org"
+}
+
+######
+##
+##  Complete generic environment
+##
+######
+
+module "deployment" {
+  source = "../../modules/deployment"
+
+  name = local.name
+  environment = local.environment
+  region = local.region
+
+  MAILGUN_SMTP_USERNAME = local.MAILGUN_SMTP_USERNAME
+  NEXT_PUBLIC_SUPABASE_URL = local.NEXT_PUBLIC_SUPABASE_URL
+  NEXT_PUBLIC_SUPABASE_PUBLIC_KEY = local.NEXT_PUBLIC_SUPABASE_PUBLIC_KEY
+  ASSETS_BUCKET_NAME = local.ASSETS_BUCKET_NAME
+}

infrastructure/terraform/aws/modules/core-services/main.tf

@@ -30,15 +30,6 @@ resource "aws_secretsmanager_secret" "honeycomb_api_key" {
   name = "honeycombio-apikey-${var.cluster_info.name}-${var.cluster_info.environment}"
 }
 
-# N.B. since we have to tell terraform about this secret in order to
-#   configure the Honeycomb module, we might as well set it up automatically
-#   for Secrets Manager too. This pattern is not ideal design on the part of
-#   Honeycomb.
-resource "aws_secretsmanager_secret_version" "honeycomb_api_key" {
-  secret_id     = aws_secretsmanager_secret.honeycomb_api_key.id
-  secret_string = var.HONEYCOMB_API_KEY
-}
-
 # generate password and make it accessible through aws secrets manager
 resource "random_password" "rds_db_password" {
   length           = 16

infrastructure/terraform/aws/modules/core-services/variables.tf

@@ -18,12 +18,3 @@ variable "assets_bucket_url_name" {
   description = "Name for the asset bucket -- typically a domain like assets.v7.pubpub.org"
   type = string
 }
-
-# TODO: Possibly, this factoring could be improved
-# by making it easier to deploy the Core without a honeycomb
-# account.
-variable "HONEYCOMB_API_KEY" {
-  description = "API key for the honeycomb environment"
-  type = string
-  sensitive = true
-}

infrastructure/terraform/aws/main.tf → infrastructure/terraform/aws/modules/deployment/main.tf

@@ -13,17 +13,14 @@ terraform {
       version = ">= 0.22.0"
     }
   }
-  backend "s3" {
-    # contents provided in NAME.s3.tfbackend
-  }
 }
 
-provider "aws" {
-  region = var.region
-}
+# provider "aws" {
+#   region = var.region
+# }
 
 module "cluster" {
-  source = "./modules/v7-cluster"
+  source = "../v7-cluster"
 
   name = var.name
   environment = var.environment
@@ -35,15 +32,14 @@ module "cluster" {
 }
 
 module "core_dependency_services" {
-  source = "./modules/core-services"
+  source = "../core-services"
 
   cluster_info = module.cluster.cluster_info
   assets_bucket_url_name = var.ASSETS_BUCKET_NAME
-  HONEYCOMB_API_KEY = var.HONEYCOMB_API_KEY
 }
 
 module "service_core" {
-  source = "./modules/container-generic"
+  source = "../container-generic"
 
   service_name = "core"
   cluster_info = module.cluster.cluster_info
@@ -107,7 +103,7 @@ module "service_core" {
 }
 
 module "service_flock" {
-  source = "./modules/container-generic"
+  source = "../container-generic"
 
   service_name = "jobs"
   cluster_info = module.cluster.cluster_info
@@ -133,7 +129,7 @@ module "service_flock" {
 }
 
  module "service_intg_submissions" {
-   source = "./modules/container-generic"
+   source = "../container-generic"
 
    service_name = "integration-submissions"
    cluster_info = module.cluster.cluster_info
@@ -166,7 +162,7 @@ module "service_flock" {
  }
 
  module "service_intg_evaluations" {
-   source = "./modules/container-generic"
+   source = "../container-generic"
 
    service_name = "integration-evaluations"
    cluster_info = module.cluster.cluster_info
@@ -199,7 +195,7 @@ module "service_flock" {
  }
 
  module "service_bastion" {
-   source = "./modules/container-generic"
+   source = "../container-generic"
 
    service_name = "bastion"
    cluster_info = module.cluster.cluster_info
@@ -230,9 +226,26 @@ module "service_flock" {
    }
  }
 
+
+# N.B. This invocation means that the deployment including honeycomb cannot succeed
+#   until after you have inserted the secret into the AWS console. This only happens
+#   in this one case because with things like ECS, you can successfully "apply"
+#   even if secrets are not present; the containers will simply fail to start.
+#   However, this last section of TF code can be commented out for a first apply,
+#   then go and insert secret in console, then reapply with this.
+#
+#   This is the result of an awkward design pattern, where instead of the
+#   Honeycomb provider being configured to search for an API key in the env,
+#   the modules themselves expect an API key as an inline var and fail if
+#   it is not set. This is probably because the API keys are different for
+#   different environments, rather than per account/user/etc.
+data "aws_secretsmanager_secret_version" "honeycomb_api_key" {
+  secret_id = module.core_dependency_services.secrets.honeycomb_api_key
+}
+
 module "observability_honeycomb_integration" {
-  source = "./modules/honeycomb-integration"
+  source = "../honeycomb-integration"
 
   cluster_info = module.cluster.cluster_info
-  HONEYCOMB_API_KEY = var.HONEYCOMB_API_KEY
+  HONEYCOMB_API_KEY = data.aws_secretsmanager_secret_version.honeycomb_api_key.secret_string
 }