From 8509fd721743a430a84df2fcf97afb428366bc7e Mon Sep 17 00:00:00 2001 From: "eve n.u" Date: Wed, 7 Feb 2024 12:40:03 -0800 Subject: [PATCH] update modularization --- infrastructure/terraform/aws/main.tf | 50 ++++++++-- .../aws/modules/core-services/README.md | 98 +++++++++++++++++++ .../aws/modules/core-services/main.tf | 91 +++++++++++++++++ .../aws/modules/core-services/outputs.tf | 18 ++++ .../aws/modules/core-services/variables.tf | 15 +++ .../aws/modules/ecs-service/variables.tf | 1 + .../terraform/aws/modules/v7-cluster/ecr.tf | 10 ++ .../aws/modules/v7-cluster/outputs.tf | 25 ++--- .../terraform/aws/modules/v7-cluster/rds.tf | 59 ----------- infrastructure/terraform/aws/variables.tf | 27 +++++ 10 files changed, 309 insertions(+), 85 deletions(-) create mode 100644 infrastructure/terraform/aws/modules/core-services/README.md create mode 100644 infrastructure/terraform/aws/modules/core-services/main.tf create mode 100644 infrastructure/terraform/aws/modules/core-services/outputs.tf create mode 100644 infrastructure/terraform/aws/modules/core-services/variables.tf delete mode 100644 infrastructure/terraform/aws/modules/v7-cluster/rds.tf diff --git a/infrastructure/terraform/aws/main.tf b/infrastructure/terraform/aws/main.tf index 1a43a5ed1..df090ffdf 100644 --- a/infrastructure/terraform/aws/main.tf +++ b/infrastructure/terraform/aws/main.tf @@ -29,26 +29,58 @@ module "cluster" { availability_zones = ["us-east-1a", "us-east-1c"] } +module "core_dependency_services" { + source = "./modules/core-services" + + cluster_info = module.cluster.cluster_info +} + module "service_core" { source = "./modules/ecs-service" service_name = "core" cluster_info = module.cluster.cluster_info + repository_url = module.cluster.ecr_repository_urls.core - repository_url = module.cluster.ecr_repository_url + configuration = { + container_port = 3000 + environment = [ + { name = "DATABASE_URL", value = module.core_dependency_services.rds_connection_string_sans_password }, + { name = "ASSETS_REGION", value = var.region }, + { name = "ASSETS_BUCKET_NAME", value = var.ASSETS_BUCKET_NAME }, + { name = "ASSETS_UPLOAD_KEY", value = var.ASSETS_UPLOAD_KEY }, + { name = "NEXT_PUBLIC_PUBPUB_URL", value = var.pubpub_url }, + { name = "MAILGUN_SMTP_USERNAME", value = var.MAILGUN_SMTP_USERNAME }, + { name = "NEXT_PUBLIC_SUPABASE_URL", value = var.NEXT_PUBLIC_SUPABASE_URL }, + ] + + secrets = [ + { name = "DATABASE_PASSWORD", valueFrom = module.core_dependency_services.rds_db_password_id }, + { name = "API_KEY", valueFrom = module.core_dependency_services.api_key_secret_id }, + ] + } +} + +module "service_flock" { + source = "./modules/ecs-service" + + service_name = "jobs" + cluster_info = module.cluster.cluster_info + + repository_url = module.cluster.ecr_repository_urls.jobs configuration = { container_port = 3000 environment = [ - {name = "API_KEY", value = "undefined"}, - {name = "JWT_SECRET", value = "undefined"}, - {name = "MAILGUN_SMTP_USERNAME", value = "undefined"}, - {name = "NEXT_PUBLIC_PUBPUB_URL", value = "https://v7.pubpub.org"}, - {name = "NEXT_PUBLIC_SUPABASE_URL", value = "undefined"}, - {name = "SENTRY_AUTH_TOKEN", value = "undefined"}, - {name = "SUPABASE_SERVICE_ROLE_KEY", value = "undefined"}, - {name = "SUPABASE_WEBHOOKS_API_KEY", value = "undefined"}, + {name = "PUBPUB_URL", value = var.pubpub_url }, + {name = "DATABASE_URL", value = module.core_dependency_services.rds_connection_string_sans_password }, + # Secrets - TODO move these to aws secrets + ] + + secrets = [ + { name = "DATABASE_PASSWORD", valueFrom = module.core_dependency_services.rds_db_password_id }, + { name = "API_KEY", valueFrom = module.core_dependency_services.api_key_secret_id }, ] } } diff --git a/infrastructure/terraform/aws/modules/core-services/README.md b/infrastructure/terraform/aws/modules/core-services/README.md new file mode 100644 index 000000000..1620c6aa0 --- /dev/null +++ b/infrastructure/terraform/aws/modules/core-services/README.md @@ -0,0 +1,98 @@ +# Setup +In a `main.tf` file for a workspace that needs a cluster, +you can use this module like: +``` +module "cluster" { + source = "../path/to/this/directory" + // version is disallowed when using path-based modules + + environment = "staging" + region = "us-east-2" + hosted_zone_id = "SOME-ZONE-ID" + // all other variables are optional +} +``` + +then +``` +terraform init +terraform apply +``` + +You will see these resources under `module.cluster.xyz`. + +## Managing the ECS Task Definition +Working with ECS task definitions in Terraform +is kind of awkward. + +This module creates an ECS task definition, +so that it can set up the ECS service that uses that task definition, +but expects that future task revisions will be created by a CI pipeline +as new images are created and pushed. +The `deploy_on_merge.yml` has an example of such a pipeline. +In this pipeline, +we get the ECS task definition from a file, +and interpolate the image, +to create a new revision. +Terraform ignores changes +made by the pipeline, +due to the `lifecycle` setting +on the ECS service resource. + +If you make changes to the task definition resource in this module, +and run `terraform apply` in the `ecs-staging` directory, +Terraform will update the task, +but the next time a new commit is pushed to git, +that change will be overwritten +by the definition in the `ecs-staging` directory, +that's used by the pipeline. + +Ideally these definitions would come from the same source +so if you're reading this +perhaps today is the day +to make that refactor! + +More information about the general wonkiness +of managing ECS with Terraform +can be found in [this Terraform issue.](https://github.com/hashicorp/terraform-provider-aws/issues/632) + +## Rotating the RDS Password +The RDS password is retrieved from AWS Secrets Manager +but that password is managed manually, +and rotating it requires downtime. + +To rotate it, you'll need to perform the following steps: +- Update the value of the Secrets Manager entry through the AWS console +- Update the value in the RDS instance through the AWS console. (At this point, the core container will stop being able to access the database.) +- Recreate the core container's service with `aws update-service cluster $CLUSTER_NAME --service $SERVICE_NAME --force-new-deployment` + +In the future the RDS should probably [manage its own password](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/rds-secrets-manager.html) +which will probably require changing the service's code +to fetch the password from Secrets Manager itself +rather than getting it passed in from an environment variable. + +## Rotating the ACM Certificate + +ACM issues certs that last for 1 year. They send you an email prior to renewing, but will automatically rotate the cert. + +Since we rely on DNS validation, it is necessary that our validation CNAMEs are present in route53, provided by this module. +The required records MAY not change, but if they have changed behind the scenes, this will catch up our terraform state: + +```bash +# from clean state , no changes to code + +# updates our state file's records of the Domain Validation Options on the cert (DVOs). +terraform apply + +# should show that new DNS records need to be created/updated, matching the DVOs. +terraform plan -out TMP.tfplan +terraform apply TMP.tfplan +``` + +For more info: see [AWS Docs](https://docs.aws.amazon.com/acm/latest/userguide/dns-renewal-validation.html). + +## Development + +When you change the resources in this directory, you must run `terraform apply` in the calling workspace to see changes. + +More info on developing [terraform modules](https://developer.hashicorp.com/terraform/language/modules/develop). diff --git a/infrastructure/terraform/aws/modules/core-services/main.tf b/infrastructure/terraform/aws/modules/core-services/main.tf new file mode 100644 index 000000000..dbc568ce9 --- /dev/null +++ b/infrastructure/terraform/aws/modules/core-services/main.tf @@ -0,0 +1,91 @@ +# aws terraform provider config + +terraform { + required_version = ">= 0.12.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.0" + } + } +} + +# Static secrets +resource "random_password" "api_key" { + length = 32 + special = true + override_special = "-_.~!#$&'()*+,/:;=?@[]" +} + +resource "aws_secretsmanager_secret" "api_key" { + name = "api-key-${var.cluster_info.name}-${var.cluster_info.environment}" +} + +resource "aws_secretsmanager_secret_version" "api_key" { + secret_id = aws_secretsmanager_secret.api_key.id + secret_string = random_password.api_key.result +} + +# generate password and make it accessible through aws secrets manager +resource "random_password" "rds_db_password" { + length = 16 + special = true + override_special = "!#$%&*()-_=+[]{}<>:?" +} + +resource "aws_secretsmanager_secret" "rds_db_password" { + name = "rds-db-password-${var.cluster_info.name}-${var.cluster_info.environment}" +} + +resource "aws_secretsmanager_secret_version" "password" { + secret_id = aws_secretsmanager_secret.rds_db_password.id + secret_string = random_password.rds_db_password.result +} + +# network config +resource "aws_db_subnet_group" "ecs_dbs" { + name = "${var.cluster_info.name}_ecs_db_${var.cluster_info.environment}" + subnet_ids = var.cluster_info.private_subnet_ids + + tags = { + Name = "subnet group for ECS RDS instances" + } +} + +resource "aws_security_group" "ecs_tasks_rds_instances" { + name = "${var.cluster_info.name}-sg-rds-${var.cluster_info.environment}" + vpc_id = var.cluster_info.vpc_id + + ingress { + protocol = "tcp" + from_port = 5432 + to_port = 5432 + security_groups = var.cluster_info.container_security_group_ids + } +} + +# the actual database instance +resource "aws_db_instance" "core_postgres" { + identifier = "${var.cluster_info.name}-core-postgres-${var.cluster_info.environment}" + allocated_storage = 20 + db_name = "${var.cluster_info.name}_${var.cluster_info.environment}_core_postgres" + db_subnet_group_name = aws_db_subnet_group.ecs_dbs.name + engine = "postgres" + engine_version = "14" + instance_class = "db.t3.small" + vpc_security_group_ids = [aws_security_group.ecs_tasks_rds_instances.id] + username = var.cluster_info.name + password = random_password.rds_db_password.result + parameter_group_name = "default.postgres14" + skip_final_snapshot = true + + lifecycle { + ignore_changes = [ + password, + ] + } +} + + +### TODO : add Sentry? other stuff? + diff --git a/infrastructure/terraform/aws/modules/core-services/outputs.tf b/infrastructure/terraform/aws/modules/core-services/outputs.tf new file mode 100644 index 000000000..f3590935e --- /dev/null +++ b/infrastructure/terraform/aws/modules/core-services/outputs.tf @@ -0,0 +1,18 @@ +locals { + db_user = aws_db_instance.core_postgres.username + db_name = aws_db_instance.core_postgres.db_name + db_host = aws_db_instance.core_postgres.address + db_sslmode = "require" +} + +output "api_key_secret_id" { + value = aws_secretsmanager_secret.api_key.id +} + +output "rds_db_password_id" { + value = aws_secretsmanager_secret.rds_db_password.id +} + +output "rds_connection_string_sans_password" { + value = "postgresql://${local.db_user}@${local.db_host}:5432/${local.db_name}?sslmode=${local.db_sslmode}" +} diff --git a/infrastructure/terraform/aws/modules/core-services/variables.tf b/infrastructure/terraform/aws/modules/core-services/variables.tf new file mode 100644 index 000000000..ecd025c2a --- /dev/null +++ b/infrastructure/terraform/aws/modules/core-services/variables.tf @@ -0,0 +1,15 @@ +variable "cluster_info" { + description = "infrastructure values output from v7-cluster" + + type = object({ + region = string + name = string + vpc_id = string + cluster_arn = string + environment = string + private_subnet_ids = list(string) + container_security_group_ids = list(string) + cloudwatch_log_group_name = string + lb_target_group_arn = string + }) +} diff --git a/infrastructure/terraform/aws/modules/ecs-service/variables.tf b/infrastructure/terraform/aws/modules/ecs-service/variables.tf index fabd60288..a5482c797 100644 --- a/infrastructure/terraform/aws/modules/ecs-service/variables.tf +++ b/infrastructure/terraform/aws/modules/ecs-service/variables.tf @@ -4,6 +4,7 @@ variable "cluster_info" { type = object({ region = string name = string + vpc_id = string cluster_arn = string environment = string private_subnet_ids = list(string) diff --git a/infrastructure/terraform/aws/modules/v7-cluster/ecr.tf b/infrastructure/terraform/aws/modules/v7-cluster/ecr.tf index 8c72a849d..c3d58c2a1 100644 --- a/infrastructure/terraform/aws/modules/v7-cluster/ecr.tf +++ b/infrastructure/terraform/aws/modules/v7-cluster/ecr.tf @@ -17,6 +17,7 @@ resource "aws_ecr_repository" "pubpub_v7_core" { } } +# TODO: integrations may want to be one image alone ...? resource "aws_ecr_repository" "pubpub_v7_intg_submissions" { name = "pubpub-v7-integration-submissions" image_tag_mutability = "MUTABLE" @@ -25,3 +26,12 @@ resource "aws_ecr_repository" "pubpub_v7_intg_submissions" { scan_on_push = false # can set this to true if we want } } + +resource "aws_ecr_repository" "pubpub_v7_jobs" { + name = "pubpub-v7-jobs" + image_tag_mutability = "MUTABLE" + + image_scanning_configuration { + scan_on_push = false # can set this to true if we want + } +} diff --git a/infrastructure/terraform/aws/modules/v7-cluster/outputs.tf b/infrastructure/terraform/aws/modules/v7-cluster/outputs.tf index 5d5b18aea..80e2e8b1a 100644 --- a/infrastructure/terraform/aws/modules/v7-cluster/outputs.tf +++ b/infrastructure/terraform/aws/modules/v7-cluster/outputs.tf @@ -1,26 +1,17 @@ -locals { - db_user = aws_db_instance.core_postgres.username - db_name = aws_db_instance.core_postgres.db_name - db_host = aws_db_instance.core_postgres.address - db_sslmode = "require" -} - -output "ecr_repository_url" { - value = aws_ecr_repository.pubpub_v7.repository_url -} - -output "rds_db_password_id" { - value = aws_secretsmanager_secret.rds_db_password.id -} - -output "rds_connection_string_sans_password" { - value = "postgresql://${local.db_user}@${local.db_host}:5432/${local.db_name}?sslmode=${local.db_sslmode}" +output "ecr_repository_urls" { + value = { + root = aws_ecr_repository.pubpub_v7.repository_url + core = aws_ecr_repository.pubpub_v7_core.repository_url + intg_submissions = aws_ecr_repository.pubpub_v7_intg_submissions.repository_url + jobs = aws_ecr_repository.pubpub_v7_jobs.repository_url + } } output "cluster_info" { value = { region = var.region name = var.name + vpc_id = aws_vpc.main.id environment = var.environment cluster_arn = module.ecs_cluster.arn private_subnet_ids = aws_subnet.private.*.id diff --git a/infrastructure/terraform/aws/modules/v7-cluster/rds.tf b/infrastructure/terraform/aws/modules/v7-cluster/rds.tf deleted file mode 100644 index 960c3a10e..000000000 --- a/infrastructure/terraform/aws/modules/v7-cluster/rds.tf +++ /dev/null @@ -1,59 +0,0 @@ -# generate password and make it accessible through aws secrets manager -resource "random_password" "rds_db_password" { - length = 16 - special = true - override_special = "!#$%&*()-_=+[]{}<>:?" -} - -resource "aws_secretsmanager_secret" "rds_db_password" { - name = "rds-db-password-${var.name}-${var.environment}" -} - -resource "aws_secretsmanager_secret_version" "password" { - secret_id = aws_secretsmanager_secret.rds_db_password.id - secret_string = random_password.rds_db_password.result -} - -# network config -resource "aws_db_subnet_group" "ecs_dbs" { - name = "${var.name}_ecs_db_${var.environment}" - subnet_ids = aws_subnet.private.*.id - - tags = { - Name = "subnet group for ECS RDS instances" - } -} - -resource "aws_security_group" "ecs_tasks_rds_instances" { - name = "${var.name}-sg-rds-${var.environment}" - vpc_id = aws_vpc.main.id - - ingress { - protocol = "tcp" - from_port = 5432 - to_port = 5432 - security_groups = [aws_security_group.ecs_tasks.id] - } -} - -# the actual database instance -resource "aws_db_instance" "core_postgres" { - identifier = "${var.name}-core-postgres-${var.environment}" - allocated_storage = 20 - db_name = "${var.name}_${var.environment}_core_postgres" - db_subnet_group_name = aws_db_subnet_group.ecs_dbs.name - engine = "postgres" - engine_version = "14" - instance_class = "db.t3.small" - vpc_security_group_ids = [aws_security_group.ecs_tasks_rds_instances.id] - username = var.name - password = random_password.rds_db_password.result - parameter_group_name = "default.postgres14" - skip_final_snapshot = true - - lifecycle { - ignore_changes = [ - password, - ] - } -} diff --git a/infrastructure/terraform/aws/variables.tf b/infrastructure/terraform/aws/variables.tf index 43bf7735d..be212d1bc 100644 --- a/infrastructure/terraform/aws/variables.tf +++ b/infrastructure/terraform/aws/variables.tf @@ -13,3 +13,30 @@ variable "environment" { description = "Functional name for this environment" type = string } + +variable "pubpub_url" { + description = "URL where pubpub will be addressable (include https://)" + type = string +} + +variable "MAILGUN_SMTP_USERNAME" { + description = "SMTP Username for Mailgun service" + type = string +} + +variable "NEXT_PUBLIC_SUPABASE_URL" { + description = "URL to Supabase public address for this install" + type = string +} + +# TODO deprecate this in favor of a Terraformed bucket +variable "ASSETS_BUCKET_NAME" { + description = "Name of the S3 bucket to store assets" + type = string +} + +# TODO: deprecate this in favor of terraformed iam/service roles +variable "ASSETS_UPLOAD_KEY" { + description = "AWS access key ID for uploading to s3" + type = string +}