From b46fbcc1667d8d18be5694501c70277590f86642 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 2 Jun 2023 08:46:18 -0600 Subject: [PATCH 01/38] tf changes --- .../{veda-wfs3 => features-api}/.gitignore | 0 .../.terraform.lock.hcl | 38 +--- terraform/{veda-wfs3 => features-api}/dns.tf | 10 +- terraform/{veda-wfs3 => features-api}/ecr.tf | 6 +- .../{veda-wfs3 => features-api}/ecs_api.tf | 56 +++--- .../github_deploy_user.tf | 0 terraform/{veda-wfs3 => features-api}/init.tf | 6 +- .../load_balancer.tf} | 29 ++- .../{veda-wfs3 => features-api}/outputs.tf | 4 - terraform/features-api/rds.tf | 65 +++++++ .../secret_manager.tf | 31 +++- terraform/features-api/security_group.tf | 41 +++++ .../{veda-wfs3 => features-api}/variables.tf | 24 +-- terraform/features-api/vars/dev.tf | 14 ++ terraform/features-api/vars/staging.tf | 14 ++ ...nition.json => container_definition.tftpl} | 0 terraform/modules/aws_ecs_service/main.tf | 50 +++--- .../modules/aws_ecs_service/variables.tf | 6 +- .../lambda_function.py | 57 ------ .../requirements.txt | 1 - terraform/veda-wfs3/rds.tf | 65 ------- terraform/veda-wfs3/s3_event_bridge_lambda.tf | 167 ------------------ terraform/veda-wfs3/vars/dev.tf | 12 -- terraform/veda-wfs3/vars/west2-staging.tf | 13 -- terraform/veda-wfs3/vpc.tf | 41 ----- {veda-wfs3-app => wfs3-app}/Dockerfile | 0 {veda-wfs3-app => wfs3-app}/cd.sh | 0 {veda-wfs3-app => wfs3-app}/dbconntest.py | 0 {veda-wfs3-app => wfs3-app}/fast_api_main.py | 0 {veda-wfs3-app => wfs3-app}/requirements.txt | 0 {veda-wfs3-app => wfs3-app}/startup.sh | 0 31 files changed, 268 insertions(+), 482 deletions(-) rename terraform/{veda-wfs3 => features-api}/.gitignore (100%) rename terraform/{veda-wfs3 => features-api}/.terraform.lock.hcl (55%) rename terraform/{veda-wfs3 => features-api}/dns.tf (78%) rename terraform/{veda-wfs3 => features-api}/ecr.tf (57%) rename terraform/{veda-wfs3 => features-api}/ecs_api.tf (67%) rename terraform/{veda-wfs3 => features-api}/github_deploy_user.tf (100%) rename terraform/{veda-wfs3 => features-api}/init.tf (72%) rename terraform/{veda-wfs3/load_balancer_west_2.tf => features-api/load_balancer.tf} (84%) rename terraform/{veda-wfs3 => features-api}/outputs.tf (79%) create mode 100644 terraform/features-api/rds.tf rename terraform/{veda-wfs3 => features-api}/secret_manager.tf (54%) create mode 100644 terraform/features-api/security_group.tf rename terraform/{veda-wfs3 => features-api}/variables.tf (62%) create mode 100644 terraform/features-api/vars/dev.tf create mode 100644 terraform/features-api/vars/staging.tf rename terraform/modules/aws_ecs_service/{container_definition.json => container_definition.tftpl} (100%) delete mode 100644 terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/lambda_function.py delete mode 100644 terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/requirements.txt delete mode 100644 terraform/veda-wfs3/rds.tf delete mode 100644 terraform/veda-wfs3/s3_event_bridge_lambda.tf delete mode 100644 terraform/veda-wfs3/vars/dev.tf delete mode 100644 terraform/veda-wfs3/vars/west2-staging.tf delete mode 100644 terraform/veda-wfs3/vpc.tf rename {veda-wfs3-app => wfs3-app}/Dockerfile (100%) rename {veda-wfs3-app => wfs3-app}/cd.sh (100%) rename {veda-wfs3-app => wfs3-app}/dbconntest.py (100%) rename {veda-wfs3-app => wfs3-app}/fast_api_main.py (100%) rename {veda-wfs3-app => wfs3-app}/requirements.txt (100%) rename {veda-wfs3-app => wfs3-app}/startup.sh (100%) diff --git a/terraform/veda-wfs3/.gitignore b/terraform/features-api/.gitignore similarity index 100% rename from terraform/veda-wfs3/.gitignore rename to terraform/features-api/.gitignore diff --git a/terraform/veda-wfs3/.terraform.lock.hcl b/terraform/features-api/.terraform.lock.hcl similarity index 55% rename from terraform/veda-wfs3/.terraform.lock.hcl rename to terraform/features-api/.terraform.lock.hcl index a758db9..6fc5d64 100644 --- a/terraform/veda-wfs3/.terraform.lock.hcl +++ b/terraform/features-api/.terraform.lock.hcl @@ -1,29 +1,11 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/hashicorp/archive" { - version = "2.3.0" - hashes = [ - "h1:pTPG9Kf1Qg2aPsZLXDa6OvLqsEXaMrKnp0Z4Q/TIBPA=", - "zh:0869128d13abe12b297b0cd13b8767f10d6bf047f5afc4215615aabc39c2eb4f", - "zh:481ed837d63ba3aa45dd8736da83e911e3509dee0e7961bf5c00ed2644f807b3", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:9f08fe2977e2166849be24fb9f394e4d2697414d463f7996fd0d7beb4e19a29c", - "zh:9fe566deeafd460d27999ca0bbfd85426a5fcfcb40007b23884deb76da127b6f", - "zh:a1bd9a60925d9769e0da322e4523330ee86af9dc2e770cba1d0247a999ef29cb", - "zh:bb4094c8149f74308b22a87e1ac19bcccca76e8ef021b571074d9bccf1c0c6f0", - "zh:c8984c9def239041ce41ec8e19bbd76a49e74ed2024ff736dad60429dee89bcc", - "zh:ea4bb5ae73db1de3a586e62f39106f5e56770804a55aa5e6b4f642df973e0e75", - "zh:f44a9d596ecc3a8c5653f56ba0cd202ad93b49f76767f4608daf7260b813289e", - "zh:f5c5e6cc9f7f070020ab7d95fcc9ed8e20d5cf219978295a71236e22cbb6d508", - "zh:fd2273f51dcc8f43403bf1e425ba9db08a57c3ddcba5ad7a51742ccde21ca611", - ] -} - provider "registry.terraform.io/hashicorp/aws" { version = "4.58.0" constraints = "~> 4.0" hashes = [ + "h1:YIRXIr1ji0HLWLU0ae+UbUNOHc9MJaLrMHxH3LIQ/Vk=", "h1:znLROwEAINbYzAG5X7Ep04whM7KxkQGrvhFdhSvNKEk=", "zh:14b2b2dfbc7ee705c412d762b1485ee08958c816a64ac74f5769e946e4a1d265", "zh:17a37e6825e2023b18987d31c0cbb9336654ea146b68e6c90710ea4636af71ae", @@ -46,6 +28,7 @@ provider "registry.terraform.io/hashicorp/aws" { provider "registry.terraform.io/hashicorp/random" { version = "3.4.3" hashes = [ + "h1:saZR+mhthL0OZl4SyHXZraxyaBNVMxiZzks78nWcZ2o=", "h1:tL3katm68lX+4lAncjQA9AXL4GR/VM+RPwqYf4D2X8Q=", "zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752", "zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b", @@ -61,20 +44,3 @@ provider "registry.terraform.io/hashicorp/random" { "zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93", ] } - -provider "registry.terraform.io/hashicorp/template" { - version = "2.2.0" - hashes = [ - "h1:0wlehNaxBX7GJQnPfQwTNvvAf38Jm0Nv7ssKGMaG6Og=", - "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", - "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", - "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", - "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", - "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", - "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", - "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", - "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", - "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", - "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", - ] -} diff --git a/terraform/veda-wfs3/dns.tf b/terraform/features-api/dns.tf similarity index 78% rename from terraform/veda-wfs3/dns.tf rename to terraform/features-api/dns.tf index 9ad8883..65ab077 100644 --- a/terraform/veda-wfs3/dns.tf +++ b/terraform/features-api/dns.tf @@ -4,10 +4,10 @@ data "aws_route53_zone" "zone" { } resource "aws_acm_certificate" "cert" { - provider = aws.west2 - domain_name = "*.${data.aws_route53_zone.zone.name}" - validation_method = "DNS" - tags = var.tags + provider = aws.west2 + domain_name = "*.${data.aws_route53_zone.zone.name}" + validation_method = "DNS" + tags = var.tags lifecycle { create_before_destroy = true @@ -28,7 +28,7 @@ resource "aws_route53_record" "subdomain_record" { } resource "aws_lb_listener_certificate" "cert" { - provider = aws.west2 + provider = aws.west2 listener_arn = aws_alb_listener.alb_listener_ecs.arn certificate_arn = aws_acm_certificate.cert.arn } \ No newline at end of file diff --git a/terraform/veda-wfs3/ecr.tf b/terraform/features-api/ecr.tf similarity index 57% rename from terraform/veda-wfs3/ecr.tf rename to terraform/features-api/ecr.tf index 5d8d7ab..b09467a 100644 --- a/terraform/veda-wfs3/ecr.tf +++ b/terraform/features-api/ecr.tf @@ -1,10 +1,10 @@ module "ecr_registry" { - source = "github.com/developmentseed/tf-seed/modules/aws_ecr" + source = "github.com/developmentseed/tf-seed/modules/aws_ecr" environment = var.env - registry_name = var.registry_name + registry_name = var.project_name enable_registry_scanning = true mutable_image_tags = true enable_deploy_user = true iam_deploy_username = aws_iam_user.deploy_user.name - tags = var.tags + tags = var.tags } \ No newline at end of file diff --git a/terraform/veda-wfs3/ecs_api.tf b/terraform/features-api/ecs_api.tf similarity index 67% rename from terraform/veda-wfs3/ecs_api.tf rename to terraform/features-api/ecs_api.tf index a791f6e..644f944 100644 --- a/terraform/veda-wfs3/ecs_api.tf +++ b/terraform/features-api/ecs_api.tf @@ -1,9 +1,20 @@ +data "aws_subnets" "private" { + filter { + name = "vpc-id" + values = [var.vpc_id] + } + + tags = { + "aws-cdk:subnet-name" = "private" + } +} + module "ecs_cluster" { - source = "../modules/aws_ecs_service" + source = "../modules/aws_ecs_service" environment = var.env region = var.region - vpc_id = module.networking.vpc_id - subnet_ids = module.networking.private_subnets_id + vpc_id = var.vpc_id + subnet_ids = data.aws_subnets.private.ids service_name = "${var.project_name}-service" service_port = var.service_port @@ -18,11 +29,11 @@ module "ecs_cluster" { container_secrets = [ { - name = "AWS_CONFIG" + name = "AWS_CONFIG" valueFrom = aws_secretsmanager_secret.config.arn }, { - name = "DB_CONFIG" + name = "DB_CONFIG" valueFrom = aws_secretsmanager_secret.db_config.arn }, ] @@ -37,53 +48,54 @@ module "ecs_cluster" { value = "True" }, { - name = "OTEL_PROPAGATORS" + name = "OTEL_PROPAGATORS" value = "xray" }, { - name = "OTEL_PYTHON_ID_GENERATOR" + name = "OTEL_PYTHON_ID_GENERATOR" value = "xray" }, { - name = "OTEL_RESOURCE_ATTRIBUTES" + name = "OTEL_RESOURCE_ATTRIBUTES" value = "service.name=veda-wfs3-${var.env}" }, { - name = "OTEL_RESOURCE_ATTRIBUTES" + name = "OTEL_RESOURCE_ATTRIBUTES" value = "service.name=veda-wfs3-${var.env}" }, { - name = "OTEL_TRACES_SAMPLER" + name = "OTEL_TRACES_SAMPLER" value = "traceidratio" }, { - name = "OTEL_TRACES_SAMPLER_ARG" + name = "OTEL_TRACES_SAMPLER_ARG" value = "0.5" }, { - name = "FORWARDED_ALLOW_IPS" + name = "FORWARDED_ALLOW_IPS" value = "*" }, { // stupid hack b/c of FastAPI and Starlette bug - name = "FAST_API_SCHEME" - value = var.env == "west2-staging" ? "https" : "http" + name = "FAST_API_SCHEME" + value = var.env == "dev" ? "https" : "http" //quick hack for now, TODO: include 'contains' function } ] - container_ingress_cidrs = ["0.0.0.0/0"] + container_ingress_cidrs = ["0.0.0.0/0"] container_ingress_sg_ids = [] - use_adot_as_sidecar = true - use_ecr = true + use_adot_as_sidecar = false + use_ecr = true ecr_repository_name = module.ecr_registry.registry_name - image = "${module.ecr_registry.repository_url}:latest" + ecr_repository_arn = module.ecr_registry.registry_arn + image = "${module.ecr_registry.repository_url}:latest" - load_balancer = true - lb_type = "application" - lb_target_group_arn = aws_alb_target_group.alb_target_group.arn + load_balancer = true + lb_type = "application" + lb_target_group_arn = aws_alb_target_group.alb_target_group.arn lb_security_group_id = aws_security_group.web_inbound_sg.id - lb_container_port = var.service_port + lb_container_port = var.service_port tags = var.tags } diff --git a/terraform/veda-wfs3/github_deploy_user.tf b/terraform/features-api/github_deploy_user.tf similarity index 100% rename from terraform/veda-wfs3/github_deploy_user.tf rename to terraform/features-api/github_deploy_user.tf diff --git a/terraform/veda-wfs3/init.tf b/terraform/features-api/init.tf similarity index 72% rename from terraform/veda-wfs3/init.tf rename to terraform/features-api/init.tf index ef5e99c..276e1c3 100644 --- a/terraform/veda-wfs3/init.tf +++ b/terraform/features-api/init.tf @@ -17,8 +17,8 @@ terraform { } } backend "s3" { - bucket = "veda-wfs3-tf-state-bucket" - key = "root" - region = "us-west-1" + bucket = "ghg-wfs3-tf-state-bucket" + key = "root" + region = "us-west-2" } } diff --git a/terraform/veda-wfs3/load_balancer_west_2.tf b/terraform/features-api/load_balancer.tf similarity index 84% rename from terraform/veda-wfs3/load_balancer_west_2.tf rename to terraform/features-api/load_balancer.tf index b9ca611..60b6033 100644 --- a/terraform/veda-wfs3/load_balancer_west_2.tf +++ b/terraform/features-api/load_balancer.tf @@ -1,9 +1,26 @@ +data "aws_subnets" "public" { + filter { + name = "vpc-id" + values = [var.vpc_id] + } + + tags = { + "aws-cdk:subnet-name" = "public" + } +} + +data "aws_security_groups" "security_groups" { + filter { + name = "vpc-id" + values = [var.vpc_id] + } +} /* security group for ALB */ resource "aws_security_group" "web_inbound_sg" { name = "tf-${var.project_name}-${var.env}-web-inbound-sg" description = "Allow HTTP from Anywhere into ALB" - vpc_id = module.networking.vpc_id + vpc_id = var.vpc_id ingress { from_port = 80 @@ -34,7 +51,7 @@ resource "aws_security_group" "web_inbound_sg" { resource "aws_security_group" "https_web_inbound_sg" { name = "tf-${var.project_name}-${var.env}-https-web-inbound-sg" description = "Allow HTTPS from Anywhere into ALB" - vpc_id = module.networking.vpc_id + vpc_id = var.vpc_id ingress { from_port = 443 @@ -64,11 +81,11 @@ resource "aws_security_group" "https_web_inbound_sg" { resource "aws_alb" "alb_ecs" { name = "tf-${var.project_name}-${var.env}-alb" - subnets = module.networking.public_subnets_id - security_groups = concat(module.networking.security_groups_ids, [aws_security_group.https_web_inbound_sg.id]) + subnets = data.aws_subnets.public.ids + security_groups = [aws_security_group.https_web_inbound_sg.id] tags = merge({ - Name = "tf-${var.project_name}-alb" + Name = "tf-${var.project_name}-alb" }, var.tags) } @@ -76,7 +93,7 @@ resource "aws_alb_target_group" "alb_target_group" { name = "tf-${var.project_name}-${var.env}-tgp" port = var.service_port protocol = "HTTP" - vpc_id = module.networking.vpc_id + vpc_id = var.vpc_id target_type = "ip" deregistration_delay = 60 diff --git a/terraform/veda-wfs3/outputs.tf b/terraform/features-api/outputs.tf similarity index 79% rename from terraform/veda-wfs3/outputs.tf rename to terraform/features-api/outputs.tf index b76568d..24013cc 100644 --- a/terraform/veda-wfs3/outputs.tf +++ b/terraform/features-api/outputs.tf @@ -17,7 +17,3 @@ output "protocol_on_aws_alb_listener" { description = "HTTP/HTTPS protocol on the ALB Listener" value = aws_alb_listener.alb_listener_ecs.protocol } - -output "s3_event_bridge_lambda_arn" { - value = "${aws_lambda_function.lambda.arn}:${aws_lambda_function.lambda.version}" -} diff --git a/terraform/features-api/rds.tf b/terraform/features-api/rds.tf new file mode 100644 index 0000000..9727790 --- /dev/null +++ b/terraform/features-api/rds.tf @@ -0,0 +1,65 @@ +resource "aws_db_subnet_group" "db" { + name = "tf-${var.project_name}-${var.env}-subnet-group" + subnet_ids = data.aws_subnets.private.ids + tags = { + Name = "tf-${var.project_name}-subnet-group" + } +} + +resource "aws_db_parameter_group" "default" { + name = "tf-${var.project_name}-${var.env}-postgres14-param-group" + family = "postgres14" + + parameter { + name = "work_mem" + # NOTE: I had `work_mem` set to ~100MB and `max_connections` around 75 and TileJSON completely failed + # 16MB + value = var.env == "staging" ? "16384" : "8192" + } + + parameter { + name = "max_connections" + value = "475" + apply_method = "pending-reboot" + } + + # NOTE: here to show what shared_buffers are but doesn't really make sense why it won't provision with these + # parameter { + # name = "shared_buffers" + # value = var.env == "staging" ? "8064856" : "4032428" + # apply_method = "pending-reboot" + # } + + parameter { + name = "seq_page_cost" + value = "1" + } + + parameter { + name = "random_page_cost" + value = "1.2" + } +} + +resource "aws_db_instance" "db" { + db_name = "veda" + identifier = "${var.project_name}-${var.env}" + engine = "postgres" + engine_version = "14.3" + // https://docs.aws.amazon.com/AmazonRDS/latest/APIReference/API_CreateDBInstance.html + allocated_storage = 100 + max_allocated_storage = 500 + storage_type = "gp2" + instance_class = var.env == "staging" ? "db.r5.xlarge" : "db.r5.large" + db_subnet_group_name = aws_db_subnet_group.db.name + skip_final_snapshot = true + apply_immediately = true + backup_retention_period = 7 + vpc_security_group_ids = [aws_security_group.default_sg.id] + username = "postgres" + password = random_password.master_password.result + allow_major_version_upgrade = true + parameter_group_name = aws_db_parameter_group.default.name +} + + diff --git a/terraform/veda-wfs3/secret_manager.tf b/terraform/features-api/secret_manager.tf similarity index 54% rename from terraform/veda-wfs3/secret_manager.tf rename to terraform/features-api/secret_manager.tf index 49ef400..15615f6 100644 --- a/terraform/veda-wfs3/secret_manager.tf +++ b/terraform/features-api/secret_manager.tf @@ -13,16 +13,35 @@ resource "random_id" "sm_suffix" { byte_length = 2 } +resource "random_password" "master_password" { + length = 16 + special = false +} + resource "aws_secretsmanager_secret" "config" { - name = "aws-config-${random_id.sm_suffix.hex}" - kms_key_id = data.aws_kms_key.secretsmanager.id - tags = var.tags + name = "aws-config-${random_id.sm_suffix.hex}" + kms_key_id = data.aws_kms_key.secretsmanager.id + tags = var.tags } resource "aws_secretsmanager_secret" "db_config" { - name = "veda-wfs3-${var.env}-db-config-v3" - kms_key_id = data.aws_kms_key.secretsmanager.id - tags = var.tags + name = "${var.project_name}-wfs3-${var.env}-db" + kms_key_id = data.aws_kms_key.secretsmanager.id + tags = var.tags +} + +resource "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = aws_secretsmanager_secret.db_config.id + secret_string = < 0 ? jsonencode(var.container_command) : "" - working_directory = var.container_working_directory - container_secrets = jsonencode(var.container_secrets) - container_environment = jsonencode(var.container_environment) - service_protocol = var.service_protocol - service_port = var.service_port - use_adot_as_sidecar = var.use_adot_as_sidecar ? "on" : "" - log_group = aws_cloudwatch_log_group.service.name - region = var.region - } -} - resource "aws_ecs_task_definition" "service" { family = "tf-${var.service_name}-${var.environment}" requires_compatibilities = ["FARGATE"] @@ -239,7 +216,22 @@ resource "aws_ecs_task_definition" "service" { tags = var.tags execution_role_arn = aws_iam_role.ecs_execution_role.arn task_role_arn = aws_iam_role.ecs_execution_role.arn - container_definitions = data.template_file.container_definition.rendered + container_definitions = templatefile("${path.module}/container_definition.tftpl", + { + service_name = var.service_name + environment = var.environment + image = var.image + container_command = length(var.container_command) > 0 ? jsonencode(var.container_command) : "" + working_directory = var.container_working_directory + container_secrets = jsonencode(var.container_secrets) + container_environment = jsonencode(var.container_environment) + service_protocol = var.service_protocol + service_port = var.service_port + use_adot_as_sidecar = var.use_adot_as_sidecar ? "on" : "" + log_group = aws_cloudwatch_log_group.service.name + region = var.region + } + ) } ####################################################################### diff --git a/terraform/modules/aws_ecs_service/variables.tf b/terraform/modules/aws_ecs_service/variables.tf index bba749f..27c837c 100755 --- a/terraform/modules/aws_ecs_service/variables.tf +++ b/terraform/modules/aws_ecs_service/variables.tf @@ -11,7 +11,9 @@ variable "tags" { } variable "service_name" {} -variable "service_port" {} +variable "service_port" { + type = number +} variable "service_protocol" { type = string @@ -114,6 +116,8 @@ variable "use_adot_as_sidecar" { } variable "ecr_repository_name" {} +variable "ecr_repository_arn" {} + variable "image" {} variable "load_balancer" { diff --git a/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/lambda_function.py b/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/lambda_function.py deleted file mode 100644 index d524d83..0000000 --- a/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/lambda_function.py +++ /dev/null @@ -1,57 +0,0 @@ -import boto3 -import http.client -import os -import base64 -import ast -import json -mwaa_env_name = 'veda-pipeline-staging-mwaa' -dag_name = 'veda_discover' -mwaa_cli_command = 'dags trigger' -client = boto3.client('mwaa') - - -def lambda_handler(event, context): - for record in event['Records']: - print(f"[ RECORD ]: {record}") - s3_event_key = record['s3']['object']['key'] - print(f"[ S3 EVENT KEY ]: {s3_event_key}") - s3_filename_target = os.path.split(s3_event_key)[-1] - print(f"[ S3 FILENAME TARGET ]: {s3_filename_target}") - s3_filename_no_ext = os.path.splitext(s3_filename_target)[0] - print(f"[ S3 FILENAME NO EXT ]: {s3_filename_no_ext}") - - bucket_key_prefix = "EIS/FEDSoutput/Snapshot/" - if s3_filename_no_ext.startswith("lf_"): - bucket_key_prefix = "EIS/FEDSoutput/LFArchive/" - - # get web token - mwaa_cli_token = client.create_cli_token( - Name=mwaa_env_name - ) - print(f"[ CLI TOKEN ]: {mwaa_cli_token}") - serialized_args = json.dumps({ - "discovery": "s3", - "collection": s3_filename_no_ext, - "prefix": bucket_key_prefix, - "bucket": "veda-data-store-staging", - "filename_regex": f"^(.*){s3_filename_target}$", - "vector": True - }) - conn = http.client.HTTPSConnection(mwaa_cli_token['WebServerHostname']) - payload = f"{mwaa_cli_command} {dag_name} --conf '{serialized_args}'" - print(f"[ CLI PAYLOAD ]: {payload}") - headers = { - 'Authorization': 'Bearer ' + mwaa_cli_token['CliToken'], - 'Content-Type': 'text/plain' - } - conn.request("POST", "/aws_mwaa/cli", payload, headers) - res = conn.getresponse() - data = res.read() - dict_str = data.decode("UTF-8") - mydata = ast.literal_eval(dict_str) - print(f"[ DATA ]: {mydata}") - print(f"[ STDOUT ]: {base64.b64decode(mydata['stdout'])}") - return { - 'statusCode': 200, - 'body': json.dumps('Hello from Lambda!') - } \ No newline at end of file diff --git a/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/requirements.txt b/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/requirements.txt deleted file mode 100644 index 1db657b..0000000 --- a/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -boto3 \ No newline at end of file diff --git a/terraform/veda-wfs3/rds.tf b/terraform/veda-wfs3/rds.tf deleted file mode 100644 index e75f047..0000000 --- a/terraform/veda-wfs3/rds.tf +++ /dev/null @@ -1,65 +0,0 @@ -resource "aws_db_subnet_group" "db" { - name = "tf-${var.project_name}-${var.env}-subnet-group" - subnet_ids = module.networking.private_subnets_id - tags = { - Name = "tf-${var.project_name}-subnet-group" - } -} - -resource "aws_db_parameter_group" "default" { - name = "tf-${var.project_name}-${var.env}-postgres14-param-group" - family = "postgres14" - - parameter { - name = "work_mem" - # NOTE: I had `work_mem` set to ~100MB and `max_connections` around 75 and TileJSON completely failed - # 16MB - value = var.env == "staging" ? "16384" : "8192" - } - - parameter { - name = "max_connections" - value = "475" - apply_method = "pending-reboot" - } - -# NOTE: here to show what shared_buffers are but doesn't really make sense why it won't provision with these -# parameter { -# name = "shared_buffers" -# value = var.env == "staging" ? "8064856" : "4032428" -# apply_method = "pending-reboot" -# } - - parameter { - name = "seq_page_cost" - value = "1" - } - - parameter { - name = "random_page_cost" - value = "1.2" - } -} - -resource "aws_db_instance" "db" { - db_name = "veda" - identifier = "${var.project_name}-${var.env}" - engine = "postgres" - engine_version = "14.3" - // https://docs.aws.amazon.com/AmazonRDS/latest/APIReference/API_CreateDBInstance.html - allocated_storage = 100 - max_allocated_storage = 500 - storage_type = "gp2" - instance_class = var.env == "staging" ? "db.r5.xlarge" : "db.r5.large" - db_subnet_group_name = aws_db_subnet_group.db.name - vpc_security_group_ids = module.networking.security_groups_ids - skip_final_snapshot = true - apply_immediately = true - backup_retention_period = 7 - username = "postgres" - password = var.db_password - allow_major_version_upgrade = true - parameter_group_name = aws_db_parameter_group.default.name -} - - diff --git a/terraform/veda-wfs3/s3_event_bridge_lambda.tf b/terraform/veda-wfs3/s3_event_bridge_lambda.tf deleted file mode 100644 index 77ae894..0000000 --- a/terraform/veda-wfs3/s3_event_bridge_lambda.tf +++ /dev/null @@ -1,167 +0,0 @@ -##################################################### -# Execution Role -##################################################### -resource "aws_iam_role" "lambda_exec_role" { - provider = aws.west2 - name = "lambda-exec-role-s3-event-bridge-${var.project_name}-${var.env}" - tags = var.tags - - assume_role_policy = < Date: Sun, 4 Jun 2023 20:45:21 -0600 Subject: [PATCH 02/38] lambda db init --- db/Dockerfile | 13 ++ db/handler.py | 210 +++++++++++++++++++ scripts/build.sh | 21 ++ terraform/features-api/.terraform.lock.hcl | 81 ++++--- terraform/features-api/ecr.tf | 47 ++++- terraform/features-api/ecs_api.tf | 6 +- terraform/features-api/github_deploy_user.tf | 4 +- terraform/features-api/init.tf | 2 +- terraform/features-api/lambda.tf | 166 +++++++++++++++ terraform/features-api/rds.tf | 2 +- terraform/features-api/security_group.tf | 46 ++++ 11 files changed, 558 insertions(+), 40 deletions(-) create mode 100644 db/Dockerfile create mode 100644 db/handler.py create mode 100644 scripts/build.sh create mode 100644 terraform/features-api/lambda.tf diff --git a/db/Dockerfile b/db/Dockerfile new file mode 100644 index 0000000..6b428e0 --- /dev/null +++ b/db/Dockerfile @@ -0,0 +1,13 @@ +FROM public.ecr.aws/lambda/python:3.9 + +# WORKDIR /tmp + +RUN pip install boto3 requests "urllib3<2" psycopg["binary"] -t "${LAMBDA_TASK_ROOT}" + +COPY ./handler.py ${LAMBDA_TASK_ROOT} + +# https://stackoverflow.com/a/61746719 +# Turns out, asyncio is part of python +# RUN rm -rf /asset/asyncio* + +CMD ["handler.handler"] \ No newline at end of file diff --git a/db/handler.py b/db/handler.py new file mode 100644 index 0000000..6b220f5 --- /dev/null +++ b/db/handler.py @@ -0,0 +1,210 @@ +# """Bootstrap Postgres db.""" + +import asyncio +import json + +import boto3 +import psycopg +# import requests +from psycopg import sql +from psycopg.conninfo import make_conninfo +import os + +# def send( +# event, +# context, +# responseStatus, +# responseData, +# physicalResourceId=None, +# noEcho=False, +# ): +# """ +# Copyright 2016 Amazon Web Services, Inc. or its affiliates. All Rights Reserved. +# This file is licensed to you under the AWS Customer Agreement (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at http://aws.amazon.com/agreement/ . +# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. +# See the License for the specific language governing permissions and limitations under the License. + +# Send response from AWS Lambda. + +# Note: The cfnresponse module is available only when you use the ZipFile property to write your source code. +# It isn't available for source code that's stored in Amazon S3 buckets. +# For code in buckets, you must write your own functions to send responses. +# """ +# responseUrl = event["ResponseURL"] + +# print(responseUrl) + +# responseBody = {} +# responseBody["Status"] = responseStatus +# responseBody["Reason"] = ( +# "See the details in CloudWatch Log Stream: " + context.log_stream_name +# ) +# responseBody["PhysicalResourceId"] = physicalResourceId or context.log_stream_name +# responseBody["StackId"] = event["StackId"] +# responseBody["RequestId"] = event["RequestId"] +# responseBody["LogicalResourceId"] = event["LogicalResourceId"] +# responseBody["NoEcho"] = noEcho +# responseBody["Data"] = responseData + +# json_responseBody = json.dumps(responseBody) + +# print("Response body:\n" + json_responseBody) + +# headers = {"content-type": "", "content-length": str(len(json_responseBody))} + +# try: +# response = requests.put(responseUrl, data=json_responseBody, headers=headers) +# print("Status code: " + response.reason) +# except Exception as e: +# print("send(..) failed executing requests.put(..): " + str(e)) + + +def get_secret(secret_name): + """Get Secrets from secret manager.""" + print(f"Fetching {secret_name}...") + print("This is a not a test...") + client = boto3.client(service_name="secretsmanager") + response = client.get_secret_value(SecretId=secret_name) + return json.loads(response["SecretString"]) + + +def create_db(cursor, db_name: str) -> None: + """Create DB.""" + cursor.execute( + sql.SQL("SELECT 1 FROM pg_catalog.pg_database " "WHERE datname = %s"), [db_name] + ) + if cursor.fetchone(): + print(f"database {db_name} exists, not creating DB") + else: + print(f"database {db_name} not found, creating...") + cursor.execute( + sql.SQL("CREATE DATABASE {db_name}").format(db_name=sql.Identifier(db_name)) + ) + + +def create_user(cursor, username: str, password: str) -> None: + """Create User.""" + cursor.execute( + sql.SQL( + "DO $$ " + "BEGIN " + " IF NOT EXISTS ( " + " SELECT 1 FROM pg_roles " + " WHERE rolname = {user}) " + " THEN " + " CREATE USER {username} " + " WITH PASSWORD {password}; " + " ELSE " + " ALTER USER {username} " + " WITH PASSWORD {password}; " + " END IF; " + "END " + "$$; " + ).format(username=sql.Identifier(username), password=password, user=username) + ) + + +def create_permissions(cursor, db_name: str, username: str) -> None: + """Add permissions.""" + cursor.execute( + sql.SQL( + "GRANT CONNECT ON DATABASE {db_name} TO {username};" + "GRANT CREATE ON DATABASE {db_name} TO {username};" # Allow schema creation + "GRANT USAGE ON SCHEMA public TO {username};" + "ALTER DEFAULT PRIVILEGES IN SCHEMA public " + "GRANT ALL PRIVILEGES ON TABLES TO {username};" + "ALTER DEFAULT PRIVILEGES IN SCHEMA public " + "GRANT ALL PRIVILEGES ON SEQUENCES TO {username};" + ).format( + db_name=sql.Identifier(db_name), + username=sql.Identifier(username), + ) + ) + + +def register_extensions(cursor) -> None: + """Add PostGIS extension.""" + cursor.execute(sql.SQL("CREATE EXTENSION IF NOT EXISTS postgis;")) + + +def handler(event, context): + """Lambda Handler.""" + print(f"Event: {event}") + + if event["tf"]["action"] not in ["create", "update"]: + print("failed") + return 0 + # return send(event, context, "SUCCESS", {"msg": "No action to be taken"}) + + try: + connection_params = get_secret(os.environ['CONN_SECRET_ARN']) + user_params = event["user_params"] + print("Connecting to DB...") + con_str = make_conninfo( + dbname=connection_params.get("dbname", "postgres"), + user=connection_params["username"], + password=connection_params["password"], + host=connection_params["host"], + port=connection_params["port"], + ) + with psycopg.connect(con_str, autocommit=True) as conn: + with conn.cursor() as cur: + print("Creating database...") + create_db( + cursor=cur, + db_name=user_params["dbname"], + ) + + print("Creating user...") + create_user( + cursor=cur, + username=user_params["username"], + password=user_params["password"], + ) + + print("Setting permissions...") + create_permissions( + cursor=cur, + db_name=user_params["dbname"], + username=user_params["username"], + ) + + # Install extensions on the user DB with + # superuser permissions, since they will + # otherwise fail to install when run as + # the non-superuser within the pgstac + # migrations. + print("Connecting to DB...") + con_str = make_conninfo( + dbname=user_params["dbname"], + user=connection_params["username"], + password=connection_params["password"], + host=connection_params["host"], + port=connection_params["port"], + ) + with psycopg.connect(con_str, autocommit=True) as conn: + with conn.cursor() as cur: + print("Registering PostGIS ...") + register_extensions(cursor=cur) + + except Exception as e: + print(e) + return { + 'message' : e + } + + print("Event Complete") + return { + 'message' : connection_params + } + +# def handler(event, context): +# # if event["tf"]["action"] in ["create", "update"]: +# message = 'Hello There {} {}!'.format(event['first_name'], event['last_name']) +# print(event["tf"]) +# print(f"Hello there, {event['first_name']} {event['last_name']}") +# return { +# 'message' : message +# } diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100644 index 0000000..32b6f87 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +# aws ecr describe-repositories \ +# | jq '.repositories | map(.repositoryUri)' \ +# | grep $TARGET_PROJECT_NAME | grep $TARGET_ENVIRONMENT \ +# | xargs -I {} bash -c "aws ecr get-login-password | docker login --username AWS --password-stdin {}" + +# aws ecr describe-repositories \ +# | jq '.repositories | map(.repositoryUri)' \ +# | grep $TARGET_PROJECT_NAME | grep $TARGET_ENVIRONMENT \ +# | sed -E 's/"|,//g' \ +# | xargs -I {} docker build -t {}:latest ../wfs3-app/ + +aws ecr describe-repositories \ + | jq '.repositories | map(.repositoryUri)' \ + | grep $TARGET_PROJECT_NAME | grep $TARGET_ENVIRONMENT \ + | sed -E 's/"|,//g' \ + | xargs -I {} docker images --format "{{json . }}" {} \ + | grep '"Tag":"latest"' \ + | jq '"\(.Repository):\(.Tag)"' \ + | xargs -I{} docker push {} \ No newline at end of file diff --git a/terraform/features-api/.terraform.lock.hcl b/terraform/features-api/.terraform.lock.hcl index 6fc5d64..16eeaeb 100644 --- a/terraform/features-api/.terraform.lock.hcl +++ b/terraform/features-api/.terraform.lock.hcl @@ -2,45 +2,62 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/aws" { - version = "4.58.0" - constraints = "~> 4.0" + version = "5.1.0" + constraints = ">= 3.29.0, ~> 5.0" hashes = [ - "h1:YIRXIr1ji0HLWLU0ae+UbUNOHc9MJaLrMHxH3LIQ/Vk=", - "h1:znLROwEAINbYzAG5X7Ep04whM7KxkQGrvhFdhSvNKEk=", - "zh:14b2b2dfbc7ee705c412d762b1485ee08958c816a64ac74f5769e946e4a1d265", - "zh:17a37e6825e2023b18987d31c0cbb9336654ea146b68e6c90710ea4636af71ae", - "zh:273127c69fb244577e5c136c46164d34f77b0c956c18d27f63d1072dd558f924", - "zh:4b2b6416d34fb3e1051c99d2a84045b136976140e34381d5fbf90e32db15272e", - "zh:7e6a8571ff15d51f892776265642ee01004b8553fd4f6f2014b6f3f2834670c7", - "zh:847c76ab2381b66666d0f79cf1ac697b5bfd0d9c3009fd11bc6ad6545d1eb427", - "zh:9a52cae08ba8d27d0639a8d2b8c61591027883058bf0cc5a639cffe1e299f019", + "h1:iDyYmwv8q94Dvr4DRG1KBxTWPZRFkRmKGa3cjCEsPZU=", + "zh:0c48f157b804c1f392adb5c14b81e756c652755e358096300ea8dd1283021129", + "zh:1a50495a6c0e5665e51df57dac6e781ec71439b11ebf05f971b6f3a3eb4eb7b2", + "zh:2959ff472c05e56d59e012118dd8d55022f005534c0ae961ce81136de9f66a4d", + "zh:2dfda9133581b99ed6e709e89a453fd2974ce88c703d3e073ec31bf99d7508ce", + "zh:2f3d92cc7a6624da42cee2202f8fb23e6d38f156ab7851884d637282cb0dc709", + "zh:3bc2a34d09cbaf439a1815846904f070c782cd8dfd60b5e0116827cda25f7549", + "zh:4ef43f1a247aa8de8690ac3bbc2b00ebaf6b2872fc8d0f5130e4a8130c874b87", + "zh:5477cb272dcaeb0030091bcf23a9f0f33b5410e44e317e9d3d49446f545dbaa4", + "zh:734c8fb4c0b79c82dd757566761dda5b91ee1ef9a2b848a748ade11e0e1cc69f", + "zh:80346c051b677f4f018da7fe06318b87c5bd0f1ec67ce78ab33baed3bb8b031a", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:9df647e8322d6f94f1843366ba39d21c4b36c8e7dcdc03711d52e27f73b0e974", - "zh:9e52037e68409802ff913b166c30e3f2035af03865cbef0c1b03762bce853941", - "zh:a30288e7c3c904d6998d1709835d7c5800a739f8608f0837f960286a2b8b6e59", - "zh:a7f24e3bda3be566468e4ad62cef1016f68c6f5a94d2e3e979485bc05626281b", - "zh:ba326ba80f5e39829b67a6d1ce54ba52b171e5e13a0a91ef5f9170a9b0cc9ce4", - "zh:c4e3fe9f2be6e244a3dfce599f4b0be9e8fffaece64cbc65f3195f825f65489b", - "zh:f20a251af37039bb2c7612dbd2c5df3a25886b4cc78f902385a2850ea6e30d08", + "zh:a865b2f88dfee13df14116c5cf53d033d2c15855f4b59b9c65337309a928df2c", + "zh:c0345f266eedaece5612c1000722b302f895d1bc5af1d5a4265f0e7000ca48bb", + "zh:d59703c8e6a9d8b4fbd3b4583b945dfff9cb2844c762c0b3990e1cef18282279", + "zh:d8d04a6a6cd2dfcb23b57e551db7b15e647f6166310fb7d883d8ec67bdc9bdc8", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.1" + hashes = [ + "h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=", + "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", + "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", + "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", + "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", + "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", + "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", + "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", + "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", + "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", + "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", ] } provider "registry.terraform.io/hashicorp/random" { - version = "3.4.3" + version = "3.5.1" hashes = [ - "h1:saZR+mhthL0OZl4SyHXZraxyaBNVMxiZzks78nWcZ2o=", - "h1:tL3katm68lX+4lAncjQA9AXL4GR/VM+RPwqYf4D2X8Q=", - "zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752", - "zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b", - "zh:686ad1ee40b812b9e016317e7f34c0d63ef837e084dea4a1f578f64a6314ad53", + "h1:IL9mSatmwov+e0+++YX2V6uel+dV6bn+fC/cnGDK3Ck=", + "zh:04e3fbd610cb52c1017d282531364b9c53ef72b6bc533acb2a90671957324a64", + "zh:119197103301ebaf7efb91df8f0b6e0dd31e6ff943d231af35ee1831c599188d", + "zh:4d2b219d09abf3b1bb4df93d399ed156cadd61f44ad3baf5cf2954df2fba0831", + "zh:6130bdde527587bbe2dcaa7150363e96dbc5250ea20154176d82bc69df5d4ce3", + "zh:6cc326cd4000f724d3086ee05587e7710f032f94fc9af35e96a386a1c6f2214f", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:84103eae7251384c0d995f5a257c72b0096605048f757b749b7b62107a5dccb3", - "zh:8ee974b110adb78c7cd18aae82b2729e5124d8f115d484215fd5199451053de5", - "zh:9dd4561e3c847e45de603f17fa0c01ae14cae8c4b7b4e6423c9ef3904b308dda", - "zh:bb07bb3c2c0296beba0beec629ebc6474c70732387477a65966483b5efabdbc6", - "zh:e891339e96c9e5a888727b45b2e1bb3fcbdfe0fd7c5b4396e4695459b38c8cb1", - "zh:ea4739860c24dfeaac6c100b2a2e357106a89d18751f7693f3c31ecf6a996f8d", - "zh:f0c76ac303fd0ab59146c39bc121c5d7d86f878e9a69294e29444d4c653786f8", - "zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93", + "zh:b6d88e1d28cf2dfa24e9fdcc3efc77adcdc1c3c3b5c7ce503a423efbdd6de57b", + "zh:ba74c592622ecbcef9dc2a4d81ed321c4e44cddf7da799faa324da9bf52a22b2", + "zh:c7c5cde98fe4ef1143bd1b3ec5dc04baf0d4cc3ca2c5c7d40d17c0e9b2076865", + "zh:dac4bad52c940cd0dfc27893507c1e92393846b024c5a9db159a93c534a3da03", + "zh:de8febe2a2acd9ac454b844a4106ed295ae9520ef54dc8ed2faf29f12716b602", + "zh:eab0d0495e7e711cca367f7d4df6e322e6c562fc52151ec931176115b83ed014", ] } diff --git a/terraform/features-api/ecr.tf b/terraform/features-api/ecr.tf index b09467a..f53a548 100644 --- a/terraform/features-api/ecr.tf +++ b/terraform/features-api/ecr.tf @@ -1,4 +1,4 @@ -module "ecr_registry" { +module "ecr_registry_wfs" { source = "github.com/developmentseed/tf-seed/modules/aws_ecr" environment = var.env registry_name = var.project_name @@ -7,4 +7,49 @@ module "ecr_registry" { enable_deploy_user = true iam_deploy_username = aws_iam_user.deploy_user.name tags = var.tags +} + +module "ecr_registry_db" { + source = "github.com/developmentseed/tf-seed/modules/aws_ecr" + environment = var.env + registry_name = "${var.project_name}-db" + enable_registry_scanning = true + mutable_image_tags = true + enable_deploy_user = true + iam_deploy_username = aws_iam_user.deploy_user.name + tags = var.tags +} + +resource "null_resource" "build_ecr_image_wfs" { + triggers = { + handler_file_path = filemd5("../../wfs3-app/fast_api_main.py") + docker_file_path = filemd5("../../wfs3-app/Dockerfile") + } + + provisioner "local-exec" { + command = < Date: Sun, 4 Jun 2023 20:56:32 -0600 Subject: [PATCH 03/38] tf resource renaming --- terraform/features-api/lambda.tf | 68 ++++++++---------------- terraform/features-api/security_group.tf | 19 ------- 2 files changed, 21 insertions(+), 66 deletions(-) diff --git a/terraform/features-api/lambda.tf b/terraform/features-api/lambda.tf index fd7cd56..4f19ef1 100644 --- a/terraform/features-api/lambda.tf +++ b/terraform/features-api/lambda.tf @@ -2,15 +2,15 @@ resource "aws_lambda_invocation" "db_init" { function_name = aws_lambda_function.lambda_init_db.function_name input = jsonencode({ - "user_params":{ - "username":"username" - "password":"password" - "dbname":"ghgc" + "user_params" : { + "username" : "username" + "password" : "password" + "dbname" : "ghgc" } }) triggers = { - folder_path = sha1(join("", [for f in fileset("../../db", "*") : filesha1("../../db/${f}")])) + folder_path = sha1(join("", [for f in fileset("../../db", "*") : filesha1("../../db/${f}")])) } # triggers = { @@ -19,10 +19,9 @@ resource "aws_lambda_invocation" "db_init" { # } lifecycle_scope = "CRUD" - qualifier = "$LATEST" + qualifier = "$LATEST" } - data "aws_iam_policy_document" "lambda_assume_role_policy" { statement { @@ -40,7 +39,7 @@ data "aws_iam_policy_document" "lambda_assume_role_policy" { data "aws_iam_policy_document" "lambda_policy" { statement { - + actions = [ "secretsmanager:DescribeSecret", "secretsmanager:GetSecretValue", @@ -60,10 +59,10 @@ data "aws_iam_policy_document" "lambda_policy" { effect = "Allow" actions = [ "ec2:DescribeNetworkInterfaces", - "ec2:CreateNetworkInterface", - "ec2:DeleteNetworkInterface", - "ec2:DescribeInstances", - "ec2:AttachNetworkInterface" + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeInstances", + "ec2:AttachNetworkInterface" ] resources = ["*"] @@ -72,7 +71,7 @@ data "aws_iam_policy_document" "lambda_policy" { statement { effect = "Allow" - actions = [ + actions = [ "logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents", @@ -95,53 +94,28 @@ data "aws_iam_policy_document" "lambda_policy" { } resource "aws_iam_role" "iam_for_lambda" { - name = "iam_for_lambda" + name = "${var.project_name}-${var.env}-lambda-initdb-role" assume_role_policy = data.aws_iam_policy_document.lambda_assume_role_policy.json } resource "aws_iam_role_policy" "lambda_execution_role_policy" { - name = "${var.project_name}-api-access-secret-manager-lambda" + name = "${var.project_name}-${var.env}-api-access-secret-manager-lambda" role = aws_iam_role.iam_for_lambda.id policy = data.aws_iam_policy_document.lambda_policy.json } - -# module "lambda_security_group" { -# source = "terraform-aws-modules/security-group/aws" -# version = "~> 4" - -# name = "${var.project_name}-${var.env}-lambda-db-init" -# description = "Lambda PG init security group" -# vpc_id = var.vpc_id -# egress_with_cidr_blocks = [ -# { -# from_port = 0 -# to_port = 0 -# protocol = "-1" -# description = "Allow all" -# cidr_blocks = "0.0.0.0/0" -# } -# ] -# } - -resource "aws_cloudwatch_log_group" "example" { - name = "/aws/lambda/${var.project_name}-initdb-function" +resource "aws_cloudwatch_log_group" "lambda_cloudwatch_group" { + name = "/aws/lambda/${var.project_name}-${var.env}-initdb-function" retention_in_days = 14 } -data "aws_ecr_image" "service_image" { - repository_name = module.ecr_registry_db.registry_name - image_tag = "latest" -} - resource "aws_lambda_function" "lambda_init_db" { code_signing_config_arn = "" description = "Lambda function to init medium DB" - image_uri = "${module.ecr_registry_db.repository_url}:latest" - function_name = "${var.project_name}-initdb-function" + image_uri = "${module.ecr_registry_db.repository_url}:latest" + function_name = "${var.project_name}-${var.env}-initdb-function" role = aws_iam_role.iam_for_lambda.arn - package_type = "Image" - source_code_hash = trimprefix(data.aws_ecr_image.service_image.id, "sha256:") + package_type = "Image" image_config { command = ["handler.handler"] @@ -149,12 +123,12 @@ resource "aws_lambda_function" "lambda_init_db" { depends_on = [ aws_iam_role_policy.lambda_execution_role_policy, - aws_cloudwatch_log_group.example, + aws_cloudwatch_log_group.lambda_cloudwatch_group, aws_db_instance.db ] vpc_config { - subnet_ids = data.aws_subnets.private.ids + subnet_ids = data.aws_subnets.private.ids security_group_ids = [aws_security_group.lambda-db-init.id] } environment { diff --git a/terraform/features-api/security_group.tf b/terraform/features-api/security_group.tf index 3d18cf4..568282a 100644 --- a/terraform/features-api/security_group.tf +++ b/terraform/features-api/security_group.tf @@ -22,25 +22,6 @@ resource "aws_security_group" "lambda-db-init" { } } - -# module "lambda_security_group" { -# source = "terraform-aws-modules/security-group/aws" -# version = "~> 4" - -# name = "${var.project_name}-${var.env}-lambda-db-init" -# description = "Lambda PG init security group" -# vpc_id = var.vpc_id -# egress_with_cidr_blocks = [ -# { -# from_port = 0 -# to_port = 0 -# protocol = "-1" -# description = "Allow all" -# cidr_blocks = "0.0.0.0/0" -# } -# ] -# } - resource "aws_security_group_rule" "ecs_service_port_addon" { description = "opened for ECS service port" type = "ingress" From 7c0f68ed0819140f8520ba468c1d1230f7deb5d6 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Mon, 5 Jun 2023 09:17:09 -0600 Subject: [PATCH 04/38] minor changes --- db/handler.py | 12 +----- terraform/features-api/lambda.tf | 63 +++++++++++++++++--------------- 2 files changed, 34 insertions(+), 41 deletions(-) diff --git a/db/handler.py b/db/handler.py index 6b220f5..16b4702 100644 --- a/db/handler.py +++ b/db/handler.py @@ -64,7 +64,6 @@ def get_secret(secret_name): """Get Secrets from secret manager.""" print(f"Fetching {secret_name}...") - print("This is a not a test...") client = boto3.client(service_name="secretsmanager") response = client.get_secret_value(SecretId=secret_name) return json.loads(response["SecretString"]) @@ -198,13 +197,4 @@ def handler(event, context): print("Event Complete") return { 'message' : connection_params - } - -# def handler(event, context): -# # if event["tf"]["action"] in ["create", "update"]: -# message = 'Hello There {} {}!'.format(event['first_name'], event['last_name']) -# print(event["tf"]) -# print(f"Hello there, {event['first_name']} {event['last_name']}") -# return { -# 'message' : message -# } + } \ No newline at end of file diff --git a/terraform/features-api/lambda.tf b/terraform/features-api/lambda.tf index 4f19ef1..8fae03b 100644 --- a/terraform/features-api/lambda.tf +++ b/terraform/features-api/lambda.tf @@ -1,3 +1,33 @@ +resource "aws_lambda_function" "lambda_init_db" { + code_signing_config_arn = "" + description = "Lambda function to init medium DB" + image_uri = "${module.ecr_registry_db.repository_url}:latest" + function_name = "${var.project_name}-${var.env}-initdb-function" + role = aws_iam_role.iam_for_lambda.arn + package_type = "Image" + + image_config { + command = ["handler.handler"] + } + + depends_on = [ + aws_iam_role_policy.lambda_execution_role_policy, + aws_cloudwatch_log_group.lambda_cloudwatch_group, + aws_db_instance.db + ] + + vpc_config { + subnet_ids = data.aws_subnets.private.ids + security_group_ids = [aws_security_group.lambda-db-init.id] + } + environment { + variables = { + CONN_SECRET_ARN = aws_secretsmanager_secret.db_config.arn + } + } +} + + resource "aws_lambda_invocation" "db_init" { function_name = aws_lambda_function.lambda_init_db.function_name @@ -9,9 +39,9 @@ resource "aws_lambda_invocation" "db_init" { } }) - triggers = { - folder_path = sha1(join("", [for f in fileset("../../db", "*") : filesha1("../../db/${f}")])) - } + # triggers = { + # folder_path = sha1(join("", [for f in fileset("../../db", "*") : filesha1("../../db/${f}")])) + # } # triggers = { # handler_file_path = filemd5("../../db/handler.py") @@ -109,32 +139,5 @@ resource "aws_cloudwatch_log_group" "lambda_cloudwatch_group" { retention_in_days = 14 } -resource "aws_lambda_function" "lambda_init_db" { - code_signing_config_arn = "" - description = "Lambda function to init medium DB" - image_uri = "${module.ecr_registry_db.repository_url}:latest" - function_name = "${var.project_name}-${var.env}-initdb-function" - role = aws_iam_role.iam_for_lambda.arn - package_type = "Image" - - image_config { - command = ["handler.handler"] - } - - depends_on = [ - aws_iam_role_policy.lambda_execution_role_policy, - aws_cloudwatch_log_group.lambda_cloudwatch_group, - aws_db_instance.db - ] - vpc_config { - subnet_ids = data.aws_subnets.private.ids - security_group_ids = [aws_security_group.lambda-db-init.id] - } - environment { - variables = { - CONN_SECRET_ARN = aws_secretsmanager_secret.db_config.arn - } - } -} From c3ed84f99c0f4cd20e3a5f6f1e2b4b420bf22cfc Mon Sep 17 00:00:00 2001 From: smohiudd Date: Mon, 5 Jun 2023 13:05:15 -0600 Subject: [PATCH 05/38] minor tf changes --- README.md | 4 ++-- terraform/features-api/ecr.tf | 10 ++++++---- terraform/features-api/ecs_api.tf | 2 +- terraform/features-api/variables.tf | 8 +++++++- terraform/features-api/vars/dev.tf | 5 +---- wfs3-app/fast_api_main.py | 16 ++++++++-------- 6 files changed, 25 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index f48cd6e..8add51e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# VEDA Features API +# GHGC Features API -Hosting and serving collections of vector data features for VEDA +Hosting and serving collections of vector data features for GHGC --- diff --git a/terraform/features-api/ecr.tf b/terraform/features-api/ecr.tf index f53a548..2424dd3 100644 --- a/terraform/features-api/ecr.tf +++ b/terraform/features-api/ecr.tf @@ -22,8 +22,9 @@ module "ecr_registry_db" { resource "null_resource" "build_ecr_image_wfs" { triggers = { - handler_file_path = filemd5("../../wfs3-app/fast_api_main.py") - docker_file_path = filemd5("../../wfs3-app/Dockerfile") + folder_path = sha1(join("", [for f in fileset("../../wfs3-app", "*") : filesha1("../../wfs3-app/${f}")])) + # handler_file_path = filemd5("../../wfs3-app/fast_api_main.py") + # docker_file_path = filemd5("../../wfs3-app/Dockerfile") } provisioner "local-exec" { @@ -39,8 +40,9 @@ resource "null_resource" "build_ecr_image_wfs" { resource "null_resource" "build_ecr_image_db_init" { triggers = { - handler_file_path = filemd5("../../db/handler.py") - docker_file_path = filemd5("../../db/Dockerfile") + folder_path = sha1(join("", [for f in fileset("../../db", "*") : filesha1("../../db/${f}")])) + # handler_file_path = filemd5("../../db/handler.py") + # docker_file_path = filemd5("../../db/Dockerfile") } provisioner "local-exec" { diff --git a/terraform/features-api/ecs_api.tf b/terraform/features-api/ecs_api.tf index 3cf01d4..d9d8a84 100644 --- a/terraform/features-api/ecs_api.tf +++ b/terraform/features-api/ecs_api.tf @@ -5,7 +5,7 @@ data "aws_subnets" "private" { } tags = { - "aws-cdk:subnet-name" = "private" + "aws-cdk:subnet-name" = var.db_public_subnet ? "public" : "private" } } diff --git a/terraform/features-api/variables.tf b/terraform/features-api/variables.tf index add1337..7ce71a4 100755 --- a/terraform/features-api/variables.tf +++ b/terraform/features-api/variables.tf @@ -44,7 +44,13 @@ variable "dns_subdomain" { } -variable "alb_protocol" {} +variable "alb_protocol" { + default = "HTTPS" +} variable "vpc_id" {} +variable "db_public_subnet" { + type = bool + default = true +} diff --git a/terraform/features-api/vars/dev.tf b/terraform/features-api/vars/dev.tf index 70dc63e..20a304a 100644 --- a/terraform/features-api/vars/dev.tf +++ b/terraform/features-api/vars/dev.tf @@ -2,13 +2,10 @@ region = "us-west-2" registry_name = "features-api-registry" env = "dev" project_name = "veda-features-api" +db_public_subnet = false availability_zones = ["us-west-2a", "us-west-2b"] service_port = 8080 dns_zone_name = "delta-backend.com" dns_subdomain = "ghg-dev" -alb_protocol = "HTTPS" tags = {"project": "veda", "service": "veda-features-api-dev"} -default_secret = { - "noop": "boop", -} vpc_id = "vpc-0512162c42da5e645" diff --git a/wfs3-app/fast_api_main.py b/wfs3-app/fast_api_main.py index 4fb55b3..5e747e4 100644 --- a/wfs3-app/fast_api_main.py +++ b/wfs3-app/fast_api_main.py @@ -130,13 +130,13 @@ async def ping(): return JSONResponse(status_code=200, content={"ping": "pong"}) -@app.get("/refresh") -async def refresh(request: Request): - """Return parsed catalog data for testing.""" - with tracer.start_as_current_span("refresh"): - refresh_counter.add(1, {"refresh": "count"}) - await connect_to_db(app, settings=postgresql_settings) - await register_collection_catalog(app) - return JSONResponse(status_code=200, content={"status": "refreshed"}) +# @app.get("/refresh") +# async def refresh(request: Request): +# """Return parsed catalog data for testing.""" +# with tracer.start_as_current_span("refresh"): +# refresh_counter.add(1, {"refresh": "count"}) +# await connect_to_db(app, settings=postgresql_settings) +# await register_collection_catalog(app) +# return JSONResponse(status_code=200, content={"status": "refreshed"}) FastAPIInstrumentor.instrument_app(app, excluded_urls="/conformance,/healthz") From 956c2edd6f295d4dc963398216a81bbb4400a2b5 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Tue, 6 Jun 2023 16:48:07 -0500 Subject: [PATCH 06/38] Add cicd --- .github/actions/terraform-deploy.yml | 73 ++++++++++++++++++++++++++++ .github/workflows/cicd.yml | 58 ++++++++++++++++++++++ scripts/sync-env.sh | 6 +++ 3 files changed, 137 insertions(+) create mode 100644 .github/actions/terraform-deploy.yml create mode 100644 .github/workflows/cicd.yml create mode 100644 scripts/sync-env.sh diff --git a/.github/actions/terraform-deploy.yml b/.github/actions/terraform-deploy.yml new file mode 100644 index 0000000..8e5b6e0 --- /dev/null +++ b/.github/actions/terraform-deploy.yml @@ -0,0 +1,73 @@ +name: Deploy + +inputs: + env_aws_secret_name: + required: true + type: string + env-file: + required: true + type: string + dir: + required: false + type: string + default: "." + +runs: + using: "composite" + + steps: + - name: Set up Python + if: env.infra_deploy + uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: Install python dependencies + if: env.infra_deploy + shell: bash + working-directory: ${{ inputs.dir }} + run: pip install -r deploy_requirements.txt + + - name: Get relevant environment configuration from aws secrets + shell: bash + working-directory: ${{ inputs.dir }} + run: | + ./scripts/sync-env.sh ${{ inputs.env_aws_secret_name }} + + - name: Setup Terraform + if: env.infra_deploy + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.3.3 + + # - name: Deploy + # if: env.infra_deploy + # shell: bash + # working-directory: ${{ inputs.dir }} + # run: | + # ./scripts/deploy.sh ${{ inputs.env-file }} <<< init + # ./scripts/deploy.sh ${{ inputs.env-file }} <<< deploy + + - name: Docker build, tag, and push image to Amazon ECR + env: + IMAGE_TAG: latest + ECR_REGISTRY: format('{0}.dkr.ecr.{1}.amazonaws.com/{2}-registry-{3}', env.ACCOUNT_ID, env.AWS_REGION, env.APP_NAME, env.STAGE) + run: | + echo $ECR_REGISTRY + echo $IMAGE_TAG + # aws ecr get-login-password | docker login --username AWS --password-stdin $ECR_REGISTRY + # cd veda-wfs3-app + # docker build -t $ECR_REGISTRY }}:$IMAGE_TAG }} . + # docker push $ECR_REGISTRY }}:$IMAGE_TAG }} + + - name: ECS refresh service + env: + ECS_SERVICE_NAME: format('{0}-service-{1}', env.APP_NAME, env.STAGE) + run: | + echo $ECS_SERVICE_NAME + # aws ecs update-service \ + # --cluster $ECS_SERVICE_NAME \ + # --service $ECS_SERVICE_NAME \ + # --task-definition $ECS_SERVICE_NAME \ + # --force-new-deployment \ No newline at end of file diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml new file mode 100644 index 0000000..662b597 --- /dev/null +++ b/.github/workflows/cicd.yml @@ -0,0 +1,58 @@ +name: CICD 🚀 + +permissions: + id-token: write + contents: read + +on: + push: + branches: + - main + - dev + - production + - update-workflows + +jobs: + define-environment: + name: Set ✨ environment ✨ + runs-on: ubuntu-latest + steps: + - name: Set the environment based on the branch + id: define_environment + run: | + if [ "${{ github.ref }}" = "refs/heads/main" ]; then + echo "env_name=staging" >> $GITHUB_OUTPUT + elif [ "${{ github.ref }}" = "refs/heads/dev" ]; then + echo "env_name=development" >> $GITHUB_OUTPUT + elif [ "${{ github.ref }}" = "refs/heads/production" ]; then + echo "env_name=production" >> $GITHUB_OUTPUT + fi + - name: Print the environment + run: echo "The environment is ${{ steps.define_environment.outputs.env_name }}" + + outputs: + env_name: ${{ steps.define_environment.outputs.env_name }} + + deploy: + name: Deploy to ${{ needs.define-environment.outputs.env_name }} 🚀 + runs-on: ubuntu-latest + needs: [define-environment] + if: ${{ needs.define-environment.outputs.env_name }} + environment: ${{ needs.define-environment.outputs.env_name }} + concurrency: ${{ needs.define-environment.outputs.env_name }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "ghgc-features-api-github-${{ needs.define-environment.outputs.env_name }}-deployment" + aws-region: "us-west-2" + + - name: Run deployment + uses: "./.github/actions/cdk-deploy" + with: + env_aws_secret_name: ${{ secrets.ENV_AWS_SECRET_NAME }} diff --git a/scripts/sync-env.sh b/scripts/sync-env.sh new file mode 100644 index 0000000..3758445 --- /dev/null +++ b/scripts/sync-env.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Use this script to load environment variables for a deployment from AWS Secrets + +for s in $(aws secretsmanager get-secret-value --secret-id $1 --query SecretString --output text | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do + echo "$s" >> $GITHUB_ENV +done From 19b4d31e161cded3c64edc49c798f325acf14375 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Tue, 6 Jun 2023 16:49:30 -0500 Subject: [PATCH 07/38] Assign env to update-workflows --- .github/workflows/cicd.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 662b597..a8e4a4c 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -26,6 +26,8 @@ jobs: echo "env_name=development" >> $GITHUB_OUTPUT elif [ "${{ github.ref }}" = "refs/heads/production" ]; then echo "env_name=production" >> $GITHUB_OUTPUT + elif [ "${{ github.ref }}" = "refs/heads/update-workflows" ]; then + echo "env_name=development" >> $GITHUB_OUTPUT fi - name: Print the environment run: echo "The environment is ${{ steps.define_environment.outputs.env_name }}" From 648dd82200166d2d779814d2953e168a337bd55b Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Tue, 6 Jun 2023 16:50:30 -0500 Subject: [PATCH 08/38] Rename the action ref --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index a8e4a4c..5a2f99f 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -55,6 +55,6 @@ jobs: aws-region: "us-west-2" - name: Run deployment - uses: "./.github/actions/cdk-deploy" + uses: "./.github/actions/terraform-deploy" with: env_aws_secret_name: ${{ secrets.ENV_AWS_SECRET_NAME }} From 4d504a77336afb92082872b7ec18a10f602d7a86 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Tue, 6 Jun 2023 16:52:04 -0500 Subject: [PATCH 09/38] Move action to correct location --- .../actions/{terraform-deploy.yml => terraform-deploy/action.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/actions/{terraform-deploy.yml => terraform-deploy/action.yml} (100%) diff --git a/.github/actions/terraform-deploy.yml b/.github/actions/terraform-deploy/action.yml similarity index 100% rename from .github/actions/terraform-deploy.yml rename to .github/actions/terraform-deploy/action.yml From b535117014871a93f096f03acc10c10834f9dcb8 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Tue, 6 Jun 2023 16:53:07 -0500 Subject: [PATCH 10/38] Add `shell` --- .github/actions/terraform-deploy/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/terraform-deploy/action.yml b/.github/actions/terraform-deploy/action.yml index 8e5b6e0..59e5d27 100644 --- a/.github/actions/terraform-deploy/action.yml +++ b/.github/actions/terraform-deploy/action.yml @@ -50,6 +50,7 @@ runs: # ./scripts/deploy.sh ${{ inputs.env-file }} <<< deploy - name: Docker build, tag, and push image to Amazon ECR + shell: bash env: IMAGE_TAG: latest ECR_REGISTRY: format('{0}.dkr.ecr.{1}.amazonaws.com/{2}-registry-{3}', env.ACCOUNT_ID, env.AWS_REGION, env.APP_NAME, env.STAGE) @@ -62,6 +63,7 @@ runs: # docker push $ECR_REGISTRY }}:$IMAGE_TAG }} - name: ECS refresh service + shell: bash env: ECS_SERVICE_NAME: format('{0}-service-{1}', env.APP_NAME, env.STAGE) run: | From 7bed530c2eed3c5b20ef9637770f8a987d5269f2 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Tue, 6 Jun 2023 16:55:07 -0500 Subject: [PATCH 11/38] Permit script to run --- scripts/sync-env.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/sync-env.sh diff --git a/scripts/sync-env.sh b/scripts/sync-env.sh old mode 100644 new mode 100755 From 29cf3184bf0a77b33740e6dfeac6f2466a1edddd Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Tue, 6 Jun 2023 16:58:39 -0500 Subject: [PATCH 12/38] Add expression parenthesis --- .github/actions/terraform-deploy/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/terraform-deploy/action.yml b/.github/actions/terraform-deploy/action.yml index 59e5d27..9d07eab 100644 --- a/.github/actions/terraform-deploy/action.yml +++ b/.github/actions/terraform-deploy/action.yml @@ -53,7 +53,7 @@ runs: shell: bash env: IMAGE_TAG: latest - ECR_REGISTRY: format('{0}.dkr.ecr.{1}.amazonaws.com/{2}-registry-{3}', env.ACCOUNT_ID, env.AWS_REGION, env.APP_NAME, env.STAGE) + ECR_REGISTRY: ${{ format('{0}.dkr.ecr.{1}.amazonaws.com/{2}-registry-{3}', env.ACCOUNT_ID, env.AWS_REGION, env.APP_NAME, env.STAGE) }} run: | echo $ECR_REGISTRY echo $IMAGE_TAG @@ -65,11 +65,11 @@ runs: - name: ECS refresh service shell: bash env: - ECS_SERVICE_NAME: format('{0}-service-{1}', env.APP_NAME, env.STAGE) + ECS_SERVICE_NAME: ${{ format('{0}-service-{1}', env.APP_NAME, env.STAGE) }} run: | echo $ECS_SERVICE_NAME # aws ecs update-service \ # --cluster $ECS_SERVICE_NAME \ # --service $ECS_SERVICE_NAME \ # --task-definition $ECS_SERVICE_NAME \ - # --force-new-deployment \ No newline at end of file + # --force-new-deployment From e3a3f8bc84059dd7bcfad678f427a7bf2439498c Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Tue, 6 Jun 2023 17:00:10 -0500 Subject: [PATCH 13/38] Update env var to use $ --- .github/actions/terraform-deploy/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/terraform-deploy/action.yml b/.github/actions/terraform-deploy/action.yml index 9d07eab..5fb47c5 100644 --- a/.github/actions/terraform-deploy/action.yml +++ b/.github/actions/terraform-deploy/action.yml @@ -53,7 +53,7 @@ runs: shell: bash env: IMAGE_TAG: latest - ECR_REGISTRY: ${{ format('{0}.dkr.ecr.{1}.amazonaws.com/{2}-registry-{3}', env.ACCOUNT_ID, env.AWS_REGION, env.APP_NAME, env.STAGE) }} + ECR_REGISTRY: ${{ format('{0}.dkr.ecr.{1}.amazonaws.com/{2}-registry-{3}', $ACCOUNT_ID, $AWS_REGION, $APP_NAME, $STAGE) }} run: | echo $ECR_REGISTRY echo $IMAGE_TAG @@ -65,7 +65,7 @@ runs: - name: ECS refresh service shell: bash env: - ECS_SERVICE_NAME: ${{ format('{0}-service-{1}', env.APP_NAME, env.STAGE) }} + ECS_SERVICE_NAME: ${{ format('{0}-service-{1}', $APP_NAME, $STAGE) }} run: | echo $ECS_SERVICE_NAME # aws ecs update-service \ From e1adb8e8a4b4b0253cb3bfd944cee471ad2f6b72 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Tue, 6 Jun 2023 17:02:02 -0500 Subject: [PATCH 14/38] Revert --- .github/actions/terraform-deploy/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/terraform-deploy/action.yml b/.github/actions/terraform-deploy/action.yml index 5fb47c5..9d07eab 100644 --- a/.github/actions/terraform-deploy/action.yml +++ b/.github/actions/terraform-deploy/action.yml @@ -53,7 +53,7 @@ runs: shell: bash env: IMAGE_TAG: latest - ECR_REGISTRY: ${{ format('{0}.dkr.ecr.{1}.amazonaws.com/{2}-registry-{3}', $ACCOUNT_ID, $AWS_REGION, $APP_NAME, $STAGE) }} + ECR_REGISTRY: ${{ format('{0}.dkr.ecr.{1}.amazonaws.com/{2}-registry-{3}', env.ACCOUNT_ID, env.AWS_REGION, env.APP_NAME, env.STAGE) }} run: | echo $ECR_REGISTRY echo $IMAGE_TAG @@ -65,7 +65,7 @@ runs: - name: ECS refresh service shell: bash env: - ECS_SERVICE_NAME: ${{ format('{0}-service-{1}', $APP_NAME, $STAGE) }} + ECS_SERVICE_NAME: ${{ format('{0}-service-{1}', env.APP_NAME, env.STAGE) }} run: | echo $ECS_SERVICE_NAME # aws ecs update-service \ From f5aa0366b4e58e3357403bdc8a524ace4881fba1 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 9 Jun 2023 09:48:38 -0600 Subject: [PATCH 15/38] tf env variable, remove dns --- db/Dockerfile | 2 +- docs/IACHOWTO.md | 2 + scripts/build.sh | 22 +++-- scripts/deploy.sh | 86 +++++++++++++++++++ terraform/features-api/dns.tf | 58 ++++++------- terraform/features-api/ecs_api.tf | 4 +- terraform/features-api/load_balancer.tf | 49 ++++++----- terraform/features-api/outputs.tf | 4 + terraform/features-api/secret_manager.tf | 2 +- .../features-api/{init.tf => terraform.tf} | 2 +- terraform/features-api/terraform.tf.tmpl | 24 ++++++ terraform/features-api/terraform.tfvars | 9 ++ terraform/features-api/terraform.tfvars.tmpl | 9 ++ terraform/features-api/variables.tf | 5 +- terraform/features-api/vars/dev.tf | 13 +-- terraform/features-api/vars/staging.tf | 28 +++--- 16 files changed, 229 insertions(+), 90 deletions(-) create mode 100644 scripts/deploy.sh rename terraform/features-api/{init.tf => terraform.tf} (88%) create mode 100644 terraform/features-api/terraform.tf.tmpl create mode 100644 terraform/features-api/terraform.tfvars create mode 100644 terraform/features-api/terraform.tfvars.tmpl diff --git a/db/Dockerfile b/db/Dockerfile index 6b428e0..dbde212 100644 --- a/db/Dockerfile +++ b/db/Dockerfile @@ -1,4 +1,4 @@ -FROM public.ecr.aws/lambda/python:3.9 +FROM --platform=linux/amd64 public.ecr.aws/lambda/python:3.9 # WORKDIR /tmp diff --git a/docs/IACHOWTO.md b/docs/IACHOWTO.md index 82c6162..287a448 100644 --- a/docs/IACHOWTO.md +++ b/docs/IACHOWTO.md @@ -22,6 +22,8 @@ $ tfenv use 1.3.9 5. we also use Terraform "workspaces" so our infra state stays nicely separated in the same S3 bucket. Some quick samples of how to interact with that: ```bash +$ AWS_PROFILE= terraform workspace new west2-staging + $ AWS_PROFILE= terraform workspace list * default west2-staging diff --git a/scripts/build.sh b/scripts/build.sh index 32b6f87..91f2016 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,15 +1,19 @@ #!/bin/sh +export TARGET_ENVIRONMENT=dev +export TARGET_PROJECT_NAME=ghgc-features-api -# aws ecr describe-repositories \ -# | jq '.repositories | map(.repositoryUri)' \ -# | grep $TARGET_PROJECT_NAME | grep $TARGET_ENVIRONMENT \ -# | xargs -I {} bash -c "aws ecr get-login-password | docker login --username AWS --password-stdin {}" +cd wfs3-app/ -# aws ecr describe-repositories \ -# | jq '.repositories | map(.repositoryUri)' \ -# | grep $TARGET_PROJECT_NAME | grep $TARGET_ENVIRONMENT \ -# | sed -E 's/"|,//g' \ -# | xargs -I {} docker build -t {}:latest ../wfs3-app/ +aws ecr describe-repositories \ + | jq '.repositories | map(.repositoryUri)' \ + | grep $TARGET_PROJECT_NAME | grep $TARGET_ENVIRONMENT \ + | xargs -I {} bash -c "aws ecr get-login-password | docker login --username AWS --password-stdin {}" + +aws ecr describe-repositories \ + | jq '.repositories | map(.repositoryUri)' \ + | grep $TARGET_PROJECT_NAME | grep $TARGET_ENVIRONMENT \ + | sed -E 's/"|,//g' \ + | xargs -I {} docker build -t {}:latest ../wfs3-app/ aws ecr describe-repositories \ | jq '.repositories | map(.repositoryUri)' \ diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100644 index 0000000..4b495c1 --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,86 @@ +#! /bin/bash +# Check .env file + + +DOT_ENV=$1 + +if [ -f $DOT_ENV ] +then + set -a; source $DOT_ENV; set +a +else + echo "Run: ./scripts/deploy.sh <.env_file>" + echo "Please create $DOT_ENV file first and try again" + exit 1 +fi + +function create_state_bucket { + # $1 region + # $2 bucket_name + + aws s3 mb s3://$2 --region $1 + aws s3api put-bucket-versioning \ + --bucket $2 \ + --versioning-configuration Status=Enabled +} + +function generate_terraform_variables { + tf_vars=(tf tfvars) + for tf_var in "${tf_vars[@]}"; do + ( + echo "cat < terraform.${tf_var} + done + +} + +function check_create_remote_state { + # $1 aws_region + # $2 bucket name + # $3 dynamotable_name + AWS_REGION=$1 + STATE_BUCKET_NAME=$2 + + bucketstatus=$(aws s3api head-bucket --bucket $STATE_BUCKET_NAME 2>&1) + + if echo "${bucketstatus}" | grep 'Not Found'; + then + echo "Creating TF remote state" + create_state_bucket $AWS_REGION $STATE_BUCKET_NAME + elif echo "${bucketstatus}" | grep 'Forbidden'; + then + echo "Bucket $STATE_BUCKET_NAME exists but not owned" + exit 1 + elif echo "${bucketstatus}" | grep 'Bad Request'; + then + echo "Bucket $STATE_BUCKET_NAME specified is less than 3 or greater than 63 characters" + exit 1 + else + echo "State Bucket $STATE_BUCKET_NAME owned and exists. Continue..."; + fi +} + + +cd ./terraform/features-api +generate_terraform_variables +check_create_remote_state $AWS_REGION $STATE_BUCKET_NAME + +read -rp 'action [init|plan|deploy]: ' ACTION +case $ACTION in + init) + terraform init + ;; + plan) + terraform plan + ;; + + deploy) + terraform apply --auto-approve + ;; + *) + echo "Choose from 'init', 'plan' or 'deploy'" + exit 1 + ;; +esac + diff --git a/terraform/features-api/dns.tf b/terraform/features-api/dns.tf index 65ab077..ebde0e6 100644 --- a/terraform/features-api/dns.tf +++ b/terraform/features-api/dns.tf @@ -1,34 +1,34 @@ -data "aws_route53_zone" "zone" { - provider = aws.west2 - name = var.dns_zone_name -} +# data "aws_route53_zone" "zone" { +# provider = aws.west2 +# name = var.dns_zone_name +# } -resource "aws_acm_certificate" "cert" { - provider = aws.west2 - domain_name = "*.${data.aws_route53_zone.zone.name}" - validation_method = "DNS" - tags = var.tags +# resource "aws_acm_certificate" "cert" { +# provider = aws.west2 +# domain_name = "*.${data.aws_route53_zone.zone.name}" +# validation_method = "DNS" +# tags = var.tags - lifecycle { - create_before_destroy = true - } -} +# lifecycle { +# create_before_destroy = true +# } +# } -resource "aws_route53_record" "subdomain_record" { - provider = aws.west2 - name = "${var.dns_subdomain}.${data.aws_route53_zone.zone.name}" - zone_id = data.aws_route53_zone.zone.id - type = "A" +# resource "aws_route53_record" "subdomain_record" { +# provider = aws.west2 +# name = "${var.dns_subdomain}.${data.aws_route53_zone.zone.name}" +# zone_id = data.aws_route53_zone.zone.id +# type = "A" - alias { - name = aws_alb.alb_ecs.dns_name - zone_id = aws_alb.alb_ecs.zone_id - evaluate_target_health = true - } -} +# alias { +# name = aws_alb.alb_ecs.dns_name +# zone_id = aws_alb.alb_ecs.zone_id +# evaluate_target_health = true +# } +# } -resource "aws_lb_listener_certificate" "cert" { - provider = aws.west2 - listener_arn = aws_alb_listener.alb_listener_ecs.arn - certificate_arn = aws_acm_certificate.cert.arn -} \ No newline at end of file +# resource "aws_lb_listener_certificate" "cert" { +# provider = aws.west2 +# listener_arn = aws_alb_listener.alb_listener_ecs.arn +# certificate_arn = aws_acm_certificate.cert.arn +# } \ No newline at end of file diff --git a/terraform/features-api/ecs_api.tf b/terraform/features-api/ecs_api.tf index d9d8a84..c66be7b 100644 --- a/terraform/features-api/ecs_api.tf +++ b/terraform/features-api/ecs_api.tf @@ -78,7 +78,7 @@ module "ecs_cluster" { { // stupid hack b/c of FastAPI and Starlette bug name = "FAST_API_SCHEME" - value = var.env == "dev" ? "https" : "http" //quick hack for now, TODO: include 'contains' function + value = "http" //quick hack for now, TODO: include 'contains' function } ] @@ -94,7 +94,7 @@ module "ecs_cluster" { load_balancer = true lb_type = "application" lb_target_group_arn = aws_alb_target_group.alb_target_group.arn - lb_security_group_id = aws_security_group.web_inbound_sg.id + lb_security_group_id = aws_security_group.https_web_inbound_sg.id lb_container_port = var.service_port tags = var.tags diff --git a/terraform/features-api/load_balancer.tf b/terraform/features-api/load_balancer.tf index 60b6033..38cdd22 100644 --- a/terraform/features-api/load_balancer.tf +++ b/terraform/features-api/load_balancer.tf @@ -82,7 +82,7 @@ resource "aws_security_group" "https_web_inbound_sg" { resource "aws_alb" "alb_ecs" { name = "tf-${var.project_name}-${var.env}-alb" subnets = data.aws_subnets.public.ids - security_groups = [aws_security_group.https_web_inbound_sg.id] + security_groups = [aws_security_group.web_inbound_sg.id] tags = merge({ Name = "tf-${var.project_name}-alb" @@ -117,28 +117,27 @@ resource "aws_alb_target_group" "alb_target_group" { ] } -#resource "aws_alb_listener" "alb_listener_ecs" { -# load_balancer_arn = aws_alb.alb_ecs.arn -# port = 80 -# protocol = var.alb_protocol -# depends_on = [aws_alb_target_group.alb_target_group] -# -# default_action { -# target_group_arn = aws_alb_target_group.alb_target_group.arn -# type = "forward" -# } -#} - resource "aws_alb_listener" "alb_listener_ecs" { - load_balancer_arn = aws_alb.alb_ecs.arn - port = 443 - protocol = var.alb_protocol - ssl_policy = "ELBSecurityPolicy-2016-08" - certificate_arn = aws_acm_certificate.cert.arn - depends_on = [aws_alb_target_group.alb_target_group] - - default_action { - target_group_arn = aws_alb_target_group.alb_target_group.arn - type = "forward" - } -} \ No newline at end of file + load_balancer_arn = aws_alb.alb_ecs.arn + port = 80 + depends_on = [aws_alb_target_group.alb_target_group] + protocol = "HTTP" + default_action { + target_group_arn = aws_alb_target_group.alb_target_group.arn + type = "forward" + } +} + +# resource "aws_alb_listener" "alb_listener_ecs" { +# load_balancer_arn = aws_alb.alb_ecs.arn +# port = 443 +# protocol = var.alb_protocol +# ssl_policy = "ELBSecurityPolicy-2016-08" +# certificate_arn = aws_acm_certificate.cert.arn +# depends_on = [aws_alb_target_group.alb_target_group] + +# default_action { +# target_group_arn = aws_alb_target_group.alb_target_group.arn +# type = "forward" +# } +# } \ No newline at end of file diff --git a/terraform/features-api/outputs.tf b/terraform/features-api/outputs.tf index 24013cc..9550efb 100644 --- a/terraform/features-api/outputs.tf +++ b/terraform/features-api/outputs.tf @@ -17,3 +17,7 @@ output "protocol_on_aws_alb_listener" { description = "HTTP/HTTPS protocol on the ALB Listener" value = aws_alb_listener.alb_listener_ecs.protocol } + +output "alb_url" { + value = "https://${aws_alb.alb_ecs.dns_name}" +} diff --git a/terraform/features-api/secret_manager.tf b/terraform/features-api/secret_manager.tf index 15615f6..d7f8587 100644 --- a/terraform/features-api/secret_manager.tf +++ b/terraform/features-api/secret_manager.tf @@ -25,7 +25,7 @@ resource "aws_secretsmanager_secret" "config" { } resource "aws_secretsmanager_secret" "db_config" { - name = "${var.project_name}-wfs3-${var.env}-db" + name = "${var.project_name}-wfs3-${var.env}-db-secrets" kms_key_id = data.aws_kms_key.secretsmanager.id tags = var.tags } diff --git a/terraform/features-api/init.tf b/terraform/features-api/terraform.tf similarity index 88% rename from terraform/features-api/init.tf rename to terraform/features-api/terraform.tf index 44360ff..11e5ca1 100644 --- a/terraform/features-api/init.tf +++ b/terraform/features-api/terraform.tf @@ -17,7 +17,7 @@ terraform { } } backend "s3" { - bucket = "ghg-wfs3-tf-state-bucket" + bucket = "ghgc-features-tf-state-bucket" key = "root" region = "us-west-2" } diff --git a/terraform/features-api/terraform.tf.tmpl b/terraform/features-api/terraform.tf.tmpl new file mode 100644 index 0000000..886bb62 --- /dev/null +++ b/terraform/features-api/terraform.tf.tmpl @@ -0,0 +1,24 @@ +provider "aws" { + alias = "west1" + region = "us-west-1" +} + +provider "aws" { + alias = "west2" + region = "us-west-2" +} + +terraform { + required_version = "1.3.9" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } + backend "s3" { + bucket = "${STATE_BUCKET_NAME}" + key = "root" + region = "${AWS_REGION}" + } +} diff --git a/terraform/features-api/terraform.tfvars b/terraform/features-api/terraform.tfvars new file mode 100644 index 0000000..d757fd5 --- /dev/null +++ b/terraform/features-api/terraform.tfvars @@ -0,0 +1,9 @@ +region = "us-west-2" +registry_name = "ghgc-features-api-registry" +env = "dev" +project_name = "ghgc-features-api" +availability_zones = ["us-west-2a", "us-west-2b"] +service_port = "8080" +dns_zone_name = "dev.ghg.center" +tags = {"project": "ghgc", "service": "ghgc-features-api-dev"} +vpc_id = "vpc-0a20167ff1004d0f2" diff --git a/terraform/features-api/terraform.tfvars.tmpl b/terraform/features-api/terraform.tfvars.tmpl new file mode 100644 index 0000000..6c144b2 --- /dev/null +++ b/terraform/features-api/terraform.tfvars.tmpl @@ -0,0 +1,9 @@ +region = "${AWS_REGION}" +registry_name = "${REGISTRY_NAME}" +env = "${ENV}" +project_name = "${PROJECT_NAME}" +availability_zones = ${AZ} +service_port = "${SERVICE_PORT}" +dns_zone_name = "${DNS_ZONE_NAME}" +tags = ${TAGS} +vpc_id = "${VPC_ID}" diff --git a/terraform/features-api/variables.tf b/terraform/features-api/variables.tf index 7ce71a4..6cfb948 100755 --- a/terraform/features-api/variables.tf +++ b/terraform/features-api/variables.tf @@ -38,14 +38,15 @@ variable "default_secret" { # } variable "dns_zone_name" { + default = null } variable "dns_subdomain" { - + default = null } variable "alb_protocol" { - default = "HTTPS" + default = "HTTP" } variable "vpc_id" {} diff --git a/terraform/features-api/vars/dev.tf b/terraform/features-api/vars/dev.tf index 20a304a..1920d47 100644 --- a/terraform/features-api/vars/dev.tf +++ b/terraform/features-api/vars/dev.tf @@ -1,11 +1,12 @@ region = "us-west-2" -registry_name = "features-api-registry" +registry_name = "ghgc-features-api-registry" env = "dev" -project_name = "veda-features-api" +project_name = "ghgc-features-api" db_public_subnet = false availability_zones = ["us-west-2a", "us-west-2b"] service_port = 8080 -dns_zone_name = "delta-backend.com" -dns_subdomain = "ghg-dev" -tags = {"project": "veda", "service": "veda-features-api-dev"} -vpc_id = "vpc-0512162c42da5e645" +dns_zone_name = "dev.ghg.center" +dns_subdomain = "features-api" +tags = {"project": "ghgc", "service": "ghgc-features-api-dev"} +vpc_id = "vpc-0a20167ff1004d0f2" +alb_protocol = "HTTP" diff --git a/terraform/features-api/vars/staging.tf b/terraform/features-api/vars/staging.tf index a0b6dac..1e79402 100644 --- a/terraform/features-api/vars/staging.tf +++ b/terraform/features-api/vars/staging.tf @@ -1,14 +1,14 @@ -region = "us-west-2" -registry_name = "features-api-registry" -env = "dev" -project_name = "veda-features-api" -availability_zones = ["us-west-2a", "us-west-2b"] -service_port = 8080 -dns_zone_name = "delta-backend.com" -dns_subdomain = "ghg-dev" -alb_protocol = "HTTPS" -tags = {"project": "veda", "service": "veda-features-api-dev"} -default_secret = { - "noop": "boop", -} -vpc_id = "vpc-0512162c42da5e645" \ No newline at end of file +# region = "us-west-2" +# registry_name = "features-api-registry" +# env = "dev" +# project_name = "veda-features-api" +# availability_zones = ["us-west-2a", "us-west-2b"] +# service_port = 8080 +# dns_zone_name = "delta-backend.com" +# dns_subdomain = "ghg-dev" +# alb_protocol = "HTTPS" +# tags = {"project": "veda", "service": "veda-features-api-dev"} +# default_secret = { +# "noop": "boop", +# } +# vpc_id = "vpc-0512162c42da5e645" \ No newline at end of file From 580ab960d3b599f8fe2dc4635141397fa52511bb Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 9 Jun 2023 11:02:22 -0600 Subject: [PATCH 16/38] change tf vars --- terraform/features-api/terraform.tfvars.tmpl | 3 --- terraform/features-api/variables.tf | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/terraform/features-api/terraform.tfvars.tmpl b/terraform/features-api/terraform.tfvars.tmpl index 6c144b2..b25d340 100644 --- a/terraform/features-api/terraform.tfvars.tmpl +++ b/terraform/features-api/terraform.tfvars.tmpl @@ -2,8 +2,5 @@ region = "${AWS_REGION}" registry_name = "${REGISTRY_NAME}" env = "${ENV}" project_name = "${PROJECT_NAME}" -availability_zones = ${AZ} service_port = "${SERVICE_PORT}" -dns_zone_name = "${DNS_ZONE_NAME}" -tags = ${TAGS} vpc_id = "${VPC_ID}" diff --git a/terraform/features-api/variables.tf b/terraform/features-api/variables.tf index 6cfb948..e70311a 100755 --- a/terraform/features-api/variables.tf +++ b/terraform/features-api/variables.tf @@ -19,6 +19,7 @@ variable "tags" { variable "availability_zones" { type = list(any) description = "The az that the resources will be launched" + default = ["us-west-2a", "us-west-2b"] } variable "service_port" {} From 12abbdf041af35a2b6b488972f39dcf8fab54edc Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 9 Jun 2023 11:17:04 -0600 Subject: [PATCH 17/38] update env sync and github action --- .github/actions/terraform-deploy/action.yml | 14 +++++++------- scripts/sync-env.sh | 4 +--- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/actions/terraform-deploy/action.yml b/.github/actions/terraform-deploy/action.yml index 9d07eab..988ad75 100644 --- a/.github/actions/terraform-deploy/action.yml +++ b/.github/actions/terraform-deploy/action.yml @@ -41,13 +41,13 @@ runs: with: terraform_version: 1.3.3 - # - name: Deploy - # if: env.infra_deploy - # shell: bash - # working-directory: ${{ inputs.dir }} - # run: | - # ./scripts/deploy.sh ${{ inputs.env-file }} <<< init - # ./scripts/deploy.sh ${{ inputs.env-file }} <<< deploy + - name: Deploy + if: env.infra_deploy + shell: bash + working-directory: ${{ inputs.dir }} + run: | + ./scripts/deploy.sh .env <<< init + # ./scripts/deploy.sh .env <<< deploy - name: Docker build, tag, and push image to Amazon ECR shell: bash diff --git a/scripts/sync-env.sh b/scripts/sync-env.sh index 3758445..5f9161f 100755 --- a/scripts/sync-env.sh +++ b/scripts/sync-env.sh @@ -1,6 +1,4 @@ #!/usr/bin/env bash # Use this script to load environment variables for a deployment from AWS Secrets -for s in $(aws secretsmanager get-secret-value --secret-id $1 --query SecretString --output text | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do - echo "$s" >> $GITHUB_ENV -done +aws secretsmanager get-secret-value --secret-id $1 --query SecretString --output text | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" > .env From f36372d51acb85479aa28286aa275f38a5ebeb48 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 9 Jun 2023 11:24:40 -0600 Subject: [PATCH 18/38] update actions vars --- .github/actions/terraform-deploy/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/terraform-deploy/action.yml b/.github/actions/terraform-deploy/action.yml index 988ad75..3736e3a 100644 --- a/.github/actions/terraform-deploy/action.yml +++ b/.github/actions/terraform-deploy/action.yml @@ -17,14 +17,14 @@ runs: steps: - name: Set up Python - if: env.infra_deploy + if: vars.infra_deploy uses: actions/setup-python@v4 with: python-version: "3.10" cache: "pip" - name: Install python dependencies - if: env.infra_deploy + if: vars.infra_deploy shell: bash working-directory: ${{ inputs.dir }} run: pip install -r deploy_requirements.txt @@ -36,13 +36,13 @@ runs: ./scripts/sync-env.sh ${{ inputs.env_aws_secret_name }} - name: Setup Terraform - if: env.infra_deploy + if: vars.infra_deploy uses: hashicorp/setup-terraform@v2 with: terraform_version: 1.3.3 - name: Deploy - if: env.infra_deploy + if: vars.infra_deploy shell: bash working-directory: ${{ inputs.dir }} run: | From 6cbe1a83867881203f127983d6eb3ed32338f2f9 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 9 Jun 2023 11:26:40 -0600 Subject: [PATCH 19/38] remove python dependencies from actions --- .github/actions/terraform-deploy/action.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/terraform-deploy/action.yml b/.github/actions/terraform-deploy/action.yml index 3736e3a..7960f77 100644 --- a/.github/actions/terraform-deploy/action.yml +++ b/.github/actions/terraform-deploy/action.yml @@ -23,11 +23,11 @@ runs: python-version: "3.10" cache: "pip" - - name: Install python dependencies - if: vars.infra_deploy - shell: bash - working-directory: ${{ inputs.dir }} - run: pip install -r deploy_requirements.txt + # - name: Install python dependencies + # if: vars.infra_deploy + # shell: bash + # working-directory: ${{ inputs.dir }} + # run: pip install -r deploy_requirements.txt - name: Get relevant environment configuration from aws secrets shell: bash From 466c8ca5fe0e1eae0e899944a375f2d7a2c740e4 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 9 Jun 2023 11:30:18 -0600 Subject: [PATCH 20/38] change chmod for script --- scripts/sync-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-env.sh b/scripts/sync-env.sh index 5f9161f..c13e14c 100755 --- a/scripts/sync-env.sh +++ b/scripts/sync-env.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash # Use this script to load environment variables for a deployment from AWS Secrets - +echo Loading environment secrets from $1 aws secretsmanager get-secret-value --secret-id $1 --query SecretString --output text | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" > .env From 53f9b1e2cc46ca22ce60f5f2ff399e9b6a24c296 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 9 Jun 2023 11:32:33 -0600 Subject: [PATCH 21/38] add bash --- .github/actions/terraform-deploy/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/terraform-deploy/action.yml b/.github/actions/terraform-deploy/action.yml index 7960f77..22d620f 100644 --- a/.github/actions/terraform-deploy/action.yml +++ b/.github/actions/terraform-deploy/action.yml @@ -46,7 +46,7 @@ runs: shell: bash working-directory: ${{ inputs.dir }} run: | - ./scripts/deploy.sh .env <<< init + bash ./scripts/deploy.sh .env <<< init # ./scripts/deploy.sh .env <<< deploy - name: Docker build, tag, and push image to Amazon ECR From a4edbcdd5189be74acb235c9f33c983fd5c6009d Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 9 Jun 2023 11:34:23 -0600 Subject: [PATCH 22/38] update actions tf version --- .github/actions/terraform-deploy/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/terraform-deploy/action.yml b/.github/actions/terraform-deploy/action.yml index 22d620f..b4fe9a4 100644 --- a/.github/actions/terraform-deploy/action.yml +++ b/.github/actions/terraform-deploy/action.yml @@ -39,7 +39,7 @@ runs: if: vars.infra_deploy uses: hashicorp/setup-terraform@v2 with: - terraform_version: 1.3.3 + terraform_version: 1.3.9 - name: Deploy if: vars.infra_deploy From 69c0117b96acbc3cd91ac1f5ca478bcc36d38881 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 9 Jun 2023 11:36:45 -0600 Subject: [PATCH 23/38] remove python from actions --- .github/actions/terraform-deploy/action.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/actions/terraform-deploy/action.yml b/.github/actions/terraform-deploy/action.yml index b4fe9a4..e59b45a 100644 --- a/.github/actions/terraform-deploy/action.yml +++ b/.github/actions/terraform-deploy/action.yml @@ -16,12 +16,12 @@ runs: using: "composite" steps: - - name: Set up Python - if: vars.infra_deploy - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" + # - name: Set up Python + # if: vars.infra_deploy + # uses: actions/setup-python@v4 + # with: + # python-version: "3.10" + # cache: "pip" # - name: Install python dependencies # if: vars.infra_deploy @@ -47,7 +47,7 @@ runs: working-directory: ${{ inputs.dir }} run: | bash ./scripts/deploy.sh .env <<< init - # ./scripts/deploy.sh .env <<< deploy + bash ./scripts/deploy.sh .env <<< deploy - name: Docker build, tag, and push image to Amazon ECR shell: bash From 8e6c1e4ff83a900ff734d1c063dea8cbb9e142ea Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 9 Jun 2023 14:36:23 -0600 Subject: [PATCH 24/38] tf changes --- terraform/features-api/github_deploy_user.tf | 58 ++++++++++---------- terraform/features-api/lambda.tf | 1 + 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/terraform/features-api/github_deploy_user.tf b/terraform/features-api/github_deploy_user.tf index f594e07..4926230 100644 --- a/terraform/features-api/github_deploy_user.tf +++ b/terraform/features-api/github_deploy_user.tf @@ -1,32 +1,32 @@ -resource "aws_iam_user" "deploy_user" { - name = "${var.project_name}-${var.env}-deploy-user" - path = "/" - tags = var.tags -} +# resource "aws_iam_user" "deploy_user" { +# name = "${var.project_name}-${var.env}-deploy-user" +# path = "/" +# tags = var.tags +# } -// NOTE: we need to have extra policies added to our -// deploy user for Github AWS Actions to work -resource "aws_iam_user_policy" "deploy" { - name = "${var.registry_name}_deploy_extended" - user = aws_iam_user.deploy_user.name - policy = data.aws_iam_policy_document.extended_deploy.json -} +# // NOTE: we need to have extra policies added to our +# // deploy user for Github AWS Actions to work +# resource "aws_iam_user_policy" "deploy" { +# name = "${var.registry_name}_deploy_extended" +# user = aws_iam_user.deploy_user.name +# policy = data.aws_iam_policy_document.extended_deploy.json +# } -data "aws_iam_policy_document" "extended_deploy" { - statement { - actions = [ - "iam:PassRole", - "ecr:InitiateLayerUpload", - "ecs:RegisterTaskDefinition", - "ecs:DescribeServices", - "ecs:UpdateService", - ] +# data "aws_iam_policy_document" "extended_deploy" { +# statement { +# actions = [ +# "iam:PassRole", +# "ecr:InitiateLayerUpload", +# "ecs:RegisterTaskDefinition", +# "ecs:DescribeServices", +# "ecs:UpdateService", +# ] - resources = [ - module.ecr_registry_wfs.registry_arn, - module.ecs_cluster.service_cluster_arn, - module.ecs_cluster.service_arn, - module.ecs_cluster.ecs_execution_role_arn, - ] - } -} \ No newline at end of file +# resources = [ +# module.ecr_registry_wfs.registry_arn, +# module.ecs_cluster.service_cluster_arn, +# module.ecs_cluster.service_arn, +# module.ecs_cluster.ecs_execution_role_arn, +# ] +# } +# } \ No newline at end of file diff --git a/terraform/features-api/lambda.tf b/terraform/features-api/lambda.tf index 8fae03b..4c46b00 100644 --- a/terraform/features-api/lambda.tf +++ b/terraform/features-api/lambda.tf @@ -126,6 +126,7 @@ data "aws_iam_policy_document" "lambda_policy" { resource "aws_iam_role" "iam_for_lambda" { name = "${var.project_name}-${var.env}-lambda-initdb-role" assume_role_policy = data.aws_iam_policy_document.lambda_assume_role_policy.json + permissions_boundary = "arn:aws:iam::444055461661:policy/ghgc-features-api-permissions-boundary" } resource "aws_iam_role_policy" "lambda_execution_role_policy" { From 712750a9631b85584f7b9f7fb1f696c29501b5b8 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 9 Jun 2023 14:45:26 -0600 Subject: [PATCH 25/38] ecr user changes --- terraform/features-api/ecr.tf | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/terraform/features-api/ecr.tf b/terraform/features-api/ecr.tf index 2424dd3..e49dd0d 100644 --- a/terraform/features-api/ecr.tf +++ b/terraform/features-api/ecr.tf @@ -4,8 +4,7 @@ module "ecr_registry_wfs" { registry_name = var.project_name enable_registry_scanning = true mutable_image_tags = true - enable_deploy_user = true - iam_deploy_username = aws_iam_user.deploy_user.name + enable_deploy_user = false tags = var.tags } @@ -15,8 +14,7 @@ module "ecr_registry_db" { registry_name = "${var.project_name}-db" enable_registry_scanning = true mutable_image_tags = true - enable_deploy_user = true - iam_deploy_username = aws_iam_user.deploy_user.name + enable_deploy_user = false tags = var.tags } From 8a14201722e5978d9b53913bb6ae3e6bf60a043b Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 9 Jun 2023 14:51:08 -0600 Subject: [PATCH 26/38] ecr user change --- terraform/features-api/ecr.tf | 2 ++ terraform/features-api/terraform.tfvars | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/terraform/features-api/ecr.tf b/terraform/features-api/ecr.tf index e49dd0d..9c5a55f 100644 --- a/terraform/features-api/ecr.tf +++ b/terraform/features-api/ecr.tf @@ -5,6 +5,7 @@ module "ecr_registry_wfs" { enable_registry_scanning = true mutable_image_tags = true enable_deploy_user = false + iam_deploy_username = null tags = var.tags } @@ -15,6 +16,7 @@ module "ecr_registry_db" { enable_registry_scanning = true mutable_image_tags = true enable_deploy_user = false + iam_deploy_username = null tags = var.tags } diff --git a/terraform/features-api/terraform.tfvars b/terraform/features-api/terraform.tfvars index d757fd5..7eb1a38 100644 --- a/terraform/features-api/terraform.tfvars +++ b/terraform/features-api/terraform.tfvars @@ -2,8 +2,5 @@ region = "us-west-2" registry_name = "ghgc-features-api-registry" env = "dev" project_name = "ghgc-features-api" -availability_zones = ["us-west-2a", "us-west-2b"] service_port = "8080" -dns_zone_name = "dev.ghg.center" -tags = {"project": "ghgc", "service": "ghgc-features-api-dev"} vpc_id = "vpc-0a20167ff1004d0f2" From f717acf1d018bca6ad3f0df917afeac0ef776db0 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 9 Jun 2023 15:09:01 -0600 Subject: [PATCH 27/38] permission boundary --- terraform/features-api/ecs_api.tf | 1 + terraform/features-api/lambda.tf | 2 +- terraform/modules/aws_ecs_service/main.tf | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/terraform/features-api/ecs_api.tf b/terraform/features-api/ecs_api.tf index c66be7b..5633ac6 100644 --- a/terraform/features-api/ecs_api.tf +++ b/terraform/features-api/ecs_api.tf @@ -130,4 +130,5 @@ resource "aws_iam_role_policy" "api_ecs_execution_role_policy" { name = "${var.project_name}-api-access-secret-manager" role = module.ecs_cluster.ecs_execution_role_id policy = data.aws_iam_policy_document.api_ecs_execution_attachment.json + permissions_boundary = "arn:aws:iam::444055461661:policy/mcp-tenantOperator" } diff --git a/terraform/features-api/lambda.tf b/terraform/features-api/lambda.tf index 4c46b00..5dfd1e8 100644 --- a/terraform/features-api/lambda.tf +++ b/terraform/features-api/lambda.tf @@ -126,7 +126,7 @@ data "aws_iam_policy_document" "lambda_policy" { resource "aws_iam_role" "iam_for_lambda" { name = "${var.project_name}-${var.env}-lambda-initdb-role" assume_role_policy = data.aws_iam_policy_document.lambda_assume_role_policy.json - permissions_boundary = "arn:aws:iam::444055461661:policy/ghgc-features-api-permissions-boundary" + permissions_boundary = "arn:aws:iam::444055461661:policy/mcp-tenantOperator" } resource "aws_iam_role_policy" "lambda_execution_role_policy" { diff --git a/terraform/modules/aws_ecs_service/main.tf b/terraform/modules/aws_ecs_service/main.tf index c914b9f..dec97cd 100755 --- a/terraform/modules/aws_ecs_service/main.tf +++ b/terraform/modules/aws_ecs_service/main.tf @@ -25,6 +25,7 @@ resource "aws_iam_role" "ecs_execution_role" { name = "${var.service_name}-${var.environment}_ecs_task_execution_role" assume_role_policy = data.aws_iam_policy_document.ecs_assume_role_policy.json tags = var.tags + permissions_boundary = "arn:aws:iam::444055461661:policy/mcp-tenantOperator" } data "aws_iam_policy_document" "ecs_execution_attachment" { From 0e5a217e4505cfc5c53f928b5742553e5bfb5549 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 9 Jun 2023 15:41:52 -0600 Subject: [PATCH 28/38] permission boundary fix --- terraform/features-api/ecs_api.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/terraform/features-api/ecs_api.tf b/terraform/features-api/ecs_api.tf index 5633ac6..c66be7b 100644 --- a/terraform/features-api/ecs_api.tf +++ b/terraform/features-api/ecs_api.tf @@ -130,5 +130,4 @@ resource "aws_iam_role_policy" "api_ecs_execution_role_policy" { name = "${var.project_name}-api-access-secret-manager" role = module.ecs_cluster.ecs_execution_role_id policy = data.aws_iam_policy_document.api_ecs_execution_attachment.json - permissions_boundary = "arn:aws:iam::444055461661:policy/mcp-tenantOperator" } From 2334c730944fa0c8b068458b035648186f5827c7 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Mon, 12 Jun 2023 12:11:11 -0600 Subject: [PATCH 29/38] various tf changes --- terraform/features-api/ecs_api.tf | 4 ++-- terraform/features-api/lambda.tf | 5 +++-- terraform/features-api/outputs.tf | 2 +- terraform/features-api/secret_manager.tf | 2 ++ terraform/features-api/terraform.tfvars | 2 +- terraform/features-api/terraform.tfvars.tmpl | 2 +- terraform/features-api/variables.tf | 11 ++++------- terraform/features-api/vars/dev.tf | 12 ------------ terraform/features-api/vars/staging.tf | 14 -------------- wfs3-app/startup.sh | 9 ++++++--- 10 files changed, 20 insertions(+), 43 deletions(-) delete mode 100644 terraform/features-api/vars/dev.tf delete mode 100644 terraform/features-api/vars/staging.tf diff --git a/terraform/features-api/ecs_api.tf b/terraform/features-api/ecs_api.tf index c66be7b..a49b87a 100644 --- a/terraform/features-api/ecs_api.tf +++ b/terraform/features-api/ecs_api.tf @@ -5,7 +5,7 @@ data "aws_subnets" "private" { } tags = { - "aws-cdk:subnet-name" = var.db_public_subnet ? "public" : "private" + "aws-cdk:subnet-name" = "private" } } @@ -78,7 +78,7 @@ module "ecs_cluster" { { // stupid hack b/c of FastAPI and Starlette bug name = "FAST_API_SCHEME" - value = "http" //quick hack for now, TODO: include 'contains' function + value = "https" //quick hack for now, TODO: include 'contains' function } ] diff --git a/terraform/features-api/lambda.tf b/terraform/features-api/lambda.tf index 5dfd1e8..5018516 100644 --- a/terraform/features-api/lambda.tf +++ b/terraform/features-api/lambda.tf @@ -27,7 +27,6 @@ resource "aws_lambda_function" "lambda_init_db" { } } - resource "aws_lambda_invocation" "db_init" { function_name = aws_lambda_function.lambda_init_db.function_name @@ -137,7 +136,9 @@ resource "aws_iam_role_policy" "lambda_execution_role_policy" { resource "aws_cloudwatch_log_group" "lambda_cloudwatch_group" { name = "/aws/lambda/${var.project_name}-${var.env}-initdb-function" - retention_in_days = 14 + retention_in_days = 1 + tags = var.tags + skip_destroy = false } diff --git a/terraform/features-api/outputs.tf b/terraform/features-api/outputs.tf index 9550efb..a10a013 100644 --- a/terraform/features-api/outputs.tf +++ b/terraform/features-api/outputs.tf @@ -19,5 +19,5 @@ output "protocol_on_aws_alb_listener" { } output "alb_url" { - value = "https://${aws_alb.alb_ecs.dns_name}" + value = "http://${aws_alb.alb_ecs.dns_name}" } diff --git a/terraform/features-api/secret_manager.tf b/terraform/features-api/secret_manager.tf index d7f8587..34205c7 100644 --- a/terraform/features-api/secret_manager.tf +++ b/terraform/features-api/secret_manager.tf @@ -22,12 +22,14 @@ resource "aws_secretsmanager_secret" "config" { name = "aws-config-${random_id.sm_suffix.hex}" kms_key_id = data.aws_kms_key.secretsmanager.id tags = var.tags + recovery_window_in_days = 0 } resource "aws_secretsmanager_secret" "db_config" { name = "${var.project_name}-wfs3-${var.env}-db-secrets" kms_key_id = data.aws_kms_key.secretsmanager.id tags = var.tags + recovery_window_in_days = 0 } resource "aws_secretsmanager_secret_version" "db_credentials" { diff --git a/terraform/features-api/terraform.tfvars b/terraform/features-api/terraform.tfvars index 7eb1a38..2bb519a 100644 --- a/terraform/features-api/terraform.tfvars +++ b/terraform/features-api/terraform.tfvars @@ -2,5 +2,5 @@ region = "us-west-2" registry_name = "ghgc-features-api-registry" env = "dev" project_name = "ghgc-features-api" -service_port = "8080" +service_port = 8080 vpc_id = "vpc-0a20167ff1004d0f2" diff --git a/terraform/features-api/terraform.tfvars.tmpl b/terraform/features-api/terraform.tfvars.tmpl index b25d340..f1debe0 100644 --- a/terraform/features-api/terraform.tfvars.tmpl +++ b/terraform/features-api/terraform.tfvars.tmpl @@ -2,5 +2,5 @@ region = "${AWS_REGION}" registry_name = "${REGISTRY_NAME}" env = "${ENV}" project_name = "${PROJECT_NAME}" -service_port = "${SERVICE_PORT}" +service_port = ${SERVICE_PORT} vpc_id = "${VPC_ID}" diff --git a/terraform/features-api/variables.tf b/terraform/features-api/variables.tf index e70311a..10fbcac 100755 --- a/terraform/features-api/variables.tf +++ b/terraform/features-api/variables.tf @@ -12,7 +12,7 @@ variable "project_name" { variable "tags" { type = map(any) - default = {} + default = {"project": "ghgc", "service": "ghgc-features-api-dev"} description = "Optional tags to add to resources" } @@ -22,7 +22,9 @@ variable "availability_zones" { default = ["us-west-2a", "us-west-2b"] } -variable "service_port" {} +variable "service_port" { + default = 8080 +} # Key/Value default to prevent task definitions from stopping at runtime variable "default_secret" { @@ -51,8 +53,3 @@ variable "alb_protocol" { } variable "vpc_id" {} - -variable "db_public_subnet" { - type = bool - default = true -} diff --git a/terraform/features-api/vars/dev.tf b/terraform/features-api/vars/dev.tf deleted file mode 100644 index 1920d47..0000000 --- a/terraform/features-api/vars/dev.tf +++ /dev/null @@ -1,12 +0,0 @@ -region = "us-west-2" -registry_name = "ghgc-features-api-registry" -env = "dev" -project_name = "ghgc-features-api" -db_public_subnet = false -availability_zones = ["us-west-2a", "us-west-2b"] -service_port = 8080 -dns_zone_name = "dev.ghg.center" -dns_subdomain = "features-api" -tags = {"project": "ghgc", "service": "ghgc-features-api-dev"} -vpc_id = "vpc-0a20167ff1004d0f2" -alb_protocol = "HTTP" diff --git a/terraform/features-api/vars/staging.tf b/terraform/features-api/vars/staging.tf deleted file mode 100644 index 1e79402..0000000 --- a/terraform/features-api/vars/staging.tf +++ /dev/null @@ -1,14 +0,0 @@ -# region = "us-west-2" -# registry_name = "features-api-registry" -# env = "dev" -# project_name = "veda-features-api" -# availability_zones = ["us-west-2a", "us-west-2b"] -# service_port = 8080 -# dns_zone_name = "delta-backend.com" -# dns_subdomain = "ghg-dev" -# alb_protocol = "HTTPS" -# tags = {"project": "veda", "service": "veda-features-api-dev"} -# default_secret = { -# "noop": "boop", -# } -# vpc_id = "vpc-0512162c42da5e645" \ No newline at end of file diff --git a/wfs3-app/startup.sh b/wfs3-app/startup.sh index 867512f..4ad0687 100644 --- a/wfs3-app/startup.sh +++ b/wfs3-app/startup.sh @@ -1,6 +1,9 @@ ################################################################ # FAST API ################################################################# -opentelemetry-bootstrap --action=install \ - && opentelemetry-instrument python /opt/bitnami/python/bin/uvicorn \ - fast_api_main:app --proxy-headers --forwarded-allow-ips="*" --host 0.0.0.0 --port 8080 +# opentelemetry-bootstrap --action=install \ +# && opentelemetry-instrument python /opt/bitnami/python/bin/uvicorn \ +# fast_api_main:app --proxy-headers --forwarded-allow-ips="*" --host 0.0.0.0 --port 8080 + +python /opt/bitnami/python/bin/uvicorn \ +fast_api_main:app --proxy-headers --forwarded-allow-ips="*" --host 0.0.0.0 --port 8080 From 82fdade7e13bc27c3e9ebb398f38de53b6c4ec17 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Mon, 12 Jun 2023 12:32:43 -0600 Subject: [PATCH 30/38] load balance port 80 --- terraform/features-api/ecs_api.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/features-api/ecs_api.tf b/terraform/features-api/ecs_api.tf index a49b87a..eeab58b 100644 --- a/terraform/features-api/ecs_api.tf +++ b/terraform/features-api/ecs_api.tf @@ -94,7 +94,7 @@ module "ecs_cluster" { load_balancer = true lb_type = "application" lb_target_group_arn = aws_alb_target_group.alb_target_group.arn - lb_security_group_id = aws_security_group.https_web_inbound_sg.id + lb_security_group_id = aws_security_group.web_inbound_sg.id lb_container_port = var.service_port tags = var.tags From 48e7401020990a61ea7efbff6fd73ea157840550 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Mon, 12 Jun 2023 12:35:00 -0600 Subject: [PATCH 31/38] fast api http --- terraform/features-api/ecs_api.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/features-api/ecs_api.tf b/terraform/features-api/ecs_api.tf index eeab58b..2e7c312 100644 --- a/terraform/features-api/ecs_api.tf +++ b/terraform/features-api/ecs_api.tf @@ -78,7 +78,7 @@ module "ecs_cluster" { { // stupid hack b/c of FastAPI and Starlette bug name = "FAST_API_SCHEME" - value = "https" //quick hack for now, TODO: include 'contains' function + value = "http" //quick hack for now, TODO: include 'contains' function } ] From ceeabe8e4b82ebcd57328a65a33fc2dc84c0f67d Mon Sep 17 00:00:00 2001 From: smohiudd Date: Mon, 12 Jun 2023 13:10:03 -0600 Subject: [PATCH 32/38] lb ingress 443 rule --- terraform/features-api/load_balancer.tf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/terraform/features-api/load_balancer.tf b/terraform/features-api/load_balancer.tf index 38cdd22..aa01d27 100644 --- a/terraform/features-api/load_balancer.tf +++ b/terraform/features-api/load_balancer.tf @@ -22,6 +22,13 @@ resource "aws_security_group" "web_inbound_sg" { description = "Allow HTTP from Anywhere into ALB" vpc_id = var.vpc_id + ingress { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + ingress { from_port = 80 to_port = 80 From b1bffc6519c2e671153a07811a786dba4dbe8692 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Mon, 12 Jun 2023 13:15:48 -0600 Subject: [PATCH 33/38] add example env file --- .example.env | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .example.env diff --git a/.example.env b/.example.env new file mode 100644 index 0000000..2cb6bad --- /dev/null +++ b/.example.env @@ -0,0 +1,7 @@ +ENV=dev +AWS_REGION=us-west-2 +STATE_BUCKET_NAME=ghgc-features-tf-state-bucket +PROJECT_NAME=ghgc-features-api +REGISTRY_NAME=ghgc-features-api-registry +SERVICE_PORT=8080 +VPC_ID= From 21ab473a4bd612109df95473ffe7a97a3514e497 Mon Sep 17 00:00:00 2001 From: amarouane-ABDELHAK Date: Wed, 26 Jul 2023 13:49:26 -0500 Subject: [PATCH 34/38] Deplot feature api --- .github/actions/terraform-deploy/action.yml | 22 +++++++++------------ 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/actions/terraform-deploy/action.yml b/.github/actions/terraform-deploy/action.yml index e59b45a..1d3b4b6 100644 --- a/.github/actions/terraform-deploy/action.yml +++ b/.github/actions/terraform-deploy/action.yml @@ -16,18 +16,16 @@ runs: using: "composite" steps: - # - name: Set up Python - # if: vars.infra_deploy - # uses: actions/setup-python@v4 - # with: - # python-version: "3.10" - # cache: "pip" + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" - # - name: Install python dependencies - # if: vars.infra_deploy - # shell: bash - # working-directory: ${{ inputs.dir }} - # run: pip install -r deploy_requirements.txt + - name: Install python dependencies + shell: bash + working-directory: ${{ inputs.dir }} + run: pip install -r deploy_requirements.txt - name: Get relevant environment configuration from aws secrets shell: bash @@ -36,13 +34,11 @@ runs: ./scripts/sync-env.sh ${{ inputs.env_aws_secret_name }} - name: Setup Terraform - if: vars.infra_deploy uses: hashicorp/setup-terraform@v2 with: terraform_version: 1.3.9 - name: Deploy - if: vars.infra_deploy shell: bash working-directory: ${{ inputs.dir }} run: | From 467cadd96bc6728eba79aed0eedec4a496d49b8a Mon Sep 17 00:00:00 2001 From: amarouane-ABDELHAK Date: Tue, 31 Oct 2023 16:25:06 -0500 Subject: [PATCH 35/38] Deploy features api from mono-repo --- .github/actions/terraform-deploy/action.yml | 62 ++++++++++++-------- db/Dockerfile | 2 +- terraform/features-api/ecr.tf | 4 +- terraform/features-api/ecs_api.tf | 2 +- terraform/features-api/terraform.tf | 4 +- terraform/features-api/terraform.tf.tmpl | 2 +- terraform/features-api/terraform.tfvars | 8 +-- terraform/features-api/terraform.tfvars.tmpl | 2 +- 8 files changed, 48 insertions(+), 38 deletions(-) diff --git a/.github/actions/terraform-deploy/action.yml b/.github/actions/terraform-deploy/action.yml index 1d3b4b6..51a43ef 100644 --- a/.github/actions/terraform-deploy/action.yml +++ b/.github/actions/terraform-deploy/action.yml @@ -1,16 +1,17 @@ name: Deploy - inputs: env_aws_secret_name: required: true type: string env-file: - required: true type: string + default: ".env" dir: required: false type: string default: "." + script_path: + type: string runs: using: "composite" @@ -31,7 +32,16 @@ runs: shell: bash working-directory: ${{ inputs.dir }} run: | + if [[ -z "${{ inputs.script_path }}" ]]; then ./scripts/sync-env.sh ${{ inputs.env_aws_secret_name }} + else + python ${{ inputs.script_path }} --secret-id ${{ inputs.env_aws_secret_name }} + source ${{ inputs.env-file }} + echo "PREFIX=feature-api-${STAGE}" >> ${{ inputs.env-file }} + echo "REGISTRY_NAME=feature-api-${STAGE}" >> ${{ inputs.env-file }} + echo "ENV=${STAGE}" >> ${{ inputs.env-file }} + echo "PROJECT_NAME=veda-${STAGE}" >> ${{ inputs.env-file }} + fi - name: Setup Terraform uses: hashicorp/setup-terraform@v2 @@ -45,27 +55,27 @@ runs: bash ./scripts/deploy.sh .env <<< init bash ./scripts/deploy.sh .env <<< deploy - - name: Docker build, tag, and push image to Amazon ECR - shell: bash - env: - IMAGE_TAG: latest - ECR_REGISTRY: ${{ format('{0}.dkr.ecr.{1}.amazonaws.com/{2}-registry-{3}', env.ACCOUNT_ID, env.AWS_REGION, env.APP_NAME, env.STAGE) }} - run: | - echo $ECR_REGISTRY - echo $IMAGE_TAG - # aws ecr get-login-password | docker login --username AWS --password-stdin $ECR_REGISTRY - # cd veda-wfs3-app - # docker build -t $ECR_REGISTRY }}:$IMAGE_TAG }} . - # docker push $ECR_REGISTRY }}:$IMAGE_TAG }} - - - name: ECS refresh service - shell: bash - env: - ECS_SERVICE_NAME: ${{ format('{0}-service-{1}', env.APP_NAME, env.STAGE) }} - run: | - echo $ECS_SERVICE_NAME - # aws ecs update-service \ - # --cluster $ECS_SERVICE_NAME \ - # --service $ECS_SERVICE_NAME \ - # --task-definition $ECS_SERVICE_NAME \ - # --force-new-deployment +# - name: Docker build, tag, and push image to Amazon ECR +# shell: bash +# env: +# IMAGE_TAG: latest +# ECR_REGISTRY: ${{ format('{0}.dkr.ecr.{1}.amazonaws.com/{2}-registry-{3}', env.ACCOUNT_ID, env.AWS_REGION, env.APP_NAME, env.STAGE) }} +# run: | +# echo $ECR_REGISTRY +# echo $IMAGE_TAG +# # aws ecr get-login-password | docker login --username AWS --password-stdin $ECR_REGISTRY +# # cd veda-wfs3-app +# # docker build -t $ECR_REGISTRY }}:$IMAGE_TAG }} . +# # docker push $ECR_REGISTRY }}:$IMAGE_TAG }} +# +# - name: ECS refresh service +# shell: bash +# env: +# ECS_SERVICE_NAME: ${{ format('{0}-service-{1}', env.APP_NAME, env.STAGE) }} +# run: | +# echo $ECS_SERVICE_NAME +# # aws ecs update-service \ +# # --cluster $ECS_SERVICE_NAME \ +# # --service $ECS_SERVICE_NAME \ +# # --task-definition $ECS_SERVICE_NAME \ +# # --force-new-deployment diff --git a/db/Dockerfile b/db/Dockerfile index dbde212..8cc4921 100644 --- a/db/Dockerfile +++ b/db/Dockerfile @@ -10,4 +10,4 @@ COPY ./handler.py ${LAMBDA_TASK_ROOT} # Turns out, asyncio is part of python # RUN rm -rf /asset/asyncio* -CMD ["handler.handler"] \ No newline at end of file +CMD ["handler.handler"] diff --git a/terraform/features-api/ecr.tf b/terraform/features-api/ecr.tf index 9c5a55f..64a239d 100644 --- a/terraform/features-api/ecr.tf +++ b/terraform/features-api/ecr.tf @@ -22,7 +22,7 @@ module "ecr_registry_db" { resource "null_resource" "build_ecr_image_wfs" { triggers = { - folder_path = sha1(join("", [for f in fileset("../../wfs3-app", "*") : filesha1("../../wfs3-app/${f}")])) + folder_path = sha1(join("", [for f in fileset("../../wfs3-app", "**") : filesha1("../../wfs3-app/${f}")])) # handler_file_path = filemd5("../../wfs3-app/fast_api_main.py") # docker_file_path = filemd5("../../wfs3-app/Dockerfile") } @@ -40,7 +40,7 @@ resource "null_resource" "build_ecr_image_wfs" { resource "null_resource" "build_ecr_image_db_init" { triggers = { - folder_path = sha1(join("", [for f in fileset("../../db", "*") : filesha1("../../db/${f}")])) + folder_path = sha1(join("", [for f in fileset("../../db", "**") : filesha1("../../db/${f}")])) # handler_file_path = filemd5("../../db/handler.py") # docker_file_path = filemd5("../../db/Dockerfile") } diff --git a/terraform/features-api/ecs_api.tf b/terraform/features-api/ecs_api.tf index 2e7c312..6a2c530 100644 --- a/terraform/features-api/ecs_api.tf +++ b/terraform/features-api/ecs_api.tf @@ -5,7 +5,7 @@ data "aws_subnets" "private" { } tags = { - "aws-cdk:subnet-name" = "private" + "Scope" = "private" } } diff --git a/terraform/features-api/terraform.tf b/terraform/features-api/terraform.tf index 11e5ca1..b57e150 100644 --- a/terraform/features-api/terraform.tf +++ b/terraform/features-api/terraform.tf @@ -17,8 +17,8 @@ terraform { } } backend "s3" { - bucket = "ghgc-features-tf-state-bucket" - key = "root" + bucket = "ghgc-smce-tf-shared-state" + key = "root/features-api" region = "us-west-2" } } diff --git a/terraform/features-api/terraform.tf.tmpl b/terraform/features-api/terraform.tf.tmpl index 886bb62..646acac 100644 --- a/terraform/features-api/terraform.tf.tmpl +++ b/terraform/features-api/terraform.tf.tmpl @@ -18,7 +18,7 @@ terraform { } backend "s3" { bucket = "${STATE_BUCKET_NAME}" - key = "root" + key = "root/features-api" region = "${AWS_REGION}" } } diff --git a/terraform/features-api/terraform.tfvars b/terraform/features-api/terraform.tfvars index 2bb519a..18f1643 100644 --- a/terraform/features-api/terraform.tfvars +++ b/terraform/features-api/terraform.tfvars @@ -1,6 +1,6 @@ region = "us-west-2" -registry_name = "ghgc-features-api-registry" -env = "dev" -project_name = "ghgc-features-api" +registry_name = "feature-api-pre-dev" +env = "pre-dev" +project_name = "veda-pre-dev" service_port = 8080 -vpc_id = "vpc-0a20167ff1004d0f2" +vpc_id = "vpc-0c6727f22063d860f" diff --git a/terraform/features-api/terraform.tfvars.tmpl b/terraform/features-api/terraform.tfvars.tmpl index f1debe0..042f271 100644 --- a/terraform/features-api/terraform.tfvars.tmpl +++ b/terraform/features-api/terraform.tfvars.tmpl @@ -2,5 +2,5 @@ region = "${AWS_REGION}" registry_name = "${REGISTRY_NAME}" env = "${ENV}" project_name = "${PROJECT_NAME}" -service_port = ${SERVICE_PORT} +service_port = ${SERVICE_PORT:-8080} vpc_id = "${VPC_ID}" From 5704752cc0122b382a38d98e78077c4c48ced766 Mon Sep 17 00:00:00 2001 From: amarouane-ABDELHAK Date: Tue, 31 Oct 2023 16:31:27 -0500 Subject: [PATCH 36/38] Deploy features api from mono-repo --- .github/actions/terraform-deploy/action.yml | 28 +++------------------ 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/.github/actions/terraform-deploy/action.yml b/.github/actions/terraform-deploy/action.yml index 51a43ef..0d436db 100644 --- a/.github/actions/terraform-deploy/action.yml +++ b/.github/actions/terraform-deploy/action.yml @@ -26,7 +26,9 @@ runs: - name: Install python dependencies shell: bash working-directory: ${{ inputs.dir }} - run: pip install -r deploy_requirements.txt + run: | + python -m pip install --upgrade pip + python -m pip install boto3 - name: Get relevant environment configuration from aws secrets shell: bash @@ -55,27 +57,3 @@ runs: bash ./scripts/deploy.sh .env <<< init bash ./scripts/deploy.sh .env <<< deploy -# - name: Docker build, tag, and push image to Amazon ECR -# shell: bash -# env: -# IMAGE_TAG: latest -# ECR_REGISTRY: ${{ format('{0}.dkr.ecr.{1}.amazonaws.com/{2}-registry-{3}', env.ACCOUNT_ID, env.AWS_REGION, env.APP_NAME, env.STAGE) }} -# run: | -# echo $ECR_REGISTRY -# echo $IMAGE_TAG -# # aws ecr get-login-password | docker login --username AWS --password-stdin $ECR_REGISTRY -# # cd veda-wfs3-app -# # docker build -t $ECR_REGISTRY }}:$IMAGE_TAG }} . -# # docker push $ECR_REGISTRY }}:$IMAGE_TAG }} -# -# - name: ECS refresh service -# shell: bash -# env: -# ECS_SERVICE_NAME: ${{ format('{0}-service-{1}', env.APP_NAME, env.STAGE) }} -# run: | -# echo $ECS_SERVICE_NAME -# # aws ecs update-service \ -# # --cluster $ECS_SERVICE_NAME \ -# # --service $ECS_SERVICE_NAME \ -# # --task-definition $ECS_SERVICE_NAME \ -# # --force-new-deployment From 0f5f355b6da76be7f26463ac69c9babae1c5f0a8 Mon Sep 17 00:00:00 2001 From: amarouane-ABDELHAK Date: Tue, 31 Oct 2023 17:04:40 -0500 Subject: [PATCH 37/38] Add permission boundaries --- terraform/features-api/ecs_api.tf | 1 + terraform/features-api/lambda.tf | 6 +++++- terraform/features-api/load_balancer.tf | 2 +- terraform/features-api/terraform.tfvars.tmpl | 1 + terraform/features-api/variables.tf | 3 +++ terraform/modules/aws_ecs_service/main.tf | 7 +++++-- terraform/modules/aws_ecs_service/variables.tf | 1 + 7 files changed, 17 insertions(+), 4 deletions(-) diff --git a/terraform/features-api/ecs_api.tf b/terraform/features-api/ecs_api.tf index 6a2c530..66c9e3b 100644 --- a/terraform/features-api/ecs_api.tf +++ b/terraform/features-api/ecs_api.tf @@ -98,6 +98,7 @@ module "ecs_cluster" { lb_container_port = var.service_port tags = var.tags + permissions_boundary_policy_name = var.permissions_boundary_policy_name } ############################################################## diff --git a/terraform/features-api/lambda.tf b/terraform/features-api/lambda.tf index 5018516..b9789a0 100644 --- a/terraform/features-api/lambda.tf +++ b/terraform/features-api/lambda.tf @@ -1,3 +1,7 @@ +data "aws_caller_identity" "current" {} +locals { + permissions_boundary = var.permissions_boundary_policy_name == "" ? null : "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/${var.permissions_boundary_policy_name}" +} resource "aws_lambda_function" "lambda_init_db" { code_signing_config_arn = "" description = "Lambda function to init medium DB" @@ -125,7 +129,7 @@ data "aws_iam_policy_document" "lambda_policy" { resource "aws_iam_role" "iam_for_lambda" { name = "${var.project_name}-${var.env}-lambda-initdb-role" assume_role_policy = data.aws_iam_policy_document.lambda_assume_role_policy.json - permissions_boundary = "arn:aws:iam::444055461661:policy/mcp-tenantOperator" + permissions_boundary = local.permissions_boundary } resource "aws_iam_role_policy" "lambda_execution_role_policy" { diff --git a/terraform/features-api/load_balancer.tf b/terraform/features-api/load_balancer.tf index aa01d27..0d81ec9 100644 --- a/terraform/features-api/load_balancer.tf +++ b/terraform/features-api/load_balancer.tf @@ -5,7 +5,7 @@ data "aws_subnets" "public" { } tags = { - "aws-cdk:subnet-name" = "public" + "Scope" = "public" } } diff --git a/terraform/features-api/terraform.tfvars.tmpl b/terraform/features-api/terraform.tfvars.tmpl index 042f271..bfb24a8 100644 --- a/terraform/features-api/terraform.tfvars.tmpl +++ b/terraform/features-api/terraform.tfvars.tmpl @@ -4,3 +4,4 @@ env = "${ENV}" project_name = "${PROJECT_NAME}" service_port = ${SERVICE_PORT:-8080} vpc_id = "${VPC_ID}" +permissions_boundary_policy_name = ${PERMISSIONS_BOUNDARY_POLICY_NAME} diff --git a/terraform/features-api/variables.tf b/terraform/features-api/variables.tf index 10fbcac..7354246 100755 --- a/terraform/features-api/variables.tf +++ b/terraform/features-api/variables.tf @@ -53,3 +53,6 @@ variable "alb_protocol" { } variable "vpc_id" {} +variable "permissions_boundary_policy_name" { + default = "" +} \ No newline at end of file diff --git a/terraform/modules/aws_ecs_service/main.tf b/terraform/modules/aws_ecs_service/main.tf index dec97cd..aa5684d 100755 --- a/terraform/modules/aws_ecs_service/main.tf +++ b/terraform/modules/aws_ecs_service/main.tf @@ -6,7 +6,10 @@ # name = var.ecr_repository_name # } - +data "aws_caller_identity" "current" {} +locals { + permissions_boundary = var.permissions_boundary_policy_name == "" ? null : "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/${var.permissions_boundary_policy_name}" +} ######################################################################## # IAM ######################################################################## @@ -25,7 +28,7 @@ resource "aws_iam_role" "ecs_execution_role" { name = "${var.service_name}-${var.environment}_ecs_task_execution_role" assume_role_policy = data.aws_iam_policy_document.ecs_assume_role_policy.json tags = var.tags - permissions_boundary = "arn:aws:iam::444055461661:policy/mcp-tenantOperator" + permissions_boundary = local.permissions_boundary } data "aws_iam_policy_document" "ecs_execution_attachment" { diff --git a/terraform/modules/aws_ecs_service/variables.tf b/terraform/modules/aws_ecs_service/variables.tf index 27c837c..ecf52ea 100755 --- a/terraform/modules/aws_ecs_service/variables.tf +++ b/terraform/modules/aws_ecs_service/variables.tf @@ -135,3 +135,4 @@ variable "lb_type" { variable "lb_target_group_arn" {} variable "lb_security_group_id" {} variable "lb_container_port" {} +variable "permissions_boundary_policy_name" {} \ No newline at end of file From 4f943b19463f4713139f6bfefded2c37cf0173e8 Mon Sep 17 00:00:00 2001 From: amarouane-ABDELHAK Date: Tue, 31 Oct 2023 17:07:23 -0500 Subject: [PATCH 38/38] Add permission boundaries --- terraform/features-api/terraform.tfvars.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/features-api/terraform.tfvars.tmpl b/terraform/features-api/terraform.tfvars.tmpl index bfb24a8..fd1d991 100644 --- a/terraform/features-api/terraform.tfvars.tmpl +++ b/terraform/features-api/terraform.tfvars.tmpl @@ -4,4 +4,4 @@ env = "${ENV}" project_name = "${PROJECT_NAME}" service_port = ${SERVICE_PORT:-8080} vpc_id = "${VPC_ID}" -permissions_boundary_policy_name = ${PERMISSIONS_BOUNDARY_POLICY_NAME} +permissions_boundary_policy_name = "${PERMISSIONS_BOUNDARY_POLICY_NAME}"