Bugfixes to fix deploy following various PRs

NOAA-OWP · Nov 26, 2024 · a4d2917 · a4d2917
1 parent ea8b7ae
commit a4d2917
Show file tree

Hide file tree

Showing 14 changed files with 85 additions and 69 deletions.
diff --git a/Core/EC2/DataServices/main.tf b/Core/EC2/DataServices/main.tf
@@ -180,7 +180,6 @@ data "cloudinit_config" "startup" {
       forecast_api_2_0_commit  = var.data_services_versions["forecast_api_2_0_commit"]
       ssh_key_filename         = local.ssh_key_filename
       logging_application_name = "data_services"
-      instance                 = count.index
     })
   }
 }
@@ -210,7 +209,7 @@ resource "aws_instance" "data_services" {
   }
 
   # This runs the cloud-init config, copying the SSH key to the EC2 and running the startup.sh script.
-  user_data                   = data.cloudinit_config.startup[count.index].rendered
+  user_data                   = data.cloudinit_config.startup.rendered
   user_data_replace_on_change = true
 }
 
@@ -219,7 +218,7 @@ resource "aws_route53_record" "hydrovis" {
   name    = "data-services.${var.private_route_53_zone.name}"
   type    = "A"
   ttl     = 300
-  records = [aws_instance.data_services[0].private_ip]
+  records = [aws_instance.data_services.private_ip]
 }
 
 data "aws_ami" "linux" {

diff --git a/Core/LAMBDA/viz_functions/main.tf b/Core/LAMBDA/viz_functions/main.tf
@@ -1008,6 +1008,10 @@ output "egis_health_checker" {
   value = aws_lambda_function.egis_health_checker
 }
 
+output "test_wrds_db" {
+  value = aws_lambda_function.viz_test_wrds_db
+}
+
 output "hand_fim_processing" {
   value = module.image-based-lambdas.hand_fim_processing
 }

diff --git a/...unctions/execute_replace_route.json.tftpl → ...ions/rnr/execute_replace_route.json.tftpl b/...unctions/execute_replace_route.json.tftpl → ...ions/rnr/execute_replace_route.json.tftpl
@@ -92,7 +92,7 @@
       "Resource": "arn:aws:states:::lambda:invoke",
       "OutputPath": "$.Payload",
       "Parameters": {
-        "FunctionName": "${initialize_pipeline_arn}",
+        "FunctionName": "initialize_pipeline_arn",
         "Payload": {
           "data_bucket.$": "$.data_bucket",
           "data_key.$": "$.data_key",

diff --git a/Core/StepFunctions/rnr/main.tf b/Core/StepFunctions/rnr/main.tf
@@ -0,0 +1,52 @@
+variable "rnr_lambda_role" {
+  type        = string
+}
+
+variable "environment" {
+  type        = string
+}
+
+variable "initialize_pipeline_arn" {
+  type        = string
+}
+
+variable "rnr_domain_generator_arn" {
+  type        = string
+}
+
+variable "rnr_ec2_instance" {
+  type        = string
+}
+
+variable "fifteen_minute_trigger" {
+  type = object({
+    name = string
+  })
+}
+
+#########################################
+##     Replace Route Step Function     ##
+#########################################
+
+resource "aws_sfn_state_machine" "replace_route_step_function" {
+    name     = "hv-vpp-${var.environment}-execute-replace-route"
+    role_arn = var.rnr_lambda_role
+
+    definition = templatefile("${path.module}/execute_replace_route.json.tftpl", {
+        initialize_pipeline_arn = var.initialize_pipeline_arn
+        rnr_domain_generator_arn = var.rnr_domain_generator_arn
+        rnr_ec2_instance = var.rnr_ec2_instance
+    })
+
+    tags = {
+      "noaa:monitoring" : "true"
+    }
+}
+
+resource "aws_cloudwatch_event_target" "check_lambda_every_five_minutes" {
+  count     = var.environment == "ti" ? 0 : 1
+  rule      = var.fifteen_minute_trigger.name
+  target_id = aws_sfn_state_machine.replace_route_step_function.name
+  arn       = aws_sfn_state_machine.replace_route_step_function.arn
+  role_arn  = aws_sfn_state_machine.replace_route_step_function.role_arn
+}
diff --git a/...StepFunctions/ensure_ec2_ready.json.tftpl → ...Functions/viz/ensure_ec2_ready.json.tftpl b/...StepFunctions/ensure_ec2_ready.json.tftpl → ...Functions/viz/ensure_ec2_ready.json.tftpl
diff --git a/...pFunctions/hand_fim_processing.json.tftpl → ...ctions/viz/hand_fim_processing.json.tftpl b/...pFunctions/hand_fim_processing.json.tftpl → ...ctions/viz/hand_fim_processing.json.tftpl
diff --git a/Core/StepFunctions/main.tf → Core/StepFunctions/viz/main.tf b/Core/StepFunctions/main.tf → Core/StepFunctions/viz/main.tf
@@ -2,14 +2,18 @@ variable "viz_lambda_role" {
   type        = string
 }
 
-variable "rnr_lambda_role" {
+variable "sync_wrds_db_role" {
   type        = string
 }
 
 variable "environment" {
   type        = string
 }
 
+variable "region" {
+  type        = string
+}
+
 variable "python_preprocessing_3GB_arn" {
   type        = string
 }
@@ -62,14 +66,6 @@ variable "publish_service_arn" {
   type        = string
 }
 
-variable "initialize_pipeline_arn" {
-  type        = string
-}
-
-variable "rnr_domain_generator_arn" {
-  type        = string
-}
-
 variable "email_sns_topics" {
   description = "SnS topics"
   type        = map(any)
@@ -79,12 +75,6 @@ variable "aws_instances_to_reboot" {
   type        = list(string)
 }
 
-variable "fifteen_minute_trigger" {
-  type = object({
-    name = string
-  })
-}
-
 variable "viz_processing_pipeline_log_group" {
   type        = string
 }
@@ -98,58 +88,19 @@ variable "test_wrds_db_lambda_arn" {
   type        = string
 }
 
-#########################################
-##     Replace Route Step Function     ##
-#########################################
-
-resource "aws_sfn_state_machine" "replace_route_step_function" {
-    name     = "hv-vpp-${var.environment}-execute-replace-route"
-    role_arn = var.rnr_lambda_role
-
-    definition = templatefile("${path.module}/execute_replace_route.json.tftpl", {
-        initialize_pipeline_arn = var.initialize_pipeline_arn
-        rnr_domain_generator_arn = var.rnr_domain_generator_arn
-        rnr_ec2_instance = var.aws_instances_to_reboot[0]
-    })
-
-    tags = {
-      "noaa:monitoring" : "true"
-    }
-}
-
-resource "aws_cloudwatch_event_target" "check_lambda_every_five_minutes" {
-  count     = var.environment == "ti" ? 0 : 1
-  rule      = var.fifteen_minute_trigger.name
-  target_id = aws_sfn_state_machine.replace_route_step_function.name
-  arn       = aws_sfn_state_machine.replace_route_step_function.arn
-  role_arn  = aws_sfn_state_machine.replace_route_step_function.role_arn
-}
-
 ################################################
 ##     Reboot EC2 Instances Step Function     ##
 ################################################
 
 resource "aws_sfn_state_machine" "reboot_ec2_instances_step_function" {
     name     = "hv-vpp-${var.environment}-reboot-ec2-instances"
-    role_arn = var.rnr_lambda_role
+    role_arn = var.sync_wrds_db_role
 
     definition = templatefile("${path.module}/reboot_ec2_instances.json.tftpl", {
         aws_instances_to_reboot = var.aws_instances_to_reboot
     })
 }
 
-resource "aws_cloudwatch_event_rule" "daily_at_2330" {
-  name                = "daily_at_2330"
-  description         = "Fires every day at 23:30"
-  schedule_expression = "cron(30 23 * * ? *)"
-}
-
-resource "aws_cloudwatch_event_target" "trigger_reboot_rnr_ec2" {
-  rule      = aws_cloudwatch_event_rule.daily_at_2330.name
-  arn       = aws_sfn_state_machine.reboot_ec2_instances_step_function.arn
-  role_arn  = var.rnr_lambda_role
-}
-
 ##################################################
 ##     Viz Process Schism FIM Step Function     ##
 ##################################################
@@ -268,7 +219,7 @@ resource "aws_sfn_state_machine" "sync_wrds_location_db_step_function" {
   role_arn = var.viz_lambda_role
 
   definition = templatefile("${path.module}/sync_wrds_location_db.json.tftpl", {
-    restore_db_from_s3_dump_step_function_arn  = aws_sfn_state_machine.restore_db_from_s3_dump_step_function.arn
+    restore_db_dump_from_s3_step_function_arn = aws_sfn_state_machine.restore_db_from_s3_dump_step_function.arn
     test_wrds_db_lambda_arn = var.test_wrds_db_lambda_arn
     rds_bastion_id = var.rds_bastion_id
     region = var.region

diff --git a/...Functions/reboot_ec2_instances.json.tftpl → ...tions/viz/reboot_ec2_instances.json.tftpl b/...Functions/reboot_ec2_instances.json.tftpl → ...tions/viz/reboot_ec2_instances.json.tftpl
diff --git a/...epFunctions/restore_db_from_s3_dump.tftpl → ...ns/viz/restore_db_from_s3_dump.json.tftpl b/...epFunctions/restore_db_from_s3_dump.tftpl → ...ns/viz/restore_db_from_s3_dump.json.tftpl
diff --git a/...unctions/schism_fim_processing.json.tftpl → ...ions/viz/schism_fim_processing.json.tftpl b/...unctions/schism_fim_processing.json.tftpl → ...ions/viz/schism_fim_processing.json.tftpl
diff --git a/...StepFunctions/sync_wrds_location_db.tftpl → ...ions/viz/sync_wrds_location_db.json.tftpl b/...StepFunctions/sync_wrds_location_db.tftpl → ...ions/viz/sync_wrds_location_db.json.tftpl
diff --git a/...ctions/viz_processing_pipeline.json.tftpl → ...ns/viz/viz_processing_pipeline.json.tftpl b/...ctions/viz_processing_pipeline.json.tftpl → ...ns/viz/viz_processing_pipeline.json.tftpl
diff --git a/Core/Testing/main.tf b/Core/Testing/main.tf
@@ -62,6 +62,7 @@ resource "aws_cloudwatch_event_target" "trigger_pipeline_test_run" {
       "Records": [
         {
           "Sns": {
+            "TopicArn": "N/A",
             "Message": "{\"Records\": [{\"s3\": {\"bucket\": {\"name\": \"<s3_bucket>\"}, \"object\": {\"key\": \"<s3_key>\"}}}]}"
           }
         }

diff --git a/Core/main.tf b/Core/main.tf
@@ -619,19 +619,20 @@ module "viz-lambda-functions" {
   egis_db_name                   = local.env.egis_db_name
   egis_db_user_secret_string     = module.secrets-manager.secret_strings["egis-pg-rds-secret"]
   egis_portal_password           = local.env.viz_ec2_hydrovis_egis_pass
-  viz_pipeline_step_function_arn = module.step-functions.viz_pipeline_step_function.arn
-  sync_wrds_db_step_function_arn = module.step-functions.sync_wrds_location_db_step_function.arn
+  viz_pipeline_step_function_arn = module.viz-step-functions.viz_pipeline_step_function.arn
+  sync_wrds_db_step_function_arn = module.viz-step-functions.sync_wrds_location_db_step_function.arn
   default_tags                   = local.env.tags
   nwm_dataflow_version           = local.env.nwm_dataflow_version
   five_minute_trigger            = module.eventbridge.five_minute_eventbridge
 }
 
-module "step-functions" {
-  source = "./StepFunctions"
+module "viz-step-functions" {
+  source = "./StepFunctions/viz"
 
   viz_lambda_role                   = module.iam-roles.role_viz_pipeline.arn
-  rnr_lambda_role                   = module.iam-roles.role_sync_wrds_location_db.arn
+  sync_wrds_db_role                 = module.iam-roles.role_sync_wrds_location_db.arn
   environment                       = local.env.environment
+  region                            = local.env.region
   optimize_rasters_arn              = module.viz-lambda-functions.optimize_rasters.arn
   update_egis_data_arn              = module.viz-lambda-functions.update_egis_data.arn
   fim_data_prep_arn                 = module.viz-lambda-functions.fim_data_prep.arn
@@ -645,16 +646,24 @@ module "step-functions" {
   schism_fim_job_definition_arn     = module.viz-lambda-functions.schism_fim.job_definition.arn
   schism_fim_job_queue_arn          = module.viz-lambda-functions.schism_fim.job_queue.arn
   schism_fim_datasets_bucket        = module.s3.buckets["deployment"].bucket
-  initialize_pipeline_arn           = module.viz-lambda-functions.initialize_pipeline.arn
-  rnr_domain_generator_arn          = module.rnr-lambda-functions.rnr_domain_generator.arn
   email_sns_topics                  = module.sns.email_sns_topics
   aws_instances_to_reboot           = [module.rnr.ec2.id]
-  fifteen_minute_trigger            = module.eventbridge.fifteen_minute_eventbridge
   viz_processing_pipeline_log_group = module.cloudwatch.viz_processing_pipeline_log_group.name
   rds_bastion_id                    = module.rds-bastion.instance-id
   test_wrds_db_lambda_arn           = module.viz-lambda-functions.test_wrds_db.arn
 }
 
+module "rnr-step-functions" {
+  source = "./StepFunctions/rnr"
+
+  rnr_lambda_role                   = module.iam-roles.role_sync_wrds_location_db.arn
+  environment                       = local.env.environment
+  initialize_pipeline_arn           = module.viz-lambda-functions.initialize_pipeline.arn
+  rnr_domain_generator_arn          = module.rnr-lambda-functions.rnr_domain_generator.arn
+  rnr_ec2_instance                  = module.rnr.ec2.id
+  fifteen_minute_trigger            = module.eventbridge.fifteen_minute_eventbridge
+}
+
 # Event Bridge
 module "eventbridge" {
   source = "./EventBridge"
@@ -704,5 +713,5 @@ module "testing" {
   environment                 = local.env.environment
   s3_module                   = module.s3
   lambda_module               = module.viz-lambda-functions
-  step_function_module        = module.step-functions
+  step_function_module        = module.viz-step-functions
 }