From dadf9ef81715c1f4f120f1efa7cdab495d9c18e7 Mon Sep 17 00:00:00 2001 From: Edison <161050662+EdisonOrellana-NOAA@users.noreply.github.com> Date: Wed, 4 Dec 2024 09:25:54 -0600 Subject: [PATCH] tag = {} for all aws_s3_objects (#1003) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why This is to fix deployment errors with s3 objects having >10 tags which is [not allowed.](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html) ## Additions - `tags = {}` ## Screenshots I did a really cool regex find and replace: ![amazing-regex](https://github.com/user-attachments/assets/1c566abf-94f3-4183-b1d2-393894357828) ## Notes Drix encountered this error: ``` │ Error: error updating tags: error setting resource tags (hydrovis-ti-deployment-us-east-1/terraform_artifacts/EC2/Ingest/owp-hml-ingester.tar.gz): BadRequest: Object tags cannot be greater than 10 │ status code: 400, request id: PA30P15KR8DPBNXN, host id: HI7YPFc9BOOKetco+P8fLEvb1p/0WS+vXX8U8+w9qBZgyKPFZrSxSnPYNRnsrWwC4amexhSkdQk= │ │ with module.data-ingest-ec2.aws_s3_object.hml_ingester, │ on EC2/Ingest/[main.tf](http://main.tf/) line 86, in resource "aws_s3_object" "hml_ingester": │ 86: resource "aws_s3_object" "hml_ingester" { │ ``` --- Core/EC2/Ingest/main.tf | 10 ++++ Core/EC2/RDSBastion/main.tf | 12 +++++ Core/EC2/rnr/main.tf | 10 ++++ Core/EC2/viz/main.tf | 10 ++++ Core/LAMBDA/ingest_functions/main.tf | 2 +- Core/LAMBDA/layers/main.tf | 19 +++++++ Core/LAMBDA/rnr_functions/main.tf | 3 +- Core/LAMBDA/viz_functions/image_based/main.tf | 18 ++++++- .../viz_schism_fim_processing/main.tf | 10 ++++ Core/LAMBDA/viz_functions/main.tf | 18 ++++++- Core/Testing/main.tf | 10 ++++ Core/main.tf | 49 +++++++++++++++---- 12 files changed, 157 insertions(+), 14 deletions(-) diff --git a/Core/EC2/Ingest/main.tf b/Core/EC2/Ingest/main.tf index e4c22135..fd61a185 100644 --- a/Core/EC2/Ingest/main.tf +++ b/Core/EC2/Ingest/main.tf @@ -1,3 +1,12 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + configuration_aliases = [ aws.sns, aws.no_tags] + } + } +} + ############### ## VARIABLES ## ############### @@ -84,6 +93,7 @@ locals { ############### resource "aws_s3_object" "hml_ingester" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/owp-hml-ingester.tar.gz" source = "${path.module}/../../../Source/Ingest/owp-hml-ingester.tar.gz" diff --git a/Core/EC2/RDSBastion/main.tf b/Core/EC2/RDSBastion/main.tf index fc0c09d1..6ea4a254 100644 --- a/Core/EC2/RDSBastion/main.tf +++ b/Core/EC2/RDSBastion/main.tf @@ -1,3 +1,12 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + configuration_aliases = [ aws.sns, aws.no_tags] + } + } +} + ############### ## VARIABLES ## ############### @@ -220,6 +229,7 @@ resource "aws_instance" "rds-bastion" { ############### resource "aws_s3_object" "postgis_setup" { + provider = aws.no_tags bucket = var.data_deployment_bucket key = "terraform_artifacts/${path.module}/postgis_setup.sql" source = "${path.module}/data/postgis_setup.sql" @@ -227,6 +237,7 @@ resource "aws_s3_object" "postgis_setup" { } resource "aws_s3_object" "ingest_rfcfcst_base" { + provider = aws.no_tags bucket = var.data_deployment_bucket key = "terraform_artifacts/${path.module}/ingest/rfcfcst_base.sql.gz" source = "${path.module}/data/ingest/rfcfcst_base.sql.gz" @@ -234,6 +245,7 @@ resource "aws_s3_object" "ingest_rfcfcst_base" { } resource "aws_s3_object" "ingest_ingest_users" { + provider = aws.no_tags bucket = var.data_deployment_bucket key = "terraform_artifacts/${path.module}/ingest/ingest_users.sql" content = templatefile("${path.module}/data/ingest/ingest_users.sql.tftpl", { diff --git a/Core/EC2/rnr/main.tf b/Core/EC2/rnr/main.tf index 9da08274..71b41eb6 100644 --- a/Core/EC2/rnr/main.tf +++ b/Core/EC2/rnr/main.tf @@ -1,3 +1,12 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + configuration_aliases = [ aws.sns, aws.no_tags] + } + } +} + ############### ## VARIABLES ## ############### @@ -95,6 +104,7 @@ locals { ############### resource "aws_s3_object" "replace_route" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/owp-viz-replace-route.tgz" source = "${path.module}/../../../Source/RnR/owp-viz-replace-route.tgz" diff --git a/Core/EC2/viz/main.tf b/Core/EC2/viz/main.tf index 66e8477e..8b652c66 100644 --- a/Core/EC2/viz/main.tf +++ b/Core/EC2/viz/main.tf @@ -1,3 +1,12 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + configuration_aliases = [ aws.sns, aws.no_tags] + } + } +} + ####################### ## DYNAMIC VARIABLES ## ####################### @@ -171,6 +180,7 @@ data "aws_ami" "windows" { ################ resource "aws_s3_object" "setup_upload" { + provider = aws.no_tags bucket = var.deployment_data_bucket key = "terraform_artifacts/${path.module}/scripts/viz_ec2_setup.ps1" source = "${path.module}/scripts/viz_ec2_setup.ps1" diff --git a/Core/LAMBDA/ingest_functions/main.tf b/Core/LAMBDA/ingest_functions/main.tf index 1a3533aa..c2808cdf 100644 --- a/Core/LAMBDA/ingest_functions/main.tf +++ b/Core/LAMBDA/ingest_functions/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - configuration_aliases = [ aws.sns ] + configuration_aliases = [ aws.sns, aws.no_tags] } } } diff --git a/Core/LAMBDA/layers/main.tf b/Core/LAMBDA/layers/main.tf index 212b4c68..01a6b17e 100644 --- a/Core/LAMBDA/layers/main.tf +++ b/Core/LAMBDA/layers/main.tf @@ -1,3 +1,12 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + configuration_aliases = [ aws.sns, aws.no_tags] + } + } +} + variable "environment" { description = "Hydrovis environment" type = string @@ -23,6 +32,7 @@ variable "deployment_bucket" { ###################### resource "aws_s3_object" "es_logging" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/es_logging.zip" source = "${path.module}/es_logging.zip" @@ -52,6 +62,7 @@ data "archive_file" "viz_lambda_shared_funcs_zip" { } resource "aws_s3_object" "viz_lambda_shared_funcs_zip_upload" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/viz_lambda_shared_funcs.zip" source = data.archive_file.viz_lambda_shared_funcs_zip.output_path @@ -75,6 +86,7 @@ resource "aws_lambda_layer_version" "viz_lambda_shared_funcs" { ############################# resource "aws_s3_object" "arcgis_python_api" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/arcgis_python_api.zip" source = "${path.module}/arcgis_python_api.zip" @@ -98,6 +110,7 @@ resource "aws_lambda_layer_version" "arcgis_python_api" { ################## resource "aws_s3_object" "pandas" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/pandas.zip" source = "${path.module}/pandas.zip" @@ -122,6 +135,7 @@ resource "aws_lambda_layer_version" "pandas" { resource "aws_s3_object" "geopandas" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/geopandas.zip" source = "${path.module}/geopandas.zip" @@ -145,6 +159,7 @@ resource "aws_lambda_layer_version" "geopandas" { ################################## resource "aws_s3_object" "psycopg2_sqlalchemy" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/psycopg2_sqlalchemy.zip" source = "${path.module}/psycopg2_sqlalchemy.zip" @@ -168,6 +183,7 @@ resource "aws_lambda_layer_version" "psycopg2_sqlalchemy" { ########################## resource "aws_s3_object" "huc_proc_combo" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/huc_proc_combo.zip" source = "${path.module}/huc_proc_combo.zip" @@ -191,6 +207,7 @@ resource "aws_lambda_layer_version" "huc_proc_combo" { ################## resource "aws_s3_object" "xarray" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/xarray.zip" source = "${path.module}/xarray.zip" @@ -214,6 +231,7 @@ resource "aws_lambda_layer_version" "xarray" { ################ resource "aws_s3_object" "pika" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/pika.zip" source = "${path.module}/pika.zip" @@ -237,6 +255,7 @@ resource "aws_lambda_layer_version" "pika" { #################### resource "aws_s3_object" "requests" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/requests.zip" source = "${path.module}/requests.zip" diff --git a/Core/LAMBDA/rnr_functions/main.tf b/Core/LAMBDA/rnr_functions/main.tf index f2253886..f9ba8284 100644 --- a/Core/LAMBDA/rnr_functions/main.tf +++ b/Core/LAMBDA/rnr_functions/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - configuration_aliases = [ aws.sns ] + configuration_aliases = [ aws.sns, aws.no_tags] } } } @@ -80,6 +80,7 @@ data "archive_file" "rnr_domain_generator_zip" { } resource "aws_s3_object" "rnr_domain_generator_zip_upload" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/rnr_domain_generator.zip" source = data.archive_file.rnr_domain_generator_zip.output_path diff --git a/Core/LAMBDA/viz_functions/image_based/main.tf b/Core/LAMBDA/viz_functions/image_based/main.tf index 79fa369a..d536a624 100644 --- a/Core/LAMBDA/viz_functions/image_based/main.tf +++ b/Core/LAMBDA/viz_functions/image_based/main.tf @@ -1,3 +1,12 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + configuration_aliases = [ aws.sns, aws.no_tags] + } + } +} + variable "environment" { type = string } @@ -129,6 +138,7 @@ data "archive_file" "raster_processing_zip" { } resource "aws_s3_object" "raster_processing_zip_upload" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/viz_raster_processing.zip" source = data.archive_file.raster_processing_zip.output_path @@ -253,6 +263,7 @@ data "archive_file" "optimize_rasters_zip" { } resource "aws_s3_object" "optimize_rasters_zip_upload" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/viz_optimize_rasters.zip" source = data.archive_file.optimize_rasters_zip.output_path @@ -396,6 +407,7 @@ data "archive_file" "hand_fim_processing_zip" { } resource "aws_s3_object" "hand_fim_processing_zip_upload" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/viz_hand_fim_processing.zip" source = data.archive_file.hand_fim_processing_zip.output_path @@ -496,7 +508,11 @@ data "aws_lambda_function" "viz_hand_fim_processing" { module "schism-fim" { source = "./viz_schism_fim_processing" - + providers = { + aws = aws + aws.sns = aws.sns + aws.no_tags = aws.no_tags + } environment = var.environment account_id = var.account_id region = var.region diff --git a/Core/LAMBDA/viz_functions/image_based/viz_schism_fim_processing/main.tf b/Core/LAMBDA/viz_functions/image_based/viz_schism_fim_processing/main.tf index 6416149a..3c91c7da 100644 --- a/Core/LAMBDA/viz_functions/image_based/viz_schism_fim_processing/main.tf +++ b/Core/LAMBDA/viz_functions/image_based/viz_schism_fim_processing/main.tf @@ -1,3 +1,12 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + configuration_aliases = [ aws.sns, aws.no_tags] + } + } +} + variable "environment" { type = string } @@ -87,6 +96,7 @@ data "archive_file" "schism_processing_zip" { } resource "aws_s3_object" "schism_processing_zip_upload" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/viz_schism_fim_processing.zip" source = data.archive_file.schism_processing_zip.output_path diff --git a/Core/LAMBDA/viz_functions/main.tf b/Core/LAMBDA/viz_functions/main.tf index e0144d6e..d8dd6c45 100644 --- a/Core/LAMBDA/viz_functions/main.tf +++ b/Core/LAMBDA/viz_functions/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - configuration_aliases = [ aws.sns ] + configuration_aliases = [ aws.sns, aws.no_tags] } } } @@ -239,6 +239,7 @@ data "archive_file" "egis_health_checker_zip" { } resource "aws_s3_object" "egis_health_checker_zip_upload" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/egis_health_checker.zip" source = data.archive_file.egis_health_checker_zip.output_path @@ -329,6 +330,7 @@ data "archive_file" "python_preprocessing_zip" { } resource "aws_s3_object" "python_preprocessing_zip_upload" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/viz_python_preprocessing.zip" source = data.archive_file.python_preprocessing_zip.output_path @@ -449,6 +451,7 @@ data "archive_file" "initialize_pipeline_zip" { } resource "aws_s3_object" "initialize_pipeline_zip_upload" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/viz_initialize_pipeline.zip" source = data.archive_file.initialize_pipeline_zip.output_path @@ -550,6 +553,7 @@ data "archive_file" "db_postprocess_sql_zip" { } resource "aws_s3_object" "db_postprocess_sql_zip_upload" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/viz_db_postprocess_sql.zip" source = data.archive_file.db_postprocess_sql_zip.output_path @@ -612,6 +616,7 @@ data "archive_file" "db_ingest_zip" { } resource "aws_s3_object" "db_ingest_zip_upload" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/viz_db_ingest.zip" source = data.archive_file.db_ingest_zip.output_path @@ -674,6 +679,7 @@ data "archive_file" "fim_data_prep_zip" { } resource "aws_s3_object" "fim_data_prep_zip_upload" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/viz_fim_data_prep.zip" source = data.archive_file.fim_data_prep_zip.output_path @@ -743,6 +749,7 @@ data "archive_file" "update_egis_data_zip" { } resource "aws_s3_object" "update_egis_data_zip_upload" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/viz_update_egis_data.zip" source = data.archive_file.update_egis_data_zip.output_path @@ -809,6 +816,7 @@ data "archive_file" "publish_service_zip" { } resource "aws_s3_object" "publish_service_zip_upload" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/viz_publish_service.zip" source = data.archive_file.publish_service_zip.output_path @@ -816,6 +824,7 @@ resource "aws_s3_object" "publish_service_zip_upload" { } resource "aws_s3_object" "viz_publish_mapx_files" { + provider = aws.no_tags for_each = fileset("${path.module}/viz_publish_service/services", "**/*.mapx") bucket = var.deployment_bucket key = "viz_mapx/${reverse(split("/",each.key))[0]}" @@ -897,6 +906,7 @@ data "archive_file" "viz_test_wrds_db_zip" { } resource "aws_s3_object" "viz_test_wrds_db_upload" { + provider = aws.no_tags bucket = var.deployment_bucket key = "terraform_artifacts/${path.module}/viz_update_egis_data.zip" source = data.archive_file.viz_test_wrds_db_zip.output_path @@ -946,7 +956,11 @@ resource "aws_lambda_function" "viz_test_wrds_db" { module "image-based-lambdas" { source = "./image_based" - + providers = { + aws = aws + aws.sns = aws.sns + aws.no_tags = aws.no_tags + } environment = var.environment account_id = var.account_id region = var.region diff --git a/Core/Testing/main.tf b/Core/Testing/main.tf index b83bcf61..ad3c184e 100644 --- a/Core/Testing/main.tf +++ b/Core/Testing/main.tf @@ -1,3 +1,12 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + configuration_aliases = [ aws.sns, aws.no_tags] + } + } +} + variable "environment" { type = string } @@ -82,6 +91,7 @@ data "aws_s3_objects" "test_nwm_outputs" { } resource "aws_s3_object_copy" "test" { + provider = aws.no_tags depends_on = [var.s3_module, var.lambda_module, var.step_function_module, aws_cloudwatch_event_target.trigger_pipeline_test_run] count = length(data.aws_s3_objects.test_nwm_outputs.keys) bucket = local.test_bucket diff --git a/Core/main.tf b/Core/main.tf index 9f2d6e66..c1fa66c7 100644 --- a/Core/main.tf +++ b/Core/main.tf @@ -44,6 +44,13 @@ provider "aws" { } } +provider "aws" { + alias = "no_tags" + region = local.env.region + profile = local.env.environment + shared_credentials_files = ["/cloud/aws/credentials"] +} + ###################### STAGE 1 ###################### # IAM Roles @@ -294,7 +301,11 @@ module "sagemaker" { # Lambda Layers module "lambda-layers" { source = "./LAMBDA/layers" - + providers = { + aws = aws + aws.sns = aws.sns + aws.no_tags = aws.no_tags + } environment = local.env.environment region = local.env.region viz_environment = local.env.environment == "prod" ? "production" : local.env.environment == "uat" ? "staging" : local.env.environment == "ti" ? "staging" : "development" @@ -362,7 +373,11 @@ module "rds-egis" { module "rds-bastion" { source = "./EC2/RDSBastion" - + providers = { + aws = aws + aws.sns = aws.sns + aws.no_tags = aws.no_tags + } environment = local.env.environment region = local.env.region account_id = local.env.account_id @@ -437,8 +452,8 @@ module "ingest-lambda-functions" { providers = { aws = aws aws.sns = aws.sns + aws.no_tags = aws.no_tags } - environment = local.env.environment region = local.env.region deployment_bucket = module.s3.buckets["deployment"].bucket @@ -463,7 +478,11 @@ module "ingest-lambda-functions" { # Data Ingest module "data-ingest-ec2" { source = "./EC2/Ingest" - + providers = { + aws = aws + aws.sns = aws.sns + aws.no_tags = aws.no_tags + } environment = local.env.environment region = local.env.region account_id = local.env.account_id @@ -488,7 +507,11 @@ module "data-ingest-ec2" { module "rnr" { source = "./EC2/rnr" - + providers = { + aws = aws + aws.sns = aws.sns + aws.no_tags = aws.no_tags + } environment = local.env.environment region = local.env.region account_id = local.env.account_id @@ -511,8 +534,8 @@ module "rnr-lambda-functions" { providers = { aws = aws aws.sns = aws.sns + aws.no_tags = aws.no_tags } - environment = local.env.environment region = local.env.region rnr_data_bucket = module.s3.buckets["rnr"].bucket @@ -578,8 +601,8 @@ module "viz-lambda-functions" { providers = { aws = aws aws.sns = aws.sns + aws.no_tags = aws.no_tags } - environment = local.env.environment account_id = local.env.account_id region = local.env.region @@ -678,7 +701,11 @@ module "eventbridge" { module "viz-ec2" { source = "./EC2/viz" - + providers = { + aws = aws + aws.sns = aws.sns + aws.no_tags = aws.no_tags + } environment = local.env.environment account_id = local.env.account_id region = local.env.region @@ -716,7 +743,11 @@ module "viz-ec2" { module "testing" { count = local.env.environment == "ti" ? 1 : 0 source = "./Testing" - + providers = { + aws = aws + aws.sns = aws.sns + aws.no_tags = aws.no_tags + } environment = local.env.environment s3_module = module.s3 lambda_module = module.viz-lambda-functions