From ee62d2b55e137b866849a92ea67c019e77786e76 Mon Sep 17 00:00:00 2001 From: Paul Minton Date: Wed, 7 Jun 2023 12:14:47 -0700 Subject: [PATCH 01/19] add boostrap action var --- aws_datalake/modules/emr/main.tf | 1 + aws_datalake/modules/emr/variables.tf | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/aws_datalake/modules/emr/main.tf b/aws_datalake/modules/emr/main.tf index 1b0c581..87e3436 100644 --- a/aws_datalake/modules/emr/main.tf +++ b/aws_datalake/modules/emr/main.tf @@ -17,6 +17,7 @@ resource "aws_emr_cluster" "segment_data_lake_emr_cluster" { service_role = var.iam_emr_service_role autoscaling_role = var.iam_emr_autoscaling_role security_configuration = var.security_configuration + bootstrap_action = var.bootstrap_action master_instance_group { instance_type = var.master_instance_type diff --git a/aws_datalake/modules/emr/variables.tf b/aws_datalake/modules/emr/variables.tf index 49ff7c8..5e4a2d8 100644 --- a/aws_datalake/modules/emr/variables.tf +++ b/aws_datalake/modules/emr/variables.tf @@ -101,6 +101,11 @@ variable "task_instance_max_count" { default = "4" } +variable "bootstrap_action" { + description = "Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes" + type = list(object) +} + locals { tags = merge(tomap({"vendor" = "segment"}), var.tags) } From 4f2882b77c16bf878f2887a5f2226ba26599e4ea Mon Sep 17 00:00:00 2001 From: Paul Minton Date: Wed, 7 Jun 2023 16:13:00 -0700 Subject: [PATCH 02/19] type is map of strings --- aws_datalake/modules/emr/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_datalake/modules/emr/variables.tf b/aws_datalake/modules/emr/variables.tf index 5e4a2d8..84e167a 100644 --- a/aws_datalake/modules/emr/variables.tf +++ b/aws_datalake/modules/emr/variables.tf @@ -103,7 +103,7 @@ variable "task_instance_max_count" { variable "bootstrap_action" { description = "Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes" - type = list(object) + type = list(map(string)) } locals { From 0c84704c8cd16d054825461910e17e3f61ead00e Mon Sep 17 00:00:00 2001 From: Paul Minton Date: Wed, 7 Jun 2023 16:15:42 -0700 Subject: [PATCH 03/19] default --- aws_datalake/modules/emr/variables.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/aws_datalake/modules/emr/variables.tf b/aws_datalake/modules/emr/variables.tf index 84e167a..e08872d 100644 --- a/aws_datalake/modules/emr/variables.tf +++ b/aws_datalake/modules/emr/variables.tf @@ -104,6 +104,7 @@ variable "task_instance_max_count" { variable "bootstrap_action" { description = "Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes" type = list(map(string)) + default = [] } locals { From 201d7c84d671134f039ddb6c46ea710a2319761e Mon Sep 17 00:00:00 2001 From: Paul Minton Date: Wed, 7 Jun 2023 16:23:54 -0700 Subject: [PATCH 04/19] block, not a list --- aws_datalake/modules/emr/main.tf | 2 +- aws_datalake/modules/emr/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_datalake/modules/emr/main.tf b/aws_datalake/modules/emr/main.tf index 87e3436..f673d7f 100644 --- a/aws_datalake/modules/emr/main.tf +++ b/aws_datalake/modules/emr/main.tf @@ -17,7 +17,7 @@ resource "aws_emr_cluster" "segment_data_lake_emr_cluster" { service_role = var.iam_emr_service_role autoscaling_role = var.iam_emr_autoscaling_role security_configuration = var.security_configuration - bootstrap_action = var.bootstrap_action + bootstrap_action { var.bootstrap_action } master_instance_group { instance_type = var.master_instance_type diff --git a/aws_datalake/modules/emr/variables.tf b/aws_datalake/modules/emr/variables.tf index e08872d..d2b3f53 100644 --- a/aws_datalake/modules/emr/variables.tf +++ b/aws_datalake/modules/emr/variables.tf @@ -103,7 +103,7 @@ variable "task_instance_max_count" { variable "bootstrap_action" { description = "Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes" - type = list(map(string)) + type = map(string) default = [] } From c854dd8782993eb1e50cfe622c1233f4d1b3b7d3 Mon Sep 17 00:00:00 2001 From: Paul Minton Date: Wed, 7 Jun 2023 16:34:30 -0700 Subject: [PATCH 05/19] unpack var to block --- aws_datalake/modules/emr/main.tf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/aws_datalake/modules/emr/main.tf b/aws_datalake/modules/emr/main.tf index f673d7f..8b32ef1 100644 --- a/aws_datalake/modules/emr/main.tf +++ b/aws_datalake/modules/emr/main.tf @@ -17,7 +17,11 @@ resource "aws_emr_cluster" "segment_data_lake_emr_cluster" { service_role = var.iam_emr_service_role autoscaling_role = var.iam_emr_autoscaling_role security_configuration = var.security_configuration - bootstrap_action { var.bootstrap_action } + bootstrap_action { + path = var.bootstrap_action.path + name = var.bootstrap_action.name + args = var.bootstrap_action.args + } master_instance_group { instance_type = var.master_instance_type From 0337de2ce01606cecf895e6eed40f1769694818a Mon Sep 17 00:00:00 2001 From: Paul Minton Date: Wed, 7 Jun 2023 16:47:11 -0700 Subject: [PATCH 06/19] fix default --- aws_datalake/modules/emr/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_datalake/modules/emr/variables.tf b/aws_datalake/modules/emr/variables.tf index d2b3f53..0fe0226 100644 --- a/aws_datalake/modules/emr/variables.tf +++ b/aws_datalake/modules/emr/variables.tf @@ -104,7 +104,7 @@ variable "task_instance_max_count" { variable "bootstrap_action" { description = "Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes" type = map(string) - default = [] + default = {} } locals { From 5ca34ad19171f371050112a89f630ce6d758b553 Mon Sep 17 00:00:00 2001 From: Paul Minton Date: Wed, 7 Jun 2023 16:56:46 -0700 Subject: [PATCH 07/19] any --- aws_datalake/modules/emr/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_datalake/modules/emr/variables.tf b/aws_datalake/modules/emr/variables.tf index 0fe0226..939b0da 100644 --- a/aws_datalake/modules/emr/variables.tf +++ b/aws_datalake/modules/emr/variables.tf @@ -103,7 +103,7 @@ variable "task_instance_max_count" { variable "bootstrap_action" { description = "Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes" - type = map(string) + type = map(any) default = {} } From 13af5f02f3af7412b9159407b342ebf0ad268664 Mon Sep 17 00:00:00 2001 From: Paul Minton Date: Thu, 8 Jun 2023 10:25:03 -0700 Subject: [PATCH 08/19] object not map, everything is typed --- aws_datalake/modules/emr/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_datalake/modules/emr/variables.tf b/aws_datalake/modules/emr/variables.tf index 939b0da..a720727 100644 --- a/aws_datalake/modules/emr/variables.tf +++ b/aws_datalake/modules/emr/variables.tf @@ -103,7 +103,7 @@ variable "task_instance_max_count" { variable "bootstrap_action" { description = "Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes" - type = map(any) + type = object({path = string, name = string, args = list(string)}) default = {} } From 0dac93f3adf4e68dc056ba8ca680b2170841f492 Mon Sep 17 00:00:00 2001 From: Paul Minton Date: Thu, 8 Jun 2023 10:31:55 -0700 Subject: [PATCH 09/19] copy from aws module --- aws_datalake/modules/emr/main.tf | 13 +++++++++---- aws_datalake/modules/emr/variables.tf | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/aws_datalake/modules/emr/main.tf b/aws_datalake/modules/emr/main.tf index 8b32ef1..e73eae2 100644 --- a/aws_datalake/modules/emr/main.tf +++ b/aws_datalake/modules/emr/main.tf @@ -17,10 +17,15 @@ resource "aws_emr_cluster" "segment_data_lake_emr_cluster" { service_role = var.iam_emr_service_role autoscaling_role = var.iam_emr_autoscaling_role security_configuration = var.security_configuration - bootstrap_action { - path = var.bootstrap_action.path - name = var.bootstrap_action.name - args = var.bootstrap_action.args + + dynamic "bootstrap_action" { + for_each = var.bootstrap_action + + content { + args = try(bootstrap_action.value.args, null) + name = bootstrap_action.value.name + path = bootstrap_action.value.path + } } master_instance_group { diff --git a/aws_datalake/modules/emr/variables.tf b/aws_datalake/modules/emr/variables.tf index a720727..f1c0186 100644 --- a/aws_datalake/modules/emr/variables.tf +++ b/aws_datalake/modules/emr/variables.tf @@ -103,7 +103,7 @@ variable "task_instance_max_count" { variable "bootstrap_action" { description = "Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes" - type = object({path = string, name = string, args = list(string)}) + type = any default = {} } From 09cfb2fc71873559fe7f80171247569e900dfa6d Mon Sep 17 00:00:00 2001 From: Paul Minton Date: Fri, 9 Jun 2023 09:48:30 -0700 Subject: [PATCH 10/19] attach current policy --- aws_datalake/modules/iam/main.tf | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/aws_datalake/modules/iam/main.tf b/aws_datalake/modules/iam/main.tf index 044b806..ff7f6b0 100644 --- a/aws_datalake/modules/iam/main.tf +++ b/aws_datalake/modules/iam/main.tf @@ -289,9 +289,8 @@ resource "aws_iam_instance_profile" "segment_emr_instance_profile" { role = aws_iam_role.segment_emr_instance_profile_role.name } -resource "aws_iam_role_policy" "segment_emr_instance_profile_policy" { +resource "aws_aim_policy" "segment_emr_instance_profile_policy" { name = "SegmentEMRInstanceProfilePolicy${var.suffix}" - role = aws_iam_role.segment_emr_instance_profile_role.id policy = < Date: Fri, 9 Jun 2023 10:37:13 -0700 Subject: [PATCH 11/19] add custom policy for now --- aws_datalake/modules/iam/main.tf | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/aws_datalake/modules/iam/main.tf b/aws_datalake/modules/iam/main.tf index ff7f6b0..05cbb68 100644 --- a/aws_datalake/modules/iam/main.tf +++ b/aws_datalake/modules/iam/main.tf @@ -363,11 +363,41 @@ resource "aws_aim_policy" "segment_emr_instance_profile_policy" { EOF } +resource "aws_iam_policy" "custom_emr_instance_profile_policy" { + name = "CalmEMRInstanceProfilePolicy${var.suffix}" + policy = < Date: Fri, 9 Jun 2023 10:54:47 -0700 Subject: [PATCH 12/19] typos --- aws_datalake/modules/iam/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aws_datalake/modules/iam/main.tf b/aws_datalake/modules/iam/main.tf index 05cbb68..69a44d7 100644 --- a/aws_datalake/modules/iam/main.tf +++ b/aws_datalake/modules/iam/main.tf @@ -289,7 +289,7 @@ resource "aws_iam_instance_profile" "segment_emr_instance_profile" { role = aws_iam_role.segment_emr_instance_profile_role.name } -resource "aws_aim_policy" "segment_emr_instance_profile_policy" { +resource "aws_iam_policy" "segment_emr_instance_profile_policy" { name = "SegmentEMRInstanceProfilePolicy${var.suffix}" policy = < Date: Fri, 9 Jun 2023 10:59:33 -0700 Subject: [PATCH 13/19] more typos --- aws_datalake/modules/iam/main.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aws_datalake/modules/iam/main.tf b/aws_datalake/modules/iam/main.tf index 69a44d7..c0c21fa 100644 --- a/aws_datalake/modules/iam/main.tf +++ b/aws_datalake/modules/iam/main.tf @@ -389,13 +389,13 @@ EOF } resource "aws_iam_role_policy_attachment" "segment_emr_instance_profile_policy_attachment" { - role = segment_emr_instance_profile_role.name - policy_arn = segment_emr_instance_profile_policy.arn + role = aws_iam_role.segment_emr_instance_profile_role.name + policy_arn = aws_iam_policy.segment_emr_instance_profile_policy.arn } resource "aws_iam_role_policy_attachment" "segment_emr_instance_profile_policy_custom_attachment" { - role = segment_emr_instance_profile_role.name - policy_arn = custom_emr_instance_profile_policy.arn + role = aws_iam_role.segment_emr_instance_profile_role.name + policy_arn = aws_iam_policy.custom_emr_instance_profile_policy.arn } # IAM Role for EMR Autoscaling role From 758b1bb99e7d820383d602d5dc6745f8a7ad9055 Mon Sep 17 00:00:00 2001 From: Paul Minton Date: Fri, 9 Jun 2023 11:20:53 -0700 Subject: [PATCH 14/19] change hard coded ssm path --- aws_datalake/modules/iam/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_datalake/modules/iam/main.tf b/aws_datalake/modules/iam/main.tf index c0c21fa..b34bdc5 100644 --- a/aws_datalake/modules/iam/main.tf +++ b/aws_datalake/modules/iam/main.tf @@ -381,7 +381,7 @@ resource "aws_iam_policy" "custom_emr_instance_profile_policy" { "Action": [ "ssm:GetParameters" ], - "Resource": "arn:aws:ssm:us-east-1:083265760884:parameter/data-dev/datalakes" + "Resource": "arn:aws:ssm:us-east-1:083265760884:parameter/data-dev/us-east-1/segment-data-lake" } ] } From a4c975f49316da82ac52f9e5728ecb1842ef6667 Mon Sep 17 00:00:00 2001 From: Paul Minton Date: Fri, 9 Jun 2023 13:27:36 -0700 Subject: [PATCH 15/19] wildcard --- aws_datalake/modules/iam/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aws_datalake/modules/iam/main.tf b/aws_datalake/modules/iam/main.tf index b34bdc5..630641a 100644 --- a/aws_datalake/modules/iam/main.tf +++ b/aws_datalake/modules/iam/main.tf @@ -380,8 +380,9 @@ resource "aws_iam_policy" "custom_emr_instance_profile_policy" { "Effect": "Allow", "Action": [ "ssm:GetParameters" + ], - "Resource": "arn:aws:ssm:us-east-1:083265760884:parameter/data-dev/us-east-1/segment-data-lake" + "Resource": "arn:aws:ssm:us-east-1:083265760884:parameter/data-dev/us-east-1/segment-data-lake/*" } ] } From f64115115885efa60ecb91ec51a3b3b0ed71cbab Mon Sep 17 00:00:00 2001 From: Paul Minton Date: Fri, 9 Jun 2023 13:46:29 -0700 Subject: [PATCH 16/19] parameter(no-s) --- aws_datalake/modules/iam/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_datalake/modules/iam/main.tf b/aws_datalake/modules/iam/main.tf index 630641a..81b0d59 100644 --- a/aws_datalake/modules/iam/main.tf +++ b/aws_datalake/modules/iam/main.tf @@ -379,7 +379,7 @@ resource "aws_iam_policy" "custom_emr_instance_profile_policy" { { "Effect": "Allow", "Action": [ - "ssm:GetParameters" + "ssm:GetParameter" ], "Resource": "arn:aws:ssm:us-east-1:083265760884:parameter/data-dev/us-east-1/segment-data-lake/*" From 7734c1f3d3b3a5e5f86607e12c292e1369f87dfd Mon Sep 17 00:00:00 2001 From: Paul Minton Date: Fri, 9 Jun 2023 14:31:00 -0700 Subject: [PATCH 17/19] make the policy a var --- aws_datalake/modules/iam/main.tf | 29 ++------------------------- aws_datalake/modules/iam/variables.tf | 6 ++++++ 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/aws_datalake/modules/iam/main.tf b/aws_datalake/modules/iam/main.tf index 81b0d59..82c1b96 100644 --- a/aws_datalake/modules/iam/main.tf +++ b/aws_datalake/modules/iam/main.tf @@ -363,40 +363,15 @@ resource "aws_iam_policy" "segment_emr_instance_profile_policy" { EOF } -resource "aws_iam_policy" "custom_emr_instance_profile_policy" { - name = "CalmEMRInstanceProfilePolicy${var.suffix}" - policy = < Date: Fri, 9 Jun 2023 15:10:13 -0700 Subject: [PATCH 18/19] use static var in count --- aws_datalake/modules/iam/main.tf | 2 +- aws_datalake/modules/iam/variables.tf | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/aws_datalake/modules/iam/main.tf b/aws_datalake/modules/iam/main.tf index 82c1b96..6619dad 100644 --- a/aws_datalake/modules/iam/main.tf +++ b/aws_datalake/modules/iam/main.tf @@ -369,7 +369,7 @@ resource "aws_iam_role_policy_attachment" "segment_emr_instance_profile_policy_a } resource "aws_iam_role_policy_attachment" "segment_emr_instance_profile_policy_custom_attachment" { - count = var.custom_emr_instance_profile_policy_arn ? 1 : 0 + count = var.attach_custom_emr_isntance_profile_policy ? 1 : 0 role = aws_iam_role.segment_emr_instance_profile_role.name policy_arn = var.custom_emr_instance_profile_policy_arn } diff --git a/aws_datalake/modules/iam/variables.tf b/aws_datalake/modules/iam/variables.tf index 4c36e09..cdb89a9 100644 --- a/aws_datalake/modules/iam/variables.tf +++ b/aws_datalake/modules/iam/variables.tf @@ -52,6 +52,12 @@ variable "custom_emr_instance_profile_policy_arn" { default = "" } +variable "attach_custom_emr_isntance_profile_policy" { + decription = "boolean flag to use the above policy arn" + type = bool + default = false +} + locals { tags = merge(tomap({"vendor" = "segment"}), var.tags) } From 620d1eb07bc26426dc963eb06c729bcdec1fbd30 Mon Sep 17 00:00:00 2001 From: Paul Minton Date: Fri, 9 Jun 2023 15:12:54 -0700 Subject: [PATCH 19/19] typo --- aws_datalake/modules/iam/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_datalake/modules/iam/variables.tf b/aws_datalake/modules/iam/variables.tf index cdb89a9..d519efb 100644 --- a/aws_datalake/modules/iam/variables.tf +++ b/aws_datalake/modules/iam/variables.tf @@ -53,7 +53,7 @@ variable "custom_emr_instance_profile_policy_arn" { } variable "attach_custom_emr_isntance_profile_policy" { - decription = "boolean flag to use the above policy arn" + description = "boolean flag to use the above policy arn" type = bool default = false }