feat: create KMS key decryption policy for S3 (#216)

* feat: create KMS key decryption policy for S3 * feat: update examples * feat: update changelog * feat: update module readme
coralogix · Feb 27, 2025 · 4b73529 · 4b73529
1 parent 2a15636
commit 4b73529
Show file tree

Hide file tree

Showing 5 changed files with 115 additions and 88 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # Changelog
 
+## v2.8.0
+#### **coralogix-aws-shipper**
+### 🚀 New components 🚀
+- Add KMS Key support for S3 Buckets
+
 ## v2.7.0
 #### **eventbridge**
 ### 🚀 New components 🚀

diff --git a/examples/coralogix-aws-shipper/variables.tf b/examples/coralogix-aws-shipper/variables.tf
@@ -71,6 +71,12 @@ variable "s3_key_suffix" {
   default     = null
 }
 
+variable "s3_bucket_kms_arn" {
+  description = "The AWS ARN of the KMS key used to encrypt/decrypt objects in the specified S3 bucket. If provided, the Lambda policy will include permissions to decrypt using this key."
+  type        = string
+  default     = null
+}
+
 variable "cs_delimiter" {
   type        = string
   description = "The delimiter used in the CSV file to process This value is applied when the S3Csv integration type is selected"

diff --git a/modules/coralogix-aws-shipper/README.md b/modules/coralogix-aws-shipper/README.md
@@ -62,6 +62,7 @@ If you're deploying multiple integrations through the same S3 bucket, you'll nee
 | <a name="input_s3_bucket_name"></a> [s3\_bucket\_name](#input\_s3\_bucket\_name) | The name of the S3 bucket to watch. | `string` | n/a | yes |
 | <a name="input_s3_key_prefix"></a> [s3\_key\_prefix](#input\_s3\_key\_prefix) | The S3 path prefix to watch. | `string` |  n/a | no |
 | <a name="input_s3_key_suffix"></a> [s3\_key\_suffix](#input\_s3\_key\_suffix) | The S3 path suffix to watch. | `string` |  n/a | no |
+| <a name="input_s3_bucket_kms_arn"></a> [s3\_bucket\_kms\_arn](#input\_s3\_bucket\_kms\_arn) | The AWS ARN of the KMS key used to encrypt/decrypt objects in the specified S3 bucket. If provided, the Lambda policy will include permissions to decrypt using this key. | `string` |  n/a | no |
 | <a name="input_csv_delimiter"></a> [csv_delimiter](#input\_csv\_delimiter) | A single character to be used as a delimiter when ingesting a CSV file with a header line. This value is applicable when the S3Csv integration type is selected, for example, “,” or ” “.  | `string` |  n/a | no |
 | <a name="input_custom_csv_header"></a> [custom\_csv\_header](#input\_custom\_csv\_header) | List seperated by cs delimiter of a new headers for your csv, the variable must be with the same delimiter as the cs_delimiter. For example, if the cs_delimiter is \";\", then the variable vakue should be name;country;age, resulting in the following new headers in Corlaogix: name, country, age | `string` | n/a | no |
 | <a name="input_newline_pattern"></a> [newline\_pattern](#input\_newline\_pattern) | A regular expression to detect a new log line for multiline logs, e.g., \n(?=\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}.\d{3}). | `string` | n/a | no |

diff --git a/modules/coralogix-aws-shipper/main.tf b/modules/coralogix-aws-shipper/main.tf
@@ -49,93 +49,102 @@ resource "aws_iam_policy" "lambda_policy" {
   description = "Policy for Lambda function ${each.value.lambda_name == null ? module.locals[each.key].function_name : each.value.lambda_name}"
   policy = jsonencode({
     Version = "2012-10-17",
-    Statement = [
-      # because its not possible to leave a condition empty than we need the add the condition for cloudwatch
-
-      # DLQ SQS Permissions
-      {
-        Effect   = "Allow",
-        Action   = var.enable_dlq ? ["sqs:SendMessage", "sqs:ReceiveMessage", "sqs:DeleteMessage", "sqs:GetQueueAttributes"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
-        Resource = var.enable_dlq ? [aws_sqs_queue.DLQ[0].arn] : ["*"]
-      },
-
-      # DLQ S3 Permissions
-      {
-        Effect   = "Allow",
-        Action   = var.enable_dlq ? ["s3:PutObject", "s3:PutObjectAcl", "s3:AbortMultipartUpload", "s3:DeleteObject", "s3:PutObjectTagging", "s3:PutObjectVersionTagging"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
-        Resource = var.enable_dlq ? ["${data.aws_s3_bucket.dlq_bucket[0].arn}/*", data.aws_s3_bucket.dlq_bucket[0].arn] : ["*"]
-      },
-
-      # Secrets Access Policy
-      {
-        Effect   = "Allow",
-        Action   = each.value.store_api_key_in_secrets_manager == null || each.value.store_api_key_in_secrets_manager == true || local.api_key_is_arn ? ["secretsmanager:GetSecretValue"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
-        Resource = local.api_key_is_arn ? [var.api_key] : each.value.store_api_key_in_secrets_manager == null || each.value.store_api_key_in_secrets_manager == true ? [aws_secretsmanager_secret.coralogix_secret[each.key].arn] : ["*"]
-      },
-
-      # Destination on Failure Policy
-      {
-        Effect   = "Allow",
-        Action   = var.notification_email != null ? ["sns:Publish"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
-        Resource = var.notification_email != null ? [aws_sns_topic.this[each.key].arn] : ["*"]
-      },
-
-      # Private Link Policy
-      {
-        Effect   = "Allow",
-        Action   = var.subnet_ids != null ? ["ec2:CreateNetworkInterface", "ec2:DescribeNetworkInterfaces", "ec2:DeleteNetworkInterface"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
-        Resource = ["*"]
-      },
-
-      # SQS S3 Integration Policy
-      {
-        Effect   = "Allow",
-        Action   = var.sqs_name != null && var.s3_bucket_name != null ? ["s3:GetObject", "s3:ListBucket", "s3:GetBucketLocation", "s3:GetObjectVersion", "s3:GetLifecycleConfiguration"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
-        Resource = var.sqs_name != null && var.s3_bucket_name != null ? ["${data.aws_s3_bucket.this[0].arn}/*", data.aws_s3_bucket.this[0].arn] : ["*"]
-      },
-
-      # EcrScan Integration Policy
-      {
-        Effect   = "Allow",
-        Action   = var.integration_type == "EcrScan" ? ["ecr:DescribeImageScanFindings"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
-        Resource = ["*"]
-      },
-
-      # S3 Integration Policy
-      {
-        Effect   = "Allow",
-        Action   = var.s3_bucket_name != null && var.sqs_name == null ? ["s3:GetObject"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
-        Resource = var.s3_bucket_name != null && var.sqs_name == null ? ["${data.aws_s3_bucket.this[0].arn}/*"] : ["*"]
-      },
-
-      #S3 with SQS Integration Policy
-      {
-        Effect   = "Allow",
-        Action   = var.s3_bucket_name != null && var.sqs_name != null ? ["sqs:ReceiveMessage", "sqs:DeleteMessage", "sqs:GetQueueAttributes"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
-        Resource = var.s3_bucket_name != null && var.sqs_name != null ? [data.aws_sqs_queue.name[0].arn] : ["*"]
-      },
-
-      # Kinesis Integration policy
-      {
-        Effect   = "Allow",
-        Action   = var.kinesis_stream_name != null ? ["kinesis:GetRecords", "kinesis:GetShardIterator", "kinesis:DescribeStream", "kinesis:ListStreams", "kinesis:ListShards", "kinesis:DescribeStreamSummary", "kinesis:SubscribeToShard"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
-        Resource = var.kinesis_stream_name != null ? [data.aws_kinesis_stream.kinesis_stream[0].arn] : ["*"]
-      },
-
-      # Kafka Integration Policy
-      {
-        Effect   = "Allow",
-        Action   = var.kafka_brokers != null ? ["ec2:CreateNetworkInterface", "ec2:DescribeNetworkInterfaces", "ec2:DescribeVpcs", "ec2:DeleteNetworkInterface", "ec2:DescribeSubnets", "ec2:DescribeSecurityGroups"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
-        Resource = ["*"]
-      },
-
-      # CloudWatch Logs Policy
-      {
-        Effect   = "Allow"
-        Action   = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"]
-        Resource = ["*"]
-      }
-    ]
+    Statement = concat(
+      [
+        # because its not possible to leave a condition empty than we need the add the condition for cloudwatch
+
+        # DLQ SQS Permissions
+        {
+          Effect   = "Allow",
+          Action   = var.enable_dlq ? ["sqs:SendMessage", "sqs:ReceiveMessage", "sqs:DeleteMessage", "sqs:GetQueueAttributes"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+          Resource = var.enable_dlq ? [aws_sqs_queue.DLQ[0].arn] : ["*"]
+        },
+
+        # DLQ S3 Permissions
+        {
+          Effect   = "Allow",
+          Action   = var.enable_dlq ? ["s3:PutObject", "s3:PutObjectAcl", "s3:AbortMultipartUpload", "s3:DeleteObject", "s3:PutObjectTagging", "s3:PutObjectVersionTagging"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+          Resource = var.enable_dlq ? ["${data.aws_s3_bucket.dlq_bucket[0].arn}/*", data.aws_s3_bucket.dlq_bucket[0].arn] : ["*"]
+        },
+
+        # Secrets Access Policy
+        {
+          Effect   = "Allow",
+          Action   = each.value.store_api_key_in_secrets_manager == null || each.value.store_api_key_in_secrets_manager == true || local.api_key_is_arn ? ["secretsmanager:GetSecretValue"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+          Resource = local.api_key_is_arn ? [var.api_key] : each.value.store_api_key_in_secrets_manager == null || each.value.store_api_key_in_secrets_manager == true ? [aws_secretsmanager_secret.coralogix_secret[each.key].arn] : ["*"]
+        },
+
+        # Destination on Failure Policy
+        {
+          Effect   = "Allow",
+          Action   = var.notification_email != null ? ["sns:Publish"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+          Resource = var.notification_email != null ? [aws_sns_topic.this[each.key].arn] : ["*"]
+        },
+
+        # Private Link Policy
+        {
+          Effect   = "Allow",
+          Action   = var.subnet_ids != null ? ["ec2:CreateNetworkInterface", "ec2:DescribeNetworkInterfaces", "ec2:DeleteNetworkInterface"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+          Resource = ["*"]
+        },
+
+        # SQS S3 Integration Policy
+        {
+          Effect   = "Allow",
+          Action   = var.sqs_name != null && var.s3_bucket_name != null ? ["s3:GetObject", "s3:ListBucket", "s3:GetBucketLocation", "s3:GetObjectVersion", "s3:GetLifecycleConfiguration"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+          Resource = var.sqs_name != null && var.s3_bucket_name != null ? ["${data.aws_s3_bucket.this[0].arn}/*", data.aws_s3_bucket.this[0].arn] : ["*"]
+        },
+
+        # EcrScan Integration Policy
+        {
+          Effect   = "Allow",
+          Action   = var.integration_type == "EcrScan" ? ["ecr:DescribeImageScanFindings"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+          Resource = ["*"]
+        },
+
+        # S3 Integration Policy
+        {
+          Effect   = "Allow",
+          Action   = var.s3_bucket_name != null && var.sqs_name == null ? ["s3:GetObject"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+          Resource = var.s3_bucket_name != null && var.sqs_name == null ? ["${data.aws_s3_bucket.this[0].arn}/*"] : ["*"]
+        },
+
+        #S3 with SQS Integration Policy
+        {
+          Effect   = "Allow",
+          Action   = var.s3_bucket_name != null && var.sqs_name != null ? ["sqs:ReceiveMessage", "sqs:DeleteMessage", "sqs:GetQueueAttributes"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+          Resource = var.s3_bucket_name != null && var.sqs_name != null ? [data.aws_sqs_queue.name[0].arn] : ["*"]
+        },
+
+        # Kinesis Integration policy
+        {
+          Effect   = "Allow",
+          Action   = var.kinesis_stream_name != null ? ["kinesis:GetRecords", "kinesis:GetShardIterator", "kinesis:DescribeStream", "kinesis:ListStreams", "kinesis:ListShards", "kinesis:DescribeStreamSummary", "kinesis:SubscribeToShard"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+          Resource = var.kinesis_stream_name != null ? [data.aws_kinesis_stream.kinesis_stream[0].arn] : ["*"]
+        },
+
+        # Kafka Integration Policy
+        {
+          Effect   = "Allow",
+          Action   = var.kafka_brokers != null ? ["ec2:CreateNetworkInterface", "ec2:DescribeNetworkInterfaces", "ec2:DescribeVpcs", "ec2:DeleteNetworkInterface", "ec2:DescribeSubnets", "ec2:DescribeSecurityGroups"] : ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+          Resource = ["*"]
+        },
+
+        # CloudWatch Logs Policy
+        {
+          Effect   = "Allow"
+          Action   = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"]
+          Resource = ["*"]
+        }
+      ],
+      var.s3_bucket_kms_arn != null ? [
+        {
+          Effect   = "Allow",
+          Action   = ["kms:Decrypt"],
+          Resource = [var.s3_bucket_kms_arn]
+        }
+      ] : []
+    )
   })
 }
 

diff --git a/modules/coralogix-aws-shipper/variables.tf b/modules/coralogix-aws-shipper/variables.tf
@@ -71,6 +71,12 @@ variable "s3_key_suffix" {
   default     = null
 }
 
+variable "s3_bucket_kms_arn" {
+  description = "The AWS ARN of the KMS key used to encrypt/decrypt objects in the specified S3 bucket. If provided, the Lambda policy will include permissions to decrypt using this key."
+  type        = string
+  default     = null
+}
+
 variable "cs_delimiter" {
   type        = string
   description = "The delimiter used in the CSV file to process This value is applied when the S3Csv integration type is selected"
@@ -115,7 +121,7 @@ variable "log_group_prefix" {
   default     = null
 }
 
-# kinesis variables 
+# kinesis variables
 
 variable "kinesis_stream_name" {
   description = "The name of Kinesis stream to subscribe to retrieving messages"