feat: Server-Side Encryption for SQS Queues (#1619)

* feat: Server-Side Encryption for SQS Queues [#187061621](https://www.pivotaltracker.com/story/show/187061621) * fix: description must be in accordance with the code [#187061621](https://www.pivotaltracker.com/story/show/187061621) * test: version is not necessary [#187061621](https://www.pivotaltracker.com/story/show/187061621) * feat: KMS support [#187061621](https://www.pivotaltracker.com/story/show/187061621) * chore: remove unnecessary comment [#187061621](https://www.pivotaltracker.com/story/show/187061621) * chore: keep variables closer to where they are used [#187061621](https://www.pivotaltracker.com/story/show/187061621) * chore: use last version functionality of CF CLI [#187061621](https://www.pivotaltracker.com/story/show/187061621) * chore: apply correct format [#187061621](https://www.pivotaltracker.com/story/show/187061621) * test: delete unnecessary env variable [#187061621](https://www.pivotaltracker.com/story/show/187061621)
cloudfoundry · Mar 15, 2024 · 80a79b5 · 80a79b5
1 parent b3bc27b
commit 80a79b5
Show file tree

Hide file tree

Showing 10 changed files with 208 additions and 48 deletions.
diff --git a/acceptance-tests/sqs_test.go b/acceptance-tests/sqs_test.go
@@ -45,7 +45,7 @@ var _ = Describe("SQS", Label("sqs"), func() {
 		Expect(got).To(Equal(message))
 	})
 
-	It("uses a Standard queue with accociated DLQ and triggers redrive", func() {
+	It("uses a Standard queue with associated DLQ and triggers redrive", func() {
 		By("creating a DLQ service instance")
 		dlqServiceInstance := services.CreateInstance(
 			"csb-aws-sqs",

diff --git a/aws-sqs.yml b/aws-sqs.yml
@@ -95,6 +95,29 @@ provision:
         * `perMessageGroupId`: for high throughput mode
         When High throughput Mode is ON, the value for `deduplication_scope` must be `messageGroup` or the operation fails.
         If not defined for a FIFO queue it defaults to `perQueue`.
+    - field_name: sqs_managed_sse_enabled
+      type: boolean
+      details: Enable SQS-managed encryption keys for encrypting messages.
+      default: true
+    - field_name: kms_master_key_id
+      type: string
+      details: | 
+        Specify the AWS KMS customer master key (CMK) for encryption.
+        The `sqs_managed_sse_enabled` property must be set to `false` if a KMS master key ID is provided.
+      default: ""
+    - field_name: kms_data_key_reuse_period_seconds
+      type: integer
+      details: Duration in seconds for reuse of a data key for encrypting messages.
+      default: 300  # 5 minutes
+      constraints:
+        minimum: 60  # Minimum 1 minute
+        maximum: 86400  # Maximum 24 hours
+    - field_name: kms_extra_key_ids
+      type: string
+      details: |
+        A comma-separated list of AWS KMS key IDs used for SSE-KMS operations.
+        Since a DLQ can receive messages from multiple sources, all the KMS key IDs used as sources must be included.
+      default: ""
   computed_inputs:
     - name: instance_name
       default: csb-sqs-${request.instance_id}
@@ -105,6 +128,7 @@ provision:
       overwrite: true
       type: object
   template_refs:
+    data: terraform/sqs/provision/data.tf
     main: terraform/sqs/provision/main.tf
     outputs: terraform/sqs/provision/outputs.tf
     provider: terraform/sqs/provision/providers.tf
@@ -126,6 +150,9 @@ provision:
     - field_name: dlq_arn
       type: string
       details: The ARN of the associated DLQ.
+    - field_name: kms_all_key_ids
+      type: string
+      details: The `kms_master_key_id` and `kms_extra_key_ids` AWS KMS key IDs used for SSE-KMS operations.
 bind:
   plan_inputs: []
   user_inputs: []
@@ -152,6 +179,10 @@ bind:
       default: ${instance.details["dlq_arn"]}
       overwrite: true
       type: string
+    - name: kms_all_key_ids
+      default: ${instance.details["kms_all_key_ids"]}
+      overwrite: true
+      type: string
   template_refs:
     data: terraform/sqs/bind/data.tf
     main: terraform/sqs/bind/main.tf

diff --git a/integration-tests/sqs_test.go b/integration-tests/sqs_test.go
@@ -87,6 +87,16 @@ var _ = Describe("SQS", Label("SQS"), func() {
 				map[string]any{"region": "-Asia-northeast1"},
 				"region: Does not match pattern '^[a-z][a-z0-9-]+$'",
 			),
+			Entry(
+				"kms_data_key_reuse_period_seconds maximum value is 86400",
+				map[string]any{"kms_data_key_reuse_period_seconds": 86401},
+				"kms_data_key_reuse_period_seconds: Must be less than or equal to 86400",
+			),
+			Entry(
+				"kms_data_key_reuse_period_seconds minimum value is 60",
+				map[string]any{"kms_data_key_reuse_period_seconds": 10},
+				"kms_data_key_reuse_period_seconds: Must be greater than or equal to 60",
+			),
 		)
 
 		It("should provision a queue", func() {
@@ -111,23 +121,29 @@ var _ = Describe("SQS", Label("SQS"), func() {
 					HaveKeyWithValue("aws_access_key_id", awsAccessKeyID),
 					HaveKeyWithValue("aws_secret_access_key", awsSecretAccessKey),
 					HaveKeyWithValue("dlq_arn", Equal("")),
+					HaveKeyWithValue("sqs_managed_sse_enabled", BeTrue()),
+					HaveKeyWithValue("kms_master_key_id", Equal("")),
+					HaveKeyWithValue("kms_data_key_reuse_period_seconds", BeNumerically("==", 300)),
 				),
 			)
 		})
 
 		It("should allow properties to be set on provision", func() {
 			_, err := broker.Provision(sqsServiceName, sqsCustomStandardPlanName, map[string]any{
-				"region":                     "africa-north-4",
-				"fifo":                       true,
-				"visibility_timeout_seconds": 60,
-				"message_retention_seconds":  60,
-				"max_message_size":           1024,
-				"delay_seconds":              600,
-				"receive_wait_time_seconds":  20,
-				"aws_access_key_id":          "fake-aws-access-key-id",
-				"aws_secret_access_key":      "fake-aws-secret-access-key",
-				"dlq_arn":                    "fake-arn",
-				"max_receive_count":          5,
+				"region":                            "africa-north-4",
+				"fifo":                              true,
+				"visibility_timeout_seconds":        60,
+				"message_retention_seconds":         60,
+				"max_message_size":                  1024,
+				"delay_seconds":                     600,
+				"receive_wait_time_seconds":         20,
+				"aws_access_key_id":                 "fake-aws-access-key-id",
+				"aws_secret_access_key":             "fake-aws-secret-access-key",
+				"dlq_arn":                           "fake-arn",
+				"max_receive_count":                 5,
+				"sqs_managed_sse_enabled":           false,
+				"kms_master_key_id":                 "xxxx",
+				"kms_data_key_reuse_period_seconds": 86_400,
 			})
 			Expect(err).NotTo(HaveOccurred())
 
@@ -144,6 +160,9 @@ var _ = Describe("SQS", Label("SQS"), func() {
 					HaveKeyWithValue("aws_secret_access_key", "fake-aws-secret-access-key"),
 					HaveKeyWithValue("dlq_arn", "fake-arn"),
 					HaveKeyWithValue("max_receive_count", BeNumerically("==", 5)),
+					HaveKeyWithValue("sqs_managed_sse_enabled", BeFalse()),
+					HaveKeyWithValue("kms_master_key_id", "xxxx"),
+					HaveKeyWithValue("kms_data_key_reuse_period_seconds", BeNumerically("==", 86_400)),
 				),
 			)
 		})
@@ -211,6 +230,9 @@ var _ = Describe("SQS", Label("SQS"), func() {
 			Entry(nil, "max_message_size", 1024),
 			Entry(nil, "delay_seconds", 300),
 			Entry(nil, "receive_wait_time_seconds", 15),
+			Entry(nil, "sqs_managed_sse_enabled", false),
+			Entry(nil, "kms_master_key_id", "xxxx"),
+			Entry(nil, "kms_data_key_reuse_period_seconds", 86_400),
 		)
 
 		DescribeTable(
@@ -264,6 +286,11 @@ var _ = Describe("SQS", Label("SQS"), func() {
 					Type:  "string",
 					Value: "arn:aws:sqs::ap-northeast-3::example-dlq",
 				},
+				{
+					Name:  "kms_all_key_ids",
+					Type:  "string",
+					Value: "alias_kms_id1,alias_kms_id2",
+				},
 			})
 			Expect(err).NotTo(HaveOccurred())
 
@@ -282,6 +309,7 @@ var _ = Describe("SQS", Label("SQS"), func() {
 					"queue_name":        "example_name",
 					"queue_url":         "example_url",
 					"dlq_arn":           "arn:aws:sqs::ap-northeast-3::example-dlq",
+					"kms_all_key_ids":   "alias_kms_id1,alias_kms_id2",
 				}),
 			)
 		})

diff --git a/terraform-tests/sqs_test.go b/terraform-tests/sqs_test.go
@@ -30,22 +30,26 @@ var _ = Describe("SQS", Label("SQS-terraform"), Ordered, func() {
 
 	BeforeEach(func() {
 		defaultVars = map[string]any{
-			"instance_name":               name,
-			"fifo":                        false,
-			"visibility_timeout_seconds":  30,
-			"message_retention_seconds":   345600,
-			"max_message_size":            262144,
-			"delay_seconds":               0,
-			"receive_wait_time_seconds":   0,
-			"labels":                      map[string]string{"label1": "value1"},
-			"aws_access_key_id":           awsAccessKeyID,
-			"aws_secret_access_key":       awsSecretAccessKey,
-			"region":                      awsRegion,
-			"dlq_arn":                     "",
-			"max_receive_count":           5,
-			"deduplication_scope":         nil,
-			"fifo_throughput_limit":       nil,
-			"content_based_deduplication": false,
+			"instance_name":                     name,
+			"fifo":                              false,
+			"visibility_timeout_seconds":        30,
+			"message_retention_seconds":         345600,
+			"max_message_size":                  262144,
+			"delay_seconds":                     0,
+			"receive_wait_time_seconds":         0,
+			"labels":                            map[string]string{"label1": "value1"},
+			"aws_access_key_id":                 awsAccessKeyID,
+			"aws_secret_access_key":             awsSecretAccessKey,
+			"region":                            awsRegion,
+			"dlq_arn":                           "",
+			"max_receive_count":                 5,
+			"deduplication_scope":               nil,
+			"fifo_throughput_limit":             nil,
+			"content_based_deduplication":       false,
+			"sqs_managed_sse_enabled":           true,
+			"kms_master_key_id":                 "",
+			"kms_extra_key_ids":                 "",
+			"kms_data_key_reuse_period_seconds": 300,
 		}
 	})
 
@@ -65,13 +69,15 @@ var _ = Describe("SQS", Label("SQS-terraform"), Ordered, func() {
 		It("should create an SQS queue with the correct properties", func() {
 			Expect(AfterValuesForType(plan, "aws_sqs_queue")).To(
 				MatchKeys(IgnoreExtras, Keys{
-					"name":                       Equal(name),
-					"fifo_queue":                 BeFalse(),
-					"visibility_timeout_seconds": BeNumerically("==", 30),
-					"message_retention_seconds":  BeNumerically("==", 345600),
-					"max_message_size":           BeNumerically("==", 262144),
-					"delay_seconds":              BeZero(),
-					"receive_wait_time_seconds":  BeZero(),
+					"name":                              Equal(name),
+					"fifo_queue":                        BeFalse(),
+					"visibility_timeout_seconds":        BeNumerically("==", 30),
+					"message_retention_seconds":         BeNumerically("==", 345600),
+					"max_message_size":                  BeNumerically("==", 262144),
+					"delay_seconds":                     BeZero(),
+					"receive_wait_time_seconds":         BeZero(),
+					"kms_master_key_id":                 BeNil(),
+					"kms_data_key_reuse_period_seconds": BeNumerically("==", 300),
 					"tags_all": MatchAllKeys(Keys{
 						"label1": Equal("value1"),
 					}),
@@ -238,4 +244,54 @@ var _ = Describe("SQS", Label("SQS-terraform"), Ordered, func() {
 			)
 		})
 	})
+
+	Context("with SQS-managed SSE enabled", func() {
+		BeforeAll(func() {
+			plan = ShowPlan(terraformProvisionDir, buildVars(defaultVars, map[string]any{
+				"sqs_managed_sse_enabled": true,
+			}))
+		})
+
+		It("should enable SQS-managed server-side encryption", func() {
+			Expect(AfterValuesForType(plan, "aws_sqs_queue")).To(
+				MatchKeys(IgnoreExtras, Keys{
+					"sqs_managed_sse_enabled": BeTrue(),
+				}),
+			)
+		})
+	})
+
+	Context("with KMS master key specified", func() {
+		BeforeAll(func() {
+			plan = ShowPlan(terraformProvisionDir, buildVars(defaultVars, map[string]any{
+				"kms_master_key_id":                 "alias/aws/sqs",
+				"kms_data_key_reuse_period_seconds": 300,
+				"sqs_managed_sse_enabled":           false,
+			}))
+		})
+
+		It("should use the specified KMS master key for encryption and data key reuse period specified", func() {
+			Expect(AfterValuesForType(plan, "aws_sqs_queue")).To(
+				MatchKeys(IgnoreExtras, Keys{
+					"kms_master_key_id":                 Equal("alias/aws/sqs"),
+					"kms_data_key_reuse_period_seconds": BeNumerically("==", 300),
+				}),
+			)
+		})
+	})
+
+	Context("with KMS master key specified and sse enabled", func() {
+		It("should throw an error", func() {
+			session, _ := FailPlan(terraformProvisionDir, buildVars(defaultVars, map[string]any{
+				"kms_master_key_id":                 "alias/aws/sqs",
+				"kms_data_key_reuse_period_seconds": 300,
+				"sqs_managed_sse_enabled":           true,
+			}))
+
+			Expect(session.ExitCode()).NotTo(Equal(0))
+			msgs := string(session.Out.Contents())
+			Expect(msgs).To(ContainSubstring(`Error: Conflicting configuration arguments`))
+			Expect(msgs).To(ContainSubstring(`\"sqs_managed_sse_enabled\": conflicts with kms_master_key_id`))
+		})
+	})
 })
diff --git a/terraform/sqs/bind/data.tf b/terraform/sqs/bind/data.tf
@@ -1,15 +1,5 @@
-data "aws_iam_policy_document" "user_policy" {
-  dynamic "statement" {
-    for_each = local.queue_policy
-    content {
-      sid       = try(statement.value.sid, null)
-      actions   = try(statement.value.actions, null)
-      resources = try(statement.value.resources, null)
-    }
-  }
-}
-
 locals {
+
   standard_access = {
     sid : "sqsAccess",
     actions : [
@@ -25,6 +15,7 @@ locals {
     ],
     resources : [var.arn]
   }
+
   dql_redrive_access = {
     sid : "sqsAccessDLQ",
     actions : [
@@ -38,5 +29,37 @@ locals {
     resources : [var.dlq_arn]
   }
 
-  queue_policy = length(var.dlq_arn) > 0 ? concat([local.standard_access], [local.dql_redrive_access]) : [local.standard_access]
+  kms_statement = {
+    sid : "kmsAccess",
+    actions : [
+      "kms:GenerateDataKey",
+      "kms:Decrypt"
+    ]
+    resources = [for key in data.aws_kms_key.customer_provided_keys : key.arn]
+  }
+
+  key_ids_list = compact(split(",", var.kms_all_key_ids))
+  has_key_ids  = length(local.key_ids_list) != 0
+
+  queue_policy = concat(
+    [local.standard_access],
+    length(var.dlq_arn) > 0 ? [local.dql_redrive_access] : [],
+    local.has_key_ids ? [local.kms_statement] : []
+  )
+}
+
+data "aws_iam_policy_document" "user_policy" {
+  dynamic "statement" {
+    for_each = local.queue_policy
+    content {
+      sid       = statement.value.sid
+      actions   = statement.value.actions
+      resources = statement.value.resources
+    }
+  }
+}
+
+data "aws_kms_key" "customer_provided_keys" {
+  count  = local.has_key_ids ? length(local.key_ids_list) : 0
+  key_id = local.key_ids_list[count.index]
 }
diff --git a/terraform/sqs/bind/variables.tf b/terraform/sqs/bind/variables.tf
@@ -10,3 +10,4 @@ variable "region" { type = string }
 variable "arn" { type = string }
 variable "user_name" { type = string }
 variable "dlq_arn" { type = string }
+variable "kms_all_key_ids" { type = string }
diff --git a/terraform/sqs/provision/data.tf b/terraform/sqs/provision/data.tf
@@ -0,0 +1,11 @@
+locals {
+  // Convert the default KMS master key ID to a list if it's not empty, or an empty list otherwise.
+  default_kms_key_id_as_list = var.kms_master_key_id != "" ? [var.kms_master_key_id] : []
+
+  // Split the extra KMS key IDs into a list if not empty, or an empty list otherwise.
+  kms_extra_key_ids_as_list = var.kms_extra_key_ids != "" ? split(",", var.kms_extra_key_ids) : []
+
+  // Combine the default and extra KMS key IDs into a single list, ensuring distinct values and removing any empty elements,
+  // then join them into a single comma-separated string.
+  kms_all_key_ids = join(",", compact(distinct(concat(local.default_kms_key_id_as_list, local.kms_extra_key_ids_as_list))))
+}
diff --git a/terraform/sqs/provision/main.tf b/terraform/sqs/provision/main.tf
@@ -16,6 +16,12 @@ resource "aws_sqs_queue" "queue" {
   deduplication_scope         = var.deduplication_scope
   fifo_throughput_limit       = var.fifo_throughput_limit
 
+  # Server-side encryption settings
+  kms_master_key_id                 = var.kms_master_key_id == "" ? null : var.kms_master_key_id
+  kms_data_key_reuse_period_seconds = var.kms_data_key_reuse_period_seconds
+
+  sqs_managed_sse_enabled = !var.sqs_managed_sse_enabled ? null : var.sqs_managed_sse_enabled
+
   lifecycle {
     prevent_destroy = true
   }

diff --git a/terraform/sqs/provision/outputs.tf b/terraform/sqs/provision/outputs.tf
@@ -3,6 +3,7 @@ output "region" { value = var.region }
 output "queue_url" { value = aws_sqs_queue.queue.id }
 output "queue_name" { value = aws_sqs_queue.queue.name }
 output "dlq_arn" { value = var.dlq_arn }
+output "kms_all_key_ids" { value = local.kms_all_key_ids }
 output "status" {
   value = format(
     "created SQS queue: %s (ARN: %s)",