Merge branch 'main' into dependabot/go_modules/github.com/hashicorp/g…

…o-getter-1.7.5

.github/workflows/release-please.yml

@@ -9,7 +9,7 @@ jobs:
     steps:
       - name: Generate token
         id: generate_token
-        uses: chanzuckerberg/[email protected]
+        uses: tibdex/github-app-token@v2@v1.1.4
         with:
           app_id: ${{ secrets.CZI_RELEASE_PLEASE_APP_ID }}
           private_key: ${{ secrets.CZI_RELEASE_PLEASE_PK }}

CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## [0.76.0](https://github.com/chanzuckerberg/cztack/compare/v0.75.0...v0.76.0) (2024-07-24)
+
+
+### Features
+
+* add condition_operator field to allow overriding StringEquals with StringLike ([#610](https://github.com/chanzuckerberg/cztack/issues/610)) ([86bab99](https://github.com/chanzuckerberg/cztack/commit/86bab991235dba69305970153f88d650dfa65802))
+
+## [0.75.0](https://github.com/chanzuckerberg/cztack/compare/v0.74.0...v0.75.0) (2024-07-24)
+
+
+### Features
+
+* add jwt_condition option to oidc variable for IAM trust relationships ([#608](https://github.com/chanzuckerberg/cztack/issues/608)) ([5320504](https://github.com/chanzuckerberg/cztack/commit/5320504677c9fda2447b61229ed1831b24f36074))
+
+## [0.74.0](https://github.com/chanzuckerberg/cztack/compare/v0.73.3...v0.74.0) (2024-07-18)
+
+
+### Features
+
+* CDI-3149 - Allow use of existing catalogs and schemas for databricks volumes ([#606](https://github.com/chanzuckerberg/cztack/issues/606)) ([7735218](https://github.com/chanzuckerberg/cztack/commit/7735218680130e8fc94d00f392c9537f45b6c1e8))
+
 ## [0.73.3](https://github.com/chanzuckerberg/cztack/compare/v0.73.2...v0.73.3) (2024-07-03)
 
 

aws-assume-role-policy/main.tf

@@ -52,8 +52,8 @@ data "aws_iam_policy_document" "assume-role" {
 
       actions = ["sts:AssumeRoleWithWebIdentity", "sts:TagSession"]
       condition {
-        test     = "StringEquals"
-        variable = "${oidc.value["provider"]}:aud"
+        test     = "${oidc.value["condition_operator"]}"
+        variable = "${oidc.value["provider"]}:${oidc.value["jwt_condition"]}"
         values   = oidc.value["client_ids"]
       }
     }

aws-assume-role-policy/variables.tf

@@ -38,9 +38,11 @@ variable "saml_idp_arns" {
 variable "oidc" {
   type = list(object(
     {
-      idp_arn : string,          # the AWS IAM IDP arn
-      client_ids : list(string), # a list of oidc client ids
-      provider : string          # your provider url, such as foo.okta.com
+      idp_arn : string,                                      # the AWS IAM IDP arn
+      client_ids : list(string),                             # a list of oidc client ids
+      provider : string,                                     # your provider url, such as foo.okta.com
+      jwt_condition : optional(string, "aud"),               # the condition to allow the JWT token
+      condition_operator : optional(string, "StringEquals"), # the condition operator for the iam statement
     }
   ))
   default     = []

aws-assume-role-policy/versions.tf

@@ -0,0 +1,3 @@
+terraform {
+  required_version = ">= 1.3.0"
+}

aws-iam-role-crossacct/variables.tf

@@ -29,9 +29,11 @@ variable "source_role_arns" {
 variable "oidc" {
   type = list(object(
     {
-      idp_arn : string,          # the AWS IAM IDP arn
-      client_ids : list(string), # a list of oidc client ids
-      provider : string          # your provider url, such as foo.okta.com
+      idp_arn : string,                      # the AWS IAM IDP arn
+      client_ids : list(string),             # a list of oidc client ids
+      provider : string,                     # your provider url, such as foo.okta.com
+      jwt_condition : optional(string),      # the condition to allow the JWT token
+      condition_operator : optional(string), # the condition operator for the iam statement
     }
   ))
 

aws-iam-role-crossacct/versions.tf

@@ -0,0 +1,3 @@
+terraform {
+  required_version = ">= 1.3.0"
+}

databricks-s3-volume/README.md

@@ -45,13 +45,16 @@
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_additional_rw_bucket_grant_arns"></a> [additional\_rw\_bucket\_grant\_arns](#input\_additional\_rw\_bucket\_grant\_arns) | (Optional) Additional AWS ARNs to grant read/write permissions to on the bucket (may be necessary for service principals, instance profiles, or users | `list(string)` | `[]` | no |
+| <a name="input_additional_rw_bucket_grant_arns"></a> [additional\_rw\_bucket\_grant\_arns](#input\_additional\_rw\_bucket\_grant\_arns) | (Optional) Additional AWS ARNs to grant read/write permissions to on the bucket (may be necessary for service principals, instance profiles, or users) | `list(string)` | `[]` | no |
 | <a name="input_bucket_object_ownership"></a> [bucket\_object\_ownership](#input\_bucket\_object\_ownership) | Set default owner of all objects within bucket (e.g., bucket vs. object owner) | `string` | `null` | no |
 | <a name="input_catalog_name"></a> [catalog\_name](#input\_catalog\_name) | Name of the Databricks existing catalog to add the volume to | `string` | n/a | yes |
 | <a name="input_catalog_owner"></a> [catalog\_owner](#input\_catalog\_owner) | User or group name of the catalog owner | `string` | n/a | yes |
+| <a name="input_create_catalog"></a> [create\_catalog](#input\_create\_catalog) | Flag to create a new catalog or look for an existing one with the given name | `bool` | n/a | yes |
 | <a name="input_catalog_r_grant_principals"></a> [catalog\_r\_grant\_principals](#input\_catalog\_r\_grant\_principals) | (Optional) Databricks groups to grant read-only permissions to on the catalog | `list(string)` | `[]` | no |
 | <a name="input_catalog_rw_grant_principals"></a> [catalog\_rw\_grant\_principals](#input\_catalog\_rw\_grant\_principals) | (Optional) Databricks groups to grant read/write permissions to on the catalog | `list(string)` | `[]` | no |
 | <a name="input_metastore_id"></a> [metastore\_id](#input\_metastore\_id) | ID of metastore to create catalog in | `string` | n/a | yes |
+| <a name="input_create_schema"></a> [create\_schema](#input\_create\_schema) | Flag to create a new schema or look for an existing one with the given name | `bool` | n/a | yes |
+| <a name="input_schema_name"></a> [schema\_name](#input\_schema\_name) | Name of the Databricks existing schema to add the volume to | `string` | n/a | yes |
 | <a name="input_schema_r_grant_principals"></a> [schema\_r\_grant\_principals](#input\_schema\_r\_grant\_principals) | (Optional) Databricks groups to grant read-only permissions to on the schema | `list(string)` | `[]` | no |
 | <a name="input_schema_rw_grant_principals"></a> [schema\_rw\_grant\_principals](#input\_schema\_rw\_grant\_principals) | (Optional) Databricks groups to grant read/write permissions to on the schema | `list(string)` | `[]` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | REQUIRED: Tags to include for this environment. | <pre>object({<br>    project : string<br>    env : string<br>    service : string<br>    owner : string<br>    managedBy : string<br>  })</pre> | n/a | yes |

databricks-s3-volume/grants.tf

@@ -1,14 +1,26 @@
+locals {
+  # Only set the grant principals if the catalog and/or schema doesn't already exist
+  catalog_r_grant_principals = var.create_catalog ? var.catalog_r_grant_principals : []
+  catalog_rw_grant_principals = var.create_catalog ? var.catalog_rw_grant_principals : []
+  schema_r_grant_principals = var.create_schema ? var.schema_r_grant_principals : []
+  schema_rw_grant_principals = var.create_schema ? var.schema_rw_grant_principals : []
+}
+
 # catalog
 resource "databricks_grant" "catalog_r" {
-  for_each   = toset(var.catalog_r_grant_principals)
-  catalog    = databricks_catalog.volume.name
+  depends_on = [databricks_catalog.volume[0]]
+  for_each   = toset(local.catalog_r_grant_principals)
+
+  catalog    = local.catalog_name
   principal  = each.value
   privileges = ["USE_CATALOG", "USE_SCHEMA", "SELECT"]
 }
 
 resource "databricks_grant" "catalog_rw" {
-  for_each  = toset(var.catalog_rw_grant_principals)
-  catalog   = databricks_catalog.volume.name
+  depends_on = [databricks_catalog.volume[0]]
+  for_each  = toset(local.catalog_rw_grant_principals)
+
+  catalog   = local.catalog_name
   principal = "Data Scientists"
   privileges = [
     "APPLY_TAG",
@@ -29,15 +41,19 @@ resource "databricks_grant" "catalog_rw" {
 
 # schema
 resource "databricks_grant" "schema_r" {
-  for_each   = toset(var.schema_r_grant_principals)
-  schema     = databricks_schema.volume.id
+  depends_on = [databricks_schema.volume[0]]
+  for_each   = toset(local.schema_r_grant_principals)
+
+  schema     = "${local.catalog_name}.${local.schema_name}"
   principal  = each.value
   privileges = ["USE_SCHEMA", "SELECT", "READ_VOLUME"]
 }
 
 resource "databricks_grant" "schema_rw" {
-  for_each  = toset(var.schema_rw_grant_principals)
-  schema    = databricks_schema.volume.id
+  depends_on = [databricks_schema.volume[0]]
+  for_each  = toset(local.schema_rw_grant_principals)
+
+  schema    = "${local.catalog_name}.${local.schema_name}"
   principal = each.value
   privileges = [
     "APPLY_TAG",

databricks-s3-volume/iam.tf

@@ -5,6 +5,8 @@ data "aws_caller_identity" "current" {
 }
 
 data "aws_iam_policy_document" "dbx_unity_aws_role_assume_role" {
+  count = var.create_catalog ? 1 : 0
+
   statement {
     principals {
       type        = "AWS"
@@ -35,13 +37,17 @@ data "aws_iam_policy_document" "dbx_unity_aws_role_assume_role" {
 }
 
 resource "aws_iam_role" "dbx_unity_aws_role" {
+  count = var.create_catalog ? 1 : 0
+
   name               = local.unity_aws_role_name
   path               = local.path
-  assume_role_policy = data.aws_iam_policy_document.dbx_unity_aws_role_assume_role.json
+  assume_role_policy = data.aws_iam_policy_document.dbx_unity_aws_role_assume_role[0].json
 }
 
 ### Policy document to access default volume bucket and assume role
 data "aws_iam_policy_document" "volume_bucket_dbx_unity_access" {
+  count = var.create_catalog ? 1 : 0
+
   depends_on = [
     module.databricks_bucket
   ]
@@ -84,10 +90,14 @@ data "aws_iam_policy_document" "volume_bucket_dbx_unity_access" {
 }
 
 resource "aws_iam_policy" "dbx_unity_access_policy" {
-  policy = data.aws_iam_policy_document.volume_bucket_dbx_unity_access.json
+  count = var.create_catalog ? 1 : 0
+
+  policy = data.aws_iam_policy_document.volume_bucket_dbx_unity_access[0].json
 }
 
 resource "aws_iam_role_policy_attachment" "dbx_unity_aws_access" {
-  policy_arn = aws_iam_policy.dbx_unity_access_policy.arn
-  role       = aws_iam_role.dbx_unity_aws_role.name
+  count = var.create_catalog ? 1 : 0
+
+  policy_arn = aws_iam_policy.dbx_unity_access_policy[0].arn
+  role       = aws_iam_role.dbx_unity_aws_role[0].name
 }

databricks-s3-volume/main.tf

@@ -3,8 +3,11 @@
 // https://docs.databricks.com/administration-guide/multiworkspace/iam-role.html#language-Your%C2%A0VPC,%C2%A0custom
 locals {
   unity_aws_role_name = "${var.catalog_name}-unity"
-  catalog_name        = replace(var.catalog_name, "-", "_") # SQL don't work with hyphens
-  schema_name         = replace(var.volume_name, "-", "_")  # SQL don't work with hyphens
+
+  # Create non-hyphenated versions of the catalog and schema names if catalog and/or schema doesnt exist. Else, use the provided names
+  catalog_name        = var.create_catalog ? replace(var.catalog_name, "-", "_") : var.catalog_name
+  schema_name         = var.create_schema ? replace(var.schema_name, "-", "_") : var.schema_name
+  volume_name         = replace(var.volume_name, "-", "_")
 
   path                   = "/databricks/"
   databricks_aws_account = "414351767826"                                                                      # Databricks' own AWS account, not CZI's. See https://docs.databricks.com/en/administration-guide/account-settings-e2/credentials.html#step-1-create-a-cross-account-iam-role
@@ -16,6 +19,8 @@ locals {
 ### NOTE: 
 
 resource "databricks_storage_credential" "volume" {
+  count = var.create_catalog ? 1 : 0
+
   depends_on = [
     resource.aws_iam_role.dbx_unity_aws_role,
     resource.aws_iam_role_policy_attachment.dbx_unity_aws_access,
@@ -24,31 +29,34 @@ resource "databricks_storage_credential" "volume" {
 
   name = local.catalog_name
   aws_iam_role {
-    role_arn = aws_iam_role.dbx_unity_aws_role.arn
+    role_arn = aws_iam_role.dbx_unity_aws_role[0].arn
   }
   comment = "Managed by Terraform - access for ${var.catalog_name}"
 }
 
 # upstream external location sometimes takes a moment to register
 resource "time_sleep" "wait_30_seconds" {
-  depends_on = [databricks_storage_credential.volume]
+  depends_on = [databricks_storage_credential.volume[0]]
 
   create_duration = "30s"
 }
 
 resource "databricks_external_location" "volume" {
+  count = var.create_catalog ? 1 : 0
   depends_on      = [time_sleep.wait_30_seconds]
 
   name            = local.catalog_name
   url             = "s3://${local.bucket_name}"
-  credential_name = databricks_storage_credential.volume.name
+  credential_name = databricks_storage_credential.volume[0].name
   comment         = "Managed by Terraform - access for ${var.catalog_name}"
 }
 
 # New catalog, schema, and volume
 
 resource "databricks_catalog" "volume" {
-  depends_on   = [databricks_external_location.volume]
+  count = var.create_catalog ? 1 : 0
+
+  depends_on   = [databricks_external_location.volume[0]]
   name         = local.catalog_name
   metastore_id = var.metastore_id
   owner        = var.catalog_owner
@@ -61,18 +69,21 @@ resource "databricks_catalog" "volume" {
 }
 
 resource "databricks_schema" "volume" {
-  catalog_name = databricks_catalog.volume.name
+  count = var.create_schema ? 1 : 0
+
+  depends_on   = [databricks_catalog.volume]
+  catalog_name = local.catalog_name
   name         = local.schema_name
   comment      = "This schema is managed by Terraform - ${var.volume_comment}"
   owner        = var.catalog_owner
   properties   = var.volume_schema_properties
 }
 
 resource "databricks_volume" "volume" {
-  depends_on       = [databricks_external_location.volume]
-  name             = "${local.catalog_name}_${local.schema_name}"
+  depends_on       = [databricks_external_location.volume, databricks_schema.volume]
+  name             = "${local.volume_name}"
   catalog_name     = local.catalog_name
-  schema_name      = databricks_schema.volume.name
+  schema_name      = local.schema_name
   volume_type      = "EXTERNAL"
   storage_location = "s3://${local.bucket_name}/${local.schema_name}"
   owner            = var.catalog_owner

databricks-s3-volume/outputs.tf

@@ -1,11 +1,11 @@
 output "dbx_unity_aws_role_arn" {
-  value = aws_iam_role.dbx_unity_aws_role.arn
+  value = length(aws_iam_role.dbx_unity_aws_role) > 0 ? aws_iam_role.dbx_unity_aws_role[0].arn : null
 }
 
 output "volume_specific_bucket_name" {
   value = length(module.databricks_bucket) > 0 ? module.databricks_bucket[0].name : null
 }
 
 output "volume_path" {
-  value = "${databricks_catalog.volume.name}.${databricks_schema.volume.name}.${databricks_volume.volume.name}"
+  value = "${local.catalog_name}.${local.schema_name}.${local.volume_name}"
 }

databricks-s3-volume/variables.tf

@@ -14,11 +14,26 @@ variable "catalog_owner" {
   type        = string
 }
 
+variable "create_catalog" {
+  description = "Flag to create a new catalog or look for an existing one with the given name"
+  type        = bool
+}
+
 variable "metastore_id" {
   description = "ID of metastore to create catalog in"
   type        = string
 }
 
+variable "schema_name" {
+  description = "Name of the Databricks schema to add the volume to"
+  type        = string
+}
+
+variable "create_schema" {
+  description = "Flag to create a new catalog or look for an existing one with the given name"
+  type        = bool
+}
+
 variable "volume_name" {
   description = "Name of the Databricks volume to create"
   type        = string

version.txt

@@ -1 +1 @@
-0.73.3
+0.76.0