From 2095c3d98aa33078c0f1d19a1ce4f76229b6a73a Mon Sep 17 00:00:00 2001 From: Aditya Ranjan <48973656+Aditya-ranjan-16@users.noreply.github.com> Date: Wed, 5 Feb 2025 19:53:01 +0530 Subject: [PATCH] chore: added missing examples and plugged test gaps (#384) --- .secrets.baseline | 2 +- common-dev-assets | 2 +- examples/backup-restore/README.MD | 6 + .../catalogValidationValues.json.template | 6 + examples/backup-restore/main.tf | 27 ++++ examples/backup-restore/outputs.tf | 12 ++ examples/backup-restore/provider.tf | 4 + examples/backup-restore/variables.tf | 47 ++++++ examples/backup-restore/version.tf | 11 ++ examples/basic/main.tf | 8 +- examples/basic/outputs.tf | 19 ++- examples/basic/provider.tf | 8 +- examples/basic/variables.tf | 17 ++ solutions/standard/main.tf | 2 +- tests/other_test.go | 20 +++ tests/pr_test.go | 145 +++++++++++------- 16 files changed, 262 insertions(+), 74 deletions(-) create mode 100644 examples/backup-restore/README.MD create mode 100644 examples/backup-restore/catalogValidationValues.json.template create mode 100644 examples/backup-restore/main.tf create mode 100644 examples/backup-restore/outputs.tf create mode 100644 examples/backup-restore/provider.tf create mode 100644 examples/backup-restore/variables.tf create mode 100644 examples/backup-restore/version.tf diff --git a/.secrets.baseline b/.secrets.baseline index b704e2cd..35efb212 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "go.sum|^.secrets.baseline$", "lines": null }, - "generated_at": "2025-01-27T10:25:29Z", + "generated_at": "2025-02-05T07:22:24Z", "plugins_used": [ { "name": "AWSKeyDetector" diff --git a/common-dev-assets b/common-dev-assets index ffeb38ae..cd5f6cf3 160000 --- a/common-dev-assets +++ b/common-dev-assets @@ -1 +1 @@ -Subproject commit ffeb38ae284999d23c25416b2f19268ca0ff49b8 +Subproject commit cd5f6cf3583de87d96d2bcf6852d9ad7af7a5193 diff --git a/examples/backup-restore/README.MD b/examples/backup-restore/README.MD new file mode 100644 index 00000000..9d8dc67c --- /dev/null +++ b/examples/backup-restore/README.MD @@ -0,0 +1,6 @@ +# Restore from backup example + +This example provides an end-to-end executable flow of how a Elasticsearch DB instance can be created from a backup instance. This example uses the IBM Cloud terraform provider to: + +- Create a new resource group if one is not passed in. +- Create a restored ICD Elasticsearch database instance pointing to the lastest backup of the existing Elasticsearch database instance crn passed. diff --git a/examples/backup-restore/catalogValidationValues.json.template b/examples/backup-restore/catalogValidationValues.json.template new file mode 100644 index 00000000..d761afae --- /dev/null +++ b/examples/backup-restore/catalogValidationValues.json.template @@ -0,0 +1,6 @@ +{ + "ibmcloud_api_key": $VALIDATION_APIKEY, + "region": "us-south", + "resource_tags": $TAGS, + "prefix": $PREFIX +} diff --git a/examples/backup-restore/main.tf b/examples/backup-restore/main.tf new file mode 100644 index 00000000..1ee9d2e5 --- /dev/null +++ b/examples/backup-restore/main.tf @@ -0,0 +1,27 @@ +############################################################################## +# Resource Group +############################################################################## + +module "resource_group" { + source = "terraform-ibm-modules/resource-group/ibm" + version = "1.1.6" + # if an existing resource group is not set (null) create a new one using prefix + resource_group_name = var.resource_group == null ? "${var.prefix}-resource-group" : null + existing_resource_group_name = var.resource_group +} + +data "ibm_database_backups" "backup_database" { + deployment_id = var.existing_database_crn +} +# New elasticsearch instance pointing to the backup instance +module "restored_icd_elasticsearch" { + source = "../.." + resource_group_id = module.resource_group.resource_group_id + name = "${var.prefix}-elasticsearch-restored" + elasticsearch_version = var.elasticsearch_version + region = var.region + tags = var.resource_tags + access_tags = var.access_tags + member_host_flavor = "multitenant" + backup_crn = data.ibm_database_backups.backup_database.backups[0].backup_id +} diff --git a/examples/backup-restore/outputs.tf b/examples/backup-restore/outputs.tf new file mode 100644 index 00000000..5ef993a0 --- /dev/null +++ b/examples/backup-restore/outputs.tf @@ -0,0 +1,12 @@ +############################################################################## +# Outputs +############################################################################## +output "restored_icd_elasticsearch_id" { + description = "Restored elasticsearch instance id" + value = module.restored_icd_elasticsearch.id +} + +output "restored_icd_elasticsearch_version" { + description = "Restored elasticsearch instance version" + value = module.restored_icd_elasticsearch.version +} diff --git a/examples/backup-restore/provider.tf b/examples/backup-restore/provider.tf new file mode 100644 index 00000000..df45ef50 --- /dev/null +++ b/examples/backup-restore/provider.tf @@ -0,0 +1,4 @@ +provider "ibm" { + ibmcloud_api_key = var.ibmcloud_api_key + region = var.region +} diff --git a/examples/backup-restore/variables.tf b/examples/backup-restore/variables.tf new file mode 100644 index 00000000..ea6b96ac --- /dev/null +++ b/examples/backup-restore/variables.tf @@ -0,0 +1,47 @@ +variable "ibmcloud_api_key" { + type = string + description = "The IBM Cloud API Key" + sensitive = true +} + +variable "region" { + type = string + description = "Region to provision all resources created by this example." + default = "us-south" +} + +variable "prefix" { + type = string + description = "Prefix to append to all resources created by this example" + default = "backup" +} + +variable "elasticsearch_version" { + type = string + description = "Version of the elasticsearch instance. If no value passed, the current ICD preferred version is used." + default = null +} + +variable "resource_group" { + type = string + description = "An existing resource group name to use for this example, if unset a new resource group will be created" + default = null +} + +variable "resource_tags" { + type = list(string) + description = "Optional list of tags to be added to created resources" + default = [] +} + +variable "access_tags" { + type = list(string) + description = "A list of access tags to apply to the elasticsearch instance created by the module, see https://cloud.ibm.com/docs/account?topic=account-access-tags-tutorial for more details" + default = [] +} + +variable "existing_database_crn" { + type = string + description = "The existing CRN of a backup resource to restore from." + default = null +} diff --git a/examples/backup-restore/version.tf b/examples/backup-restore/version.tf new file mode 100644 index 00000000..45c8d81f --- /dev/null +++ b/examples/backup-restore/version.tf @@ -0,0 +1,11 @@ +terraform { + required_version = ">= 1.3.0" + required_providers { + # Ensure that there is always 1 example locked into the lowest provider version of the range defined in the main + # module's version.tf (basic example), and 1 example that will always use the latest provider version (complete example). + ibm = { + source = "IBM-Cloud/ibm" + version = ">=1.70.0, <2.0.0" + } + } +} diff --git a/examples/basic/main.tf b/examples/basic/main.tf index b77c2b42..254e8117 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -13,14 +13,16 @@ module "resource_group" { # Elasticsearch Instance ############################################################################## -module "icd_elasticsearch" { +module "database" { source = "../../" resource_group_id = module.resource_group.resource_group_id - name = "${var.prefix}-elasticsearch" + name = "${var.prefix}-data-store" region = var.region elasticsearch_version = var.elasticsearch_version tags = var.resource_tags access_tags = var.access_tags + service_endpoints = var.service_endpoints + member_host_flavor = var.member_host_flavor service_credential_names = { "elasticsearch_admin" : "Administrator", "elasticsearch_operator" : "Operator", @@ -32,7 +34,7 @@ module "icd_elasticsearch" { # wait 60 secs to allow IAM credential access to kick in before configuring instance # without the wait, you can intermittently get "Error 401 (Unauthorized)" resource "time_sleep" "wait" { - depends_on = [module.icd_elasticsearch] + depends_on = [module.database] create_duration = "60s" } diff --git a/examples/basic/outputs.tf b/examples/basic/outputs.tf index 7da7161d..238fc39a 100644 --- a/examples/basic/outputs.tf +++ b/examples/basic/outputs.tf @@ -2,32 +2,37 @@ # Outputs ############################################################################## output "id" { - description = "Elasticsearch id" - value = module.icd_elasticsearch.id + description = "Database instance id" + value = module.database.id +} + +output "elasticsearch_crn" { + description = "Elasticsearch CRN" + value = module.database.crn } output "version" { description = "Enterprise DB instance version" - value = module.icd_elasticsearch.version + value = module.database.version } output "adminuser" { description = "Database admin user name" - value = module.icd_elasticsearch.adminuser + value = module.database.adminuser } output "hostname" { description = "Database connection hostname" - value = module.icd_elasticsearch.hostname + value = module.database.hostname } output "port" { description = "Database connection port" - value = module.icd_elasticsearch.port + value = module.database.port } output "certificate_base64" { description = "Database connection certificate" - value = module.icd_elasticsearch.certificate_base64 + value = module.database.certificate_base64 sensitive = true } diff --git a/examples/basic/provider.tf b/examples/basic/provider.tf index 354c7637..4decd8db 100644 --- a/examples/basic/provider.tf +++ b/examples/basic/provider.tf @@ -4,8 +4,8 @@ provider "ibm" { } provider "elasticsearch" { - username = module.icd_elasticsearch.service_credentials_object.credentials["elasticsearch_admin"].username - password = module.icd_elasticsearch.service_credentials_object.credentials["elasticsearch_admin"].password - url = "https://${module.icd_elasticsearch.service_credentials_object.hostname}:${module.icd_elasticsearch.service_credentials_object.port}" - cacert_file = base64decode(module.icd_elasticsearch.service_credentials_object.certificate) + username = module.database.service_credentials_object.credentials["elasticsearch_admin"].username + password = module.database.service_credentials_object.credentials["elasticsearch_admin"].password + url = "https://${module.database.service_credentials_object.hostname}:${module.database.service_credentials_object.port}" + cacert_file = base64decode(module.database.service_credentials_object.certificate) } diff --git a/examples/basic/variables.tf b/examples/basic/variables.tf index dd829012..18b25c7e 100644 --- a/examples/basic/variables.tf +++ b/examples/basic/variables.tf @@ -39,3 +39,20 @@ variable "resource_tags" { description = "Optional list of tags to be added to created resources" default = [] } + +variable "service_endpoints" { + type = string + description = "The type of endpoint of the database instance. Possible values: `public`, `private`, `public-and-private`." + default = "public" + + validation { + condition = can(regex("public|public-and-private|private", var.service_endpoints)) + error_message = "Valid values for service_endpoints are 'public', 'public-and-private', and 'private'" + } +} +variable "member_host_flavor" { + type = string + description = "The host flavor per member. [Learn more](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/database#host_flavor)." + default = "multitenant" + # Validation is done in the Terraform plan phase by the IBM provider, so no need to add extra validation here. +} diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index 1cb42003..594fc089 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -254,7 +254,7 @@ module "es_instance_crn_parser" { # Existing instance local vars locals { - existing_elasticsearch_guid = var.existing_db_instance_crn != null ? module.es_instance_crn_parser[0].guid : null + existing_elasticsearch_guid = var.existing_db_instance_crn != null ? module.es_instance_crn_parser[0].service_instance : null existing_elasticsearch_region = var.existing_db_instance_crn != null ? module.es_instance_crn_parser[0].region : null # Validate the region input matches region detected in existing instance CRN (approach based on https://github.com/hashicorp/terraform/issues/25609#issuecomment-1057614400) diff --git a/tests/other_test.go b/tests/other_test.go index cb0f8e07..e80425a9 100644 --- a/tests/other_test.go +++ b/tests/other_test.go @@ -4,6 +4,7 @@ package test import ( "crypto/rand" "encoding/base64" + "fmt" "log" "testing" @@ -83,3 +84,22 @@ func TestPlanICDVersions(t *testing.T) { t.Run(version, func(t *testing.T) { testPlanICDVersions(t, version) }) } } +func TestRunRestoredDBExample(t *testing.T) { + t.Parallel() + + options := testhelper.TestOptionsDefaultWithVars(&testhelper.TestOptions{ + Testing: t, + TerraformDir: "examples/backup-restore", + Prefix: "elastic-restored", + Region: fmt.Sprint(permanentResources["elasticsearchRegion"]), + ResourceGroup: resourceGroup, + TerraformVars: map[string]interface{}{ + "existing_database_crn": permanentResources["elasticsearchCrn"], + }, + CloudInfoService: sharedInfoSvc, + }) + + output, err := options.RunTestConsistency() + assert.Nil(t, err, "This should not have errored") + assert.NotNil(t, output, "Expected some output") +} diff --git a/tests/pr_test.go b/tests/pr_test.go index c90d2007..7e201953 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -3,16 +3,21 @@ package test import ( "fmt" - "log" - "os" - "testing" - + "github.com/gruntwork-io/terratest/modules/files" + "github.com/gruntwork-io/terratest/modules/logger" + "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/terraform" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/cloudinfo" "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/common" "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/testhelper" "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/testschematic" + "log" + "math/rand" + "os" + "strings" + "testing" ) const completeExampleTerraformDir = "examples/complete" @@ -32,6 +37,10 @@ const yamlLocation = "../common-dev-assets/common-go-assets/common-permanent-res var permanentResources map[string]interface{} var sharedInfoSvc *cloudinfo.CloudInfoService +var validICDRegions = []string{ + "eu-de", + "us-south", +} // TestMain will be run before any parallel tests, used to read data from yaml for use with tests func TestMain(m *testing.M) { @@ -46,40 +55,6 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestRunFSCloudExample(t *testing.T) { - t.Parallel() - options := testhelper.TestOptionsDefaultWithVars(&testhelper.TestOptions{ - Testing: t, - TerraformDir: fscloudExampleTerraformDir, - Prefix: "es-fs-test", - Region: "us-south", - /* - Comment out the 'ResourceGroup' input to force this test to create a unique resource group to ensure tests do - not clash. This is due to the fact that an auth policy may already exist in this resource group since we are - re-using a permanent HPCS instance. By using a new resource group, the auth policy will not already exist - since this module scopes auth policies by resource group. - */ - //ResourceGroup: resourceGroup, - TerraformVars: map[string]interface{}{ - "elasticsearch_version": latestVersion, // Always lock this test into the latest supported elasticsearch version - "access_tags": permanentResources["accessTags"], - "kms_key_crn": permanentResources["hpcs_south_root_key_crn"], - }, - CloudInfoService: sharedInfoSvc, - }) - options.SkipTestTearDown = true - output, err := options.RunTestConsistency() - assert.Nil(t, err, "This should not have errored") - assert.NotNil(t, output, "Expected some output") - - // check if outputs exist - outputs := terraform.OutputAll(options.Testing, options.TerraformOptions) - expectedOutputs := []string{"port", "hostname"} - _, outputErr := testhelper.ValidateTerraformOutputs(outputs, expectedOutputs...) - assert.NoErrorf(t, outputErr, "Some outputs not found or nil") - options.TestTearDown() -} - func TestRunStandardSolutionSchematics(t *testing.T) { t.Parallel() @@ -142,11 +117,12 @@ func TestRunStandardUpgradeSolution(t *testing.T) { t.Parallel() options := testhelper.TestOptionsDefault(&testhelper.TestOptions{ - Testing: t, - TerraformDir: standardSolutionTerraformDir, - BestRegionYAMLPath: regionSelectionPath, - Prefix: "els-st-da-upg", - ResourceGroup: resourceGroup, + Testing: t, + TerraformDir: standardSolutionTerraformDir, + BestRegionYAMLPath: regionSelectionPath, + Prefix: "els-st-da-upg", + ResourceGroup: resourceGroup, + CheckApplyResultForUpgrade: true, }) options.TerraformVars = map[string]interface{}{ @@ -168,24 +144,79 @@ func TestRunStandardUpgradeSolution(t *testing.T) { } } -func TestRunBasicExample(t *testing.T) { +func TestRunExistingInstance(t *testing.T) { t.Parallel() - options := testhelper.TestOptionsDefaultWithVars(&testhelper.TestOptions{ - Testing: t, - TerraformDir: "examples/basic", - Prefix: "es-test", - ResourceGroup: resourceGroup, - BestRegionYAMLPath: regionSelectionPath, - CloudInfoService: sharedInfoSvc, - - TerraformVars: map[string]interface{}{ - "elasticsearch_version": latestVersion, // Always lock this test into the latest supported elasticsearch version + prefix := fmt.Sprintf("elastic-t-%s", strings.ToLower(random.UniqueId())) + realTerraformDir := ".." + tempTerraformDir, _ := files.CopyTerraformFolderToTemp(realTerraformDir, fmt.Sprintf(prefix+"-%s", strings.ToLower(random.UniqueId()))) + region := validICDRegions[rand.Intn(len(validICDRegions))] + + // Verify ibmcloud_api_key variable is set + checkVariable := "TF_VAR_ibmcloud_api_key" + val, present := os.LookupEnv(checkVariable) + require.True(t, present, checkVariable+" environment variable not set") + require.NotEqual(t, "", val, checkVariable+" environment variable is empty") + + logger.Log(t, "Tempdir: ", tempTerraformDir) + existingTerraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ + TerraformDir: tempTerraformDir + "/examples/basic", + Vars: map[string]interface{}{ + "prefix": prefix, + "region": region, + "elasticsearch_version": latestVersion, + "service_endpoints": "public-and-private", }, + // Set Upgrade to true to ensure latest version of providers and modules are used by terratest. + // This is the same as setting the -upgrade=true flag with terraform. + Upgrade: true, }) - output, err := options.RunTestConsistency() - assert.Nil(t, err, "This should not have errored") - assert.NotNil(t, output, "Expected some output") + terraform.WorkspaceSelectOrNew(t, existingTerraformOptions, prefix) + _, existErr := terraform.InitAndApplyE(t, existingTerraformOptions) + if existErr != nil { + assert.True(t, existErr == nil, "Init and Apply of temp existing resource failed") + } else { + logger.Log(t, " existing_db_instance_crn: ", terraform.Output(t, existingTerraformOptions, "elasticsearch_crn")) + options := testschematic.TestSchematicOptionsDefault(&testschematic.TestSchematicOptions{ + Testing: t, + TarIncludePatterns: []string{ + "*.tf", + fmt.Sprintf("%s/*.tf", standardSolutionTerraformDir), + fmt.Sprintf("%s/*.tf", fscloudExampleTerraformDir), + fmt.Sprintf("%s/*.tf", "modules/fscloud"), + fmt.Sprintf("%s/*.sh", "scripts"), + }, + TemplateFolder: standardSolutionTerraformDir, + BestRegionYAMLPath: regionSelectionPath, + Prefix: "els-sr-da", + ResourceGroup: resourceGroup, + DeleteWorkspaceOnFail: false, + WaitJobCompleteMinutes: 60, + }) + + options.TerraformVars = []testschematic.TestSchematicTerraformVar{ + {Name: "ibmcloud_api_key", Value: options.RequiredEnvironmentVars["TF_VAR_ibmcloud_api_key"], DataType: "string", Secure: true}, + {Name: "existing_db_instance_crn", Value: terraform.Output(t, existingTerraformOptions, "elasticsearch_crn"), DataType: "string"}, + {Name: "resource_group_name", Value: fmt.Sprintf("%s-resource-group", prefix), DataType: "string"}, + {Name: "region", Value: region, DataType: "string"}, + {Name: "use_existing_resource_group", Value: true, DataType: "bool"}, + {Name: "provider_visibility", Value: "public", DataType: "string"}, + } + err := options.RunSchematicTest() + assert.Nil(t, err, "This should not have errored") + + } + envVal, _ := os.LookupEnv("DO_NOT_DESTROY_ON_FAILURE") + // Destroy the temporary existing resources if required + if t.Failed() && strings.ToLower(envVal) == "true" { + fmt.Println("Terratest failed. Debug the test and delete resources manually.") + } else { + logger.Log(t, "START: Destroy (existing resources)") + terraform.Destroy(t, existingTerraformOptions) + terraform.WorkspaceDelete(t, existingTerraformOptions, prefix) + logger.Log(t, "END: Destroy (existing resources)") + } + } // Test the DA when using IBM owned encryption keys