diff --git a/Makefile b/Makefile index 1ca84d7..7fa36d0 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,30 @@ -.PHONY: test docs fmt validate install-tools - -export EXAMPLE +.PHONY: all install-tools validate fmt docs test test-parallel test-sequential all: install-tools validate fmt docs install-tools: - @go install github.com/terraform-docs/terraform-docs@latest + go install github.com/terraform-docs/terraform-docs@latest + +TEST_ARGS := $(if $(skip-destroy),-skip-destroy=$(skip-destroy)) \ + $(if $(exception),-exception=$(exception)) \ + $(if $(example),-example=$(example)) test: - cd tests && go test -v -timeout 60m -run TestApplyNoError/$(EXAMPLE) ./deploy_test.go + cd tests && go test -v -timeout 60m -run '^TestApplyNoError$$' -args $(TEST_ARGS) . + +test-sequential: + cd tests && go test -v -timeout 120m -run '^TestApplyAllSequential$$' -args $(TEST_ARGS) . + +test-parallel: + cd tests && go test -v -timeout 60m -run '^TestApplyAllParallel$$' -args $(TEST_ARGS) . docs: @echo "Generating documentation for root and modules..." - @BASE_DIR=$$(pwd); \ - terraform-docs markdown . --output-file $$BASE_DIR/README.md --output-mode inject --hide modules; \ - for dir in $$BASE_DIR/modules/*; do \ + terraform-docs markdown . --output-file README.md --output-mode inject --hide modules + for dir in modules/*; do \ if [ -d "$$dir" ]; then \ echo "Processing $$dir..."; \ - terraform-docs markdown $$dir --output-file $$dir/README.md --output-mode inject --hide modules || echo "Skipped: $$dir"; \ + terraform-docs markdown "$$dir" --output-file "$$dir/README.md" --output-mode inject --hide modules || echo "Skipped: $$dir"; \ fi \ done @@ -28,7 +35,4 @@ validate: terraform init -backend=false terraform validate @echo "Cleaning up initialization files..." - @rm -rf .terraform - @rm -f terraform.tfstate - @rm -f terraform.tfstate.backup - @rm -f .terraform.lock.hcl + rm -rf .terraform terraform.tfstate terraform.tfstate.backup .terraform.lock.hcl diff --git a/README.md b/README.md index 1fee747..75a3e68 100644 --- a/README.md +++ b/README.md @@ -84,13 +84,7 @@ End-to-end testing is not conducted on these modules, as they are individual com ## Testing -Ensure go and terraform are installed. - -Run tests for different usage scenarios by specifying the EXAMPLE environment variable. Usage examples are in the examples directory. - -To execute a test, run `make test EXAMPLE=default` - -Replace default with the specific example you want to test. These tests ensure the module performs reliably across various configurations. +For more information, please see our testing [guidelines](./TESTING.md) ## Notes diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..2b7dd7d --- /dev/null +++ b/TESTING.md @@ -0,0 +1,13 @@ +## Testing + +Ensure Go and Terraform are installed. + +Run tests for different scenarios by setting the example flag when running the tests. + +To run a test, use make test example=default, replacing default with the example you want to test from the examples directory. + +Add skip-destroy=true to skip the destroy step, like make test example=default skip-destroy=true. + +For running all tests in parallel or sequentially with make test-parallel or make test-sequential, exclude specific examples by adding exception=example1,example2, where example1 and example2 are examples to skip. + +These tests ensure the module's reliability across configurations. diff --git a/examples/default/main.tf b/examples/default/main.tf index 73930a3..8bb55f4 100644 --- a/examples/default/main.tf +++ b/examples/default/main.tf @@ -11,7 +11,7 @@ module "rg" { groups = { demo = { - name = module.naming.resource_group.name + name = module.naming.resource_group.name_unique location = "westeurope" } } diff --git a/examples/secure-vhubs/main.tf b/examples/secure-vhubs/main.tf index f37e189..476cf32 100644 --- a/examples/secure-vhubs/main.tf +++ b/examples/secure-vhubs/main.tf @@ -11,7 +11,7 @@ module "rg" { groups = { demo = { - name = module.naming.resource_group.name + name = module.naming.resource_group.name_unique location = "westeurope" } } diff --git a/tests/deploy_test.go b/tests/deploy_test.go index 8a4d914..2e41c05 100644 --- a/tests/deploy_test.go +++ b/tests/deploy_test.go @@ -1,21 +1,57 @@ package main import ( + "flag" + "fmt" "os" "path/filepath" + "strings" "testing" "github.com/gruntwork-io/terratest/modules/terraform" ) -type TerraformModule struct { +var ( + skipDestroy bool + exception string + example string + exceptionList map[string]bool +) + +func init() { + flag.BoolVar(&skipDestroy, "skip-destroy", false, "Skip running terraform destroy after apply") + flag.StringVar(&exception, "exception", "", "Comma-separated list of examples to exclude") + flag.StringVar(&example, "example", "", "Specific example to test") +} + +func parseExceptionList() { + exceptionList = make(map[string]bool) + if exception != "" { + examples := strings.Split(exception, ",") + for _, ex := range examples { + exceptionList[strings.TrimSpace(ex)] = true + } + } +} + +type Module struct { Name string Path string Options *terraform.Options } -func NewTerraformModule(name, path string) *TerraformModule { - return &TerraformModule{ +type ModuleManager struct { + BaseExamplesPath string +} + +func NewModuleManager(baseExamplesPath string) *ModuleManager { + return &ModuleManager{ + BaseExamplesPath: baseExamplesPath, + } +} + +func NewModule(name, path string) *Module { + return &Module{ Name: name, Path: path, Options: &terraform.Options{ @@ -25,48 +61,169 @@ func NewTerraformModule(name, path string) *TerraformModule { } } -func (m *TerraformModule) Apply(t *testing.T) { +func (mm *ModuleManager) DiscoverModules() ([]*Module, error) { + var modules []*Module + + entries, err := os.ReadDir(mm.BaseExamplesPath) + if err != nil { + return nil, fmt.Errorf("failed to read examples directory: %v", err) + } + + for _, entry := range entries { + if entry.IsDir() { + moduleName := entry.Name() + if exceptionList[moduleName] { + fmt.Printf("Skipping module %s as it is in the exception list\n", moduleName) + continue + } + modulePath := filepath.Join(mm.BaseExamplesPath, moduleName) + modules = append(modules, NewModule(moduleName, modulePath)) + } + } + + return modules, nil +} + +func (m *Module) Apply(t *testing.T) error { + t.Helper() t.Logf("Applying Terraform module: %s", m.Name) terraform.WithDefaultRetryableErrors(t, m.Options) - terraform.InitAndApply(t, m.Options) + _, err := terraform.InitAndApplyE(t, m.Options) + return err } -func (m *TerraformModule) Destroy(t *testing.T) { +func (m *Module) Destroy(t *testing.T) error { + t.Helper() t.Logf("Destroying Terraform module: %s", m.Name) - terraform.Destroy(t, m.Options) - m.cleanupFiles(t) + _, err := terraform.DestroyE(t, m.Options) + if err != nil { + return fmt.Errorf("destroy failed: %v", err) + } + if err := m.cleanupFiles(t); err != nil { + return fmt.Errorf("cleanup failed: %v", err) + } + return nil } -func (m *TerraformModule) cleanupFiles(t *testing.T) { +func (m *Module) cleanupFiles(t *testing.T) error { + t.Helper() t.Logf("Cleaning up in: %s", m.Options.TerraformDir) - filesToCleanup := []string{"*.terraform*", "*tfstate*"} + filesToCleanup := []string{"*.terraform*", "*tfstate*", "*.lock.hcl"} + for _, pattern := range filesToCleanup { matches, err := filepath.Glob(filepath.Join(m.Options.TerraformDir, pattern)) if err != nil { - t.Errorf("Error matching pattern %s: %v", pattern, err) - continue + return fmt.Errorf("error matching pattern %s: %v", pattern, err) } for _, filePath := range matches { if err := os.RemoveAll(filePath); err != nil { - t.Errorf("Failed to remove %s: %v", filePath, err) + return fmt.Errorf("failed to remove %s: %v", filePath, err) } } } + return nil +} + +func RunTests(t *testing.T, modules []*Module, parallel bool) { + // Error collector to accumulate failures and reasons + var errorMessages []string + + for _, module := range modules { + module := module + t.Run(module.Name, func(t *testing.T) { + if parallel { + t.Parallel() + } + + // Defer Destroy to ensure cleanup happens, regardless of Apply success or failure + if !skipDestroy { + defer func() { + if err := module.Destroy(t); err != nil { + t.Logf("Warning: Cleanup for module %s failed: %v", module.Name, err) + } + }() + } + + // Apply the module and collect errors + if err := module.Apply(t); err != nil { + // Mark this test as failed and collect the error message + t.Fail() + errorMessages = append(errorMessages, fmt.Sprintf("Module %s failed: %v", module.Name, err)) + t.Logf("Apply failed for module %s: %v", module.Name, err) + } + }) + } + + // After all tests are complete, log the summary of errors if any + t.Cleanup(func() { + if len(errorMessages) > 0 { + t.Log("Summary of failed modules:") + for _, msg := range errorMessages { + t.Log(msg) + } + } else { + t.Log("All modules applied and destroyed successfully.") + } + }) } func TestApplyNoError(t *testing.T) { - t.Parallel() + flag.Parse() + parseExceptionList() - example := os.Getenv("EXAMPLE") if example == "" { - t.Fatal("EXAMPLE environment variable is not set") + t.Fatal("-example flag is not set") + } + + if exceptionList[example] { + t.Skipf("Skipping example %s as it is in the exception list", example) } modulePath := filepath.Join("..", "examples", example) - module := NewTerraformModule(example, modulePath) + module := NewModule(example, modulePath) + var errorMessages []string - t.Run(module.Name, func(t *testing.T) { - defer module.Destroy(t) - module.Apply(t) - }) + if err := module.Apply(t); err != nil { + errorMessages = append(errorMessages, fmt.Sprintf("Apply failed for module %s: %v", module.Name, err)) + t.Fail() + } + + if !skipDestroy { + if err := module.Destroy(t); err != nil { + errorMessages = append(errorMessages, fmt.Sprintf("Cleanup failed for module %s: %v", module.Name, err)) + } + } + + if len(errorMessages) > 0 { + fmt.Println("Summary of errors:") + for _, msg := range errorMessages { + fmt.Println(msg) + } + } +} + +func TestApplyAllSequential(t *testing.T) { + flag.Parse() + parseExceptionList() + + manager := NewModuleManager(filepath.Join("..", "examples")) + modules, err := manager.DiscoverModules() + if err != nil { + t.Fatalf("Failed to discover modules: %v", err) + } + + RunTests(t, modules, false) +} + +func TestApplyAllParallel(t *testing.T) { + flag.Parse() + parseExceptionList() + + manager := NewModuleManager(filepath.Join("..", "examples")) + modules, err := manager.DiscoverModules() + if err != nil { + t.Fatalf("Failed to discover modules: %v", err) + } + + RunTests(t, modules, true) }