Add traverse_to_nested_projects flag (#1245)

* Adds traverse_to_nested_projects flag, tests, and docs


---------

Signed-off-by: Roee Landesman <[email protected]>
Co-authored-by: Mohamed Habib <[email protected]>

Author: ri-roee

docs/howto/generate-projects.mdx

@@ -30,5 +30,15 @@ generate_projects:
 
 [Demo repo](https://github.com/diggerhq/generate_projects_demo)
 
+# Traversing nested directories
 
+You can optionally set the top-level argument `traverse_to_nested_projects` to generate a project for all sub-directories:
 
+```
+traverse_to_nested_projects: true
+generate_projects:
+  blocks:
+    - include: "environments/core/**"
+```
+
+This will create a project for all sub-directories under environments/core. If set to `false`, only the first directory with a .tf file will be evaluated.

docs/reference/digger.yml.mdx

@@ -7,6 +7,7 @@ type DiggerConfig struct {
 	Telemetry                  bool
 	Workflows                  map[string]Workflow
 	MentionDriftedProjectsInPR bool
+	TraverseToNestedProjects   bool
 }
 
 type DependencyConfiguration struct {

libs/digger_config/config.go

@@ -3,6 +3,7 @@ package digger_config
 import (
 	"errors"
 	"fmt"
+
 	"github.com/dominikbraun/graph"
 )
 
@@ -161,6 +162,12 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml) (*DiggerConfig, gra
 		diggerConfig.Telemetry = true
 	}
 
+	if diggerYaml.TraverseToNestedProjects != nil {
+		diggerConfig.TraverseToNestedProjects = *diggerYaml.TraverseToNestedProjects
+	} else {
+		diggerConfig.TraverseToNestedProjects = false
+	}
+
 	// if workflow block is not specified in yaml we create a default one, and add it to every project
 	if diggerYaml.Workflows != nil {
 		workflows := copyWorkflows(diggerYaml.Workflows)

libs/digger_config/converters.go

@@ -16,7 +16,7 @@ import (
 )
 
 type DirWalker interface {
-	GetDirs(workingDir string) ([]string, error)
+	GetDirs(workingDir string, config DiggerConfigYaml) ([]string, error)
 }
 
 type FileSystemTopLevelTerraformDirWalker struct {
@@ -46,7 +46,7 @@ func GetFilesWithExtension(workingDir string, ext string) ([]string, error) {
 	return files, nil
 }
 
-func (walker *FileSystemTopLevelTerraformDirWalker) GetDirs(workingDir string) ([]string, error) {
+func (walker *FileSystemTopLevelTerraformDirWalker) GetDirs(workingDir string, configYaml *DiggerConfigYaml) ([]string, error) {
 	var dirs []string
 	err := filepath.Walk(workingDir,
 		func(path string, info os.FileInfo, err error) error {
@@ -61,7 +61,9 @@ func (walker *FileSystemTopLevelTerraformDirWalker) GetDirs(workingDir string) (
 				terraformFiles, _ := GetFilesWithExtension(path, ".tf")
 				if len(terraformFiles) > 0 {
 					dirs = append(dirs, strings.ReplaceAll(path, workingDir+string(os.PathSeparator), ""))
-					return filepath.SkipDir
+					if configYaml.TraverseToNestedProjects != nil && !*configYaml.TraverseToNestedProjects {
+						return filepath.SkipDir
+					}
 				}
 			}
 			return nil
@@ -72,7 +74,7 @@ func (walker *FileSystemTopLevelTerraformDirWalker) GetDirs(workingDir string) (
 	return dirs, nil
 }
 
-func (walker *FileSystemModuleDirWalker) GetDirs(workingDir string) ([]string, error) {
+func (walker *FileSystemModuleDirWalker) GetDirs(workingDir string, configYaml *DiggerConfigYaml) ([]string, error) {
 	var dirs []string
 	err := filepath.Walk(workingDir,
 		func(path string, info os.FileInfo, err error) error {
@@ -92,7 +94,7 @@ func (walker *FileSystemModuleDirWalker) GetDirs(workingDir string) ([]string, e
 	return dirs, nil
 }
 
-func (walker *FileSystemTerragruntDirWalker) GetDirs(workingDir string) ([]string, error) {
+func (walker *FileSystemTerragruntDirWalker) GetDirs(workingDir string, configYaml *DiggerConfigYaml) ([]string, error) {
 	var dirs []string
 	err := filepath.Walk(workingDir,
 		func(path string, info os.FileInfo, err error) error {
@@ -199,7 +201,7 @@ func HandleYamlProjectGeneration(config *DiggerConfigYaml, terraformDir string)
 		}
 	} else if config.GenerateProjectsConfig != nil {
 		var dirWalker = &FileSystemTopLevelTerraformDirWalker{}
-		dirs, err := dirWalker.GetDirs(terraformDir)
+		dirs, err := dirWalker.GetDirs(terraformDir, config)
 
 		if err != nil {
 			fmt.Printf("Error while walking through directories: %v", err)
@@ -410,22 +412,25 @@ func AutoDetectDiggerConfig(workingDir string) (*DiggerConfigYaml, error) {
 	telemetry := true
 	configYaml.Telemetry = &telemetry
 
+	TraverseToNestedProjects := false
+	configYaml.TraverseToNestedProjects = &TraverseToNestedProjects
+
 	terragruntDirWalker := &FileSystemTerragruntDirWalker{}
 	terraformDirWalker := &FileSystemTopLevelTerraformDirWalker{}
 	moduleDirWalker := &FileSystemModuleDirWalker{}
 
-	terragruntDirs, err := terragruntDirWalker.GetDirs(workingDir)
+	terragruntDirs, err := terragruntDirWalker.GetDirs(workingDir, configYaml)
 
 	if err != nil {
 		return nil, err
 	}
 
-	terraformDirs, err := terraformDirWalker.GetDirs(workingDir)
+	terraformDirs, err := terraformDirWalker.GetDirs(workingDir, configYaml)
 	if err != nil {
 		return nil, err
 	}
 
-	moduleDirs, err := moduleDirWalker.GetDirs(workingDir)
+	moduleDirs, err := moduleDirWalker.GetDirs(workingDir, configYaml)
 
 	var modulePatterns []string
 	for _, dir := range moduleDirs {

libs/digger_config/digger_config.go

@@ -97,6 +97,7 @@ func TestNoDiggerYaml(t *testing.T) {
 	assert.Equal(t, 1, len(dg.Projects))
 	assert.Equal(t, false, dg.AutoMerge)
 	assert.Equal(t, true, dg.Telemetry)
+	assert.Equal(t, false, dg.TraverseToNestedProjects)
 	assert.Equal(t, 1, len(dg.Workflows))
 	assert.Equal(t, "default", dg.Projects[0].Name)
 	assert.Equal(t, "./", dg.Projects[0].Dir)
@@ -137,6 +138,7 @@ projects:
 	assert.Equal(t, 1, len(dg.Projects))
 	assert.Equal(t, false, dg.AutoMerge)
 	assert.Equal(t, true, dg.Telemetry)
+	assert.Equal(t, false, dg.TraverseToNestedProjects)
 	assert.Equal(t, 1, len(dg.Workflows))
 
 	assert.Equal(t, "prod", dg.Projects[0].Name)
@@ -1114,3 +1116,44 @@ func TestDiggerGenerateProjectsMultipleBlocksDemo(t *testing.T) {
 }
 
 // todo test terragrunt digger_config with terragrunt_parsing block but without terragrunt: true
+
+// TestDiggerTraverseToNestedProjects test if traverse_to_nested_projects is set to true, digger will traverse to nested projects
+func TestDiggerTraverseToNestedProjects(t *testing.T) {
+	tempDir, teardown := setUp()
+	defer teardown()
+
+	diggerCfg := `
+traverse_to_nested_projects: true
+generate_projects:
+  blocks:
+    - include: dev/**
+      aws_role_to_assume:
+        state: "arn://abc:xyz:state"
+        command: "arn://abc:xyz:cmd"
+`
+	deleteFile := createFile(path.Join(tempDir, "digger.yml"), diggerCfg)
+	defer deleteFile()
+	dirsToCreate := []string{"dev/test1", "dev/test2", "dev/project", "dev/project/test3", "testtt"}
+
+	for _, dir := range dirsToCreate {
+		err := os.MkdirAll(path.Join(tempDir, dir), os.ModePerm)
+		defer createFile(path.Join(tempDir, dir, "main.tf"), "")()
+		assert.NoError(t, err, "expected error to be nil")
+	}
+
+	dg, _, _, err := LoadDiggerConfig(tempDir)
+	assert.NoError(t, err, "expected error to be nil")
+	assert.NotNil(t, dg, "expected digger digger_config to be not nil")
+	assert.Equal(t, true, dg.TraverseToNestedProjects)
+	assert.Equal(t, 4, len(dg.Projects))
+	assert.Equal(t, "arn://abc:xyz:cmd", dg.Projects[0].AwsRoleToAssume.Command)
+	assert.Equal(t, "arn://abc:xyz:state", dg.Projects[0].AwsRoleToAssume.State)
+	assert.Equal(t, "dev_project", dg.Projects[0].Name)
+	assert.Equal(t, "dev/project", dg.Projects[0].Dir)
+	assert.Equal(t, "dev_project_test3", dg.Projects[1].Name)
+	assert.Equal(t, "dev/project/test3", dg.Projects[1].Dir)
+	assert.Equal(t, "dev_test1", dg.Projects[2].Name)
+	assert.Equal(t, "dev/test1", dg.Projects[2].Dir)
+	assert.Equal(t, "dev_test2", dg.Projects[3].Name)
+	assert.Equal(t, "dev/test2", dg.Projects[3].Dir)
+}

libs/digger_config/digger_config_test.go

@@ -13,6 +13,7 @@ type DiggerConfigYaml struct {
 	Workflows                  map[string]*WorkflowYaml     `yaml:"workflows"`
 	Telemetry                  *bool                        `yaml:"telemetry,omitempty"`
 	GenerateProjectsConfig     *GenerateProjectsConfigYaml  `yaml:"generate_projects"`
+	TraverseToNestedProjects   *bool                        `yaml:"traverse_to_nested_projects"`
 	MentionDriftedProjectsInPR *bool                        `yaml:"mention_drifted_projects_in_pr"`
 }