feat: load modules from multiple paths, support symlink as module

MODULES.md

@@ -1,26 +1,34 @@
 # Module structure
 
-A module is a directory with files. Addon-operator searches for the modules directories in `/modules` or in the path specified by the $MODULES_DIR variable. The module has the same name as the corresponding directory excluding the numeric prefix.
+A module is a directory with files. Addon-operator searches for the modules directories in `/modules` or in the paths specified by the $MODULES_DIR variable. The module has the same name as the corresponding directory excluding the numeric prefix.
 
-The file structure of the module’s directory:
+An example of the file structure of the module:
 
 ```
 /modules/001-simple-module
-├── .helmignore
-├── Chart.yaml
-├── enabled
 ├── hooks
-│   └── module-hooks.sh
-├── README.md
+│   ├── module-hook-1.sh
+│   ├── ...
+│   └── module-hook-N.sh
+├── openapi
+│   ├── config-values.yaml
+│   └── values.yaml
 ├── templates
+│   ├── config-maps.yaml
+│   ├── ...
 │   └── daemon-set.yaml
+├── enabled
+├── README.md
+├── .helmignore
+├── Chart.yaml
 └── values.yaml
 ```
 
-- `hooks` — a directory with hooks;
-- `enabled` — a script that gets the status of module (is it enabled or not). See the [modules discovery](LIFECYCLE.md#modules-discovery) process;
-- `Chart.yaml`, `.helmignore`, `templates` — a Helm chart files;
-- `README.md` — a file with the module description;
+- `hooks` — a directory with hooks.
+- `openapi` — [OpenAPI schemas](VALUES.md) for config values and for helm values.
+- `enabled` — a script that gets the status of module (is it enabled or not). See the [modules discovery](LIFECYCLE.md#modules-discovery) process.
+- `Chart.yaml`, `.helmignore`, `templates` — a Helm chart files.
+- `README.md` — an optional file with the module description.
 - `values.yaml` – default values for chart in a [YAML format](VALUES.md).
 
 The name of this module is `simple-module`. values.yaml should contain a section `simpleModule` and a `simpleModuleEnabled` flag (see [VALUES](VALUES.md#values-storage)). 

RUNNING.md

@@ -2,13 +2,15 @@
 
 ## Environment variables
 
-**MODULES_DIR** — a directory where modules are located.
-
 **GLOBAL_HOOKS_DIR** — a directory with global hook files.
 
-**ADDON_OPERATOR_NAMESPACE** — a required parameter with namespace where Addon-operator is deployed.
+**MODULES_DIR** — paths separated by colon where modules are located.
+
+**UNNUMBERED_MODULE_ORDER** — an integer number to use as the default order for modules without numbered prefix.
+
+**ADDON_OPERATOR_NAMESPACE** — a required parameter with namespace where Addon-operator is deployed.
 
-**ADDON_OPERATOR_CONFIG_MAP** — a name of ConfigMap to store values. Default is `addon-operator`.
+**ADDON_OPERATOR_CONFIG_MAP** — a name of ConfigMap to store values. Default is `addon-operator`.
 
 Namespace and config map name are used to watch for ConfigMap changes. 
 

pkg/app/app.go

@@ -31,22 +31,28 @@ var (
 
 	GlobalHooksDir = "global-hooks"
 	ModulesDir     = "modules"
+
+	UnnumberedModuleOrder = 1
 )
 
 const (
-	DefaultGlobalHooksDir  = "global-hooks"
-	DefaultModulesDir      = "modules"
 	DefaultTempDir         = "/tmp/addon-operator"
 	DefaultDebugUnixSocket = "/var/run/addon-operator/debug.socket"
 )
 
 // DefineStartCommandFlags init global flags with default values
 func DefineStartCommandFlags(kpApp *kingpin.Application, cmd *kingpin.CmdClause) {
-	cmd.Flag("modules-dir", "a path where to search for module directories").
+	cmd.Flag("modules-dir", "paths where to search for module directories").
 		Envar("MODULES_DIR").
-		Default(DefaultModulesDir).
+		Default(ModulesDir).
 		StringVar(&ModulesDir)
 
+	// TODO Delete this setting after refactoring module dependencies machinery.
+	cmd.Flag("unnumbered-modules-order", "default order for modules without numbered prefix in name").
+		Envar("UNNUMBERED_MODULES_ORDER").
+		Default(strconv.Itoa(UnnumberedModuleOrder)).
+		IntVar(&UnnumberedModuleOrder)
+
 	cmd.Flag("global-hooks-dir", "a path where to search for global hook files (and OpenAPI schemas)").
 		Envar("GLOBAL_HOOKS_DIR").
 		Default(GlobalHooksDir).

pkg/module_manager/module.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"regexp"
 	"runtime/trace"
 	"strings"
 	"time"
@@ -33,8 +32,9 @@ import (
 )
 
 type Module struct {
-	Name string
-	Path string
+	Name  string // MODULE_NAME env
+	Path  string // MODULE_DIR env
+	Order int    // MODULE_ORDER env
 	// module values from modules/values.yaml file
 	CommonStaticConfig *utils.ModuleConfig
 	// module values from modules/<module name>/values.yaml
@@ -47,10 +47,11 @@ type Module struct {
 	helm          *helm.ClientFactory
 }
 
-func NewModule(name, path string) *Module {
+func NewModule(name string, path string, order int) *Module {
 	return &Module{
 		Name:  name,
 		Path:  path,
+		Order: order,
 		State: NewModuleState(),
 	}
 }
@@ -885,61 +886,30 @@ func (m *Module) runEnabledScript(precedingEnabledModules []string, logLabels ma
 	return moduleEnabled, nil
 }
 
-var ValidModuleNameRe = regexp.MustCompile(`^[0-9][0-9][0-9]-(.*)$`)
-
-func SearchModules(modulesDir string) (modules []*Module, err error) {
-	if modulesDir == "" {
-		log.Warnf("Modules directory path is empty! No modules to load.")
-		return nil, nil
-	}
-
-	files, err := os.ReadDir(modulesDir) // returns a list of modules sorted by filename
-	if err != nil {
-		return nil, fmt.Errorf("list modules directory '%s': %s", modulesDir, err)
+// RegisterModules load all available modules from modules directory.
+func (mm *moduleManager) RegisterModules() error {
+	if mm.ModulesDir == "" {
+		log.Warnf("Empty modules directory is passed! No modules to load.")
+		return nil
 	}
 
-	badModulesDirs := make([]string, 0)
-	modules = make([]*Module, 0)
-
-	for _, file := range files {
-		if !file.IsDir() {
-			continue
-		}
-		matchRes := ValidModuleNameRe.FindStringSubmatch(file.Name())
-		if matchRes != nil {
-			moduleName := matchRes[1]
-			modulePath := filepath.Join(modulesDir, file.Name())
-			module := NewModule(moduleName, modulePath)
-			modules = append(modules, module)
-		} else {
-			badModulesDirs = append(badModulesDirs, filepath.Join(modulesDir, file.Name()))
-		}
-	}
+	log.Debug("Search and register modules")
 
-	if len(badModulesDirs) > 0 {
-		return nil, fmt.Errorf("modules directory contains directories not matched ValidModuleRegex '%s': %s", ValidModuleNameRe, strings.Join(badModulesDirs, ", "))
+	// load global and modules common static values from modules/values.yaml
+	commonStaticValues, err := LoadCommonStaticValues(mm.ModulesDir)
+	if err != nil {
+		return fmt.Errorf("load common values for modules: %s", err)
 	}
-
-	return
-}
-
-// RegisterModules load all available modules from modules directory
-// FIXME: Only 000-name modules are loaded, allow non-prefixed modules.
-func (mm *moduleManager) RegisterModules() error {
-	log.Debug("Search and register modules")
+	mm.commonStaticValues = commonStaticValues
 
 	modules, err := SearchModules(mm.ModulesDir)
 	if err != nil {
 		return err
 	}
-	log.Debugf("Found %d modules", len(modules))
 
-	// load global and modules common static values from modules/values.yaml
-	if err := mm.loadCommonStaticValues(); err != nil {
-		return fmt.Errorf("load common values for modules: %s", err)
-	}
+	log.Debugf("Found modules: %v", modules.NamesInOrder())
 
-	for _, module := range modules {
+	for _, module := range modules.List() {
 		logEntry := log.WithField("module", module.Name)
 
 		module.WithModuleManager(mm)
@@ -953,9 +923,6 @@ func (mm *moduleManager) RegisterModules() error {
 			return fmt.Errorf("bad module values")
 		}
 
-		mm.allModulesByName[module.Name] = module
-		mm.allModulesNamesInOrder = append(mm.allModulesNamesInOrder, module.Name)
-
 		// Load validation schemas
 		openAPIPath := filepath.Join(module.Path, "openapi")
 		configBytes, valuesBytes, err := ReadOpenAPIFiles(openAPIPath)
@@ -975,6 +942,7 @@ func (mm *moduleManager) RegisterModules() error {
 		logEntry.Infof("Module from '%s'. %s", module.Path, mm.ValuesValidator.SchemaStorage.ModuleSchemasDescription(module.ValuesKey()))
 	}
 
+	mm.modules = modules
 	return nil
 }
 
@@ -985,13 +953,13 @@ func (m *Module) loadStaticValues() (err error) {
 	if err != nil {
 		return err
 	}
-	log.Debugf("module %s common static values: %s", m.Name, m.CommonStaticConfig.Values.DebugString())
+	log.Debugf("module %s static values in common file: %s", m.Name, m.CommonStaticConfig.Values.DebugString())
 
-	valuesYamlPath := filepath.Join(m.Path, "values.yaml")
+	valuesYamlPath := filepath.Join(m.Path, ValuesFileName)
 
 	if _, err := os.Stat(valuesYamlPath); os.IsNotExist(err) {
 		m.StaticConfig = utils.NewModuleConfig(m.Name)
-		log.Debugf("module %s is static disabled: no values.yaml exists", m.Name)
+		log.Debugf("module %s has no static values", m.Name)
 		return nil
 	}
 
@@ -1008,30 +976,6 @@ func (m *Module) loadStaticValues() (err error) {
 	return nil
 }
 
-func (mm *moduleManager) loadCommonStaticValues() error {
-	valuesPath := filepath.Join(mm.ModulesDir, "values.yaml")
-	if _, err := os.Stat(valuesPath); os.IsNotExist(err) {
-		log.Debugf("No common static values file: %s", err)
-		return nil
-	}
-
-	valuesYaml, err := os.ReadFile(valuesPath)
-	if err != nil {
-		return fmt.Errorf("load common values file '%s': %s", valuesPath, err)
-	}
-
-	values, err := utils.NewValuesFromBytes(valuesYaml)
-	if err != nil {
-		return err
-	}
-
-	mm.commonStaticValues = values
-
-	log.Debugf("Successfully load common static values:\n%s", mm.commonStaticValues.DebugString())
-
-	return nil
-}
-
 func dumpData(filePath string, data []byte) error {
 	err := os.WriteFile(filePath, data, 0o644)
 	if err != nil {