Add generators

Makefile

@@ -54,7 +54,7 @@ test/cov.cov: clean-tests build/holo.test
 		export HOLO_BINARY=../../../build/holo.test && \
 		export HOLO_TEST_COVERDIR=$(abspath test/cov) && \
 		export HOLO_TEST_SCRIPTPATH=../../../util && \
-		$(foreach p,files run-scripts ssh-keys users-groups,\
+		$(foreach p,files run-scripts ssh-keys users-groups generators,\
 			ln -sfT ../build/holo.test test/holo-$p && \
 			./util/holo-test holo-$p $(sort $(wildcard test/$p/??-*)) && ) \
 		true

README.md

@@ -46,6 +46,7 @@ User documentation is available in man page form:
 * [holo-users-groups(8)](doc/holo-users-groups.8.pod)
 * [holorc(5)](doc/holorc.5.pod)
 * [holo-plugin-interface(7)](doc/holo-plugin-interface.7.pod)
+* [holo-generators(7)](doc/holo-generators.7.pod)
 * [holo-test(7)](doc/holo-test.7.pod) (not a public interface)
 
 For further information, visit [holocm.org](http://holocm.org).

cmd/holo/internal/generators.go

@@ -0,0 +1,181 @@
+/*******************************************************************************
+*
+* Copyright 2020 Peter Werner <[email protected]>
+*
+* This file is part of Holo.
+*
+* Holo is free software: you can redistribute it and/or modify it under the
+* terms of the GNU General Public License as published by the Free Software
+* Foundation, either version 3 of the License, or (at your option) any later
+* version.
+*
+* Holo is distributed in the hope that it will be useful, but WITHOUT ANY
+* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+* A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License along with
+* Holo. If not, see <http://www.gnu.org/licenses/>.
+*
+*******************************************************************************/
+
+package impl
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+)
+
+// RunGenerators executes all generators in the generator directory
+// and changes the resource path of plugins for which files were
+// generated to.
+func RunGenerators(config *Configuration) error {
+	inputDir := getGenertorsDir()
+	if _, err := os.Stat(inputDir); err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+	targetDir, err := getGeneratorCacheDir()
+	if err != nil {
+		return fmt.Errorf(
+			"couldn't access cache-dir ('%s') for generators: %s",
+			targetDir, err,
+		)
+	}
+	runGenerators(inputDir, targetDir)
+	for _, plugin := range config.Plugins {
+		if err := updatePluginPaths(plugin, targetDir); err != nil {
+			Errorf(Stderr,
+				"Failed to perpare generated dir for plugin '%s': %s",
+				plugin.id, err.Error(),
+			)
+		}
+	}
+	return nil
+}
+
+func runGenerators(inputDir string, targetDir string) {
+	filepath.Walk(inputDir,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				Warnf(Stderr, "%s: %s", path, err.Error())
+				return nil
+			}
+			if isExecutableFile(info) {
+				out, err := runGenerator(path, targetDir)
+				// Keep silent unless an error occurred or generator has
+				// printed output.
+				if err != nil || len(out) > 0 {
+					shortPath, _ := filepath.Rel(inputDir, path)
+					fmt.Fprintf(os.Stdout, "Ran generator %s\n", shortPath)
+					fmt.Fprintf(os.Stdout, "     found at %s\n", path)
+					Stdout.Write(out)
+					if err != nil {
+						Errorf(Stderr, err.Error())
+					}
+				}
+			}
+			return nil
+		})
+}
+
+func updatePluginPaths(plugin *Plugin, dir string) error {
+	pluginDir := plugin.ResourceDirectory()
+	newPluginDir := filepath.Join(dir, plugin.id)
+	if info, err := os.Stat(newPluginDir); err == nil && info.IsDir() {
+		// Files were generated for this plugin.
+		// Fill the plugins directory with existsing static files.
+		if err := symlinkFiles(pluginDir, newPluginDir); err != nil {
+			return err
+		}
+		// Change the plugin resource dir to point to the generated dir.
+		resource, _ := filepath.Rel(RootDirectory(), dir)
+		plugin.SetResourceRoot(resource)
+	}
+	return nil
+}
+
+func symlinkFiles(oldDir string, newDir string) error {
+	return filepath.Walk(oldDir,
+		func(oldFile string, info os.FileInfo, err error) error {
+			if err != nil || oldFile == oldDir {
+				return err
+			}
+			relPath, _ := filepath.Rel(oldDir, oldFile)
+			newFile := filepath.Join(newDir, relPath)
+			err = os.Symlink(oldFile, newFile)
+			if os.IsExist(err) {
+				// newFile already exists. Examine it.
+				newFileInfo, err := os.Lstat(newFile)
+				if err == nil && info.IsDir() && !newFileInfo.IsDir() {
+					// newFile exists but is not a directory.
+					// If oldFile is a directory trying to symlink its contents
+					// will result in errors. Skip it.
+					return filepath.SkipDir
+				}
+				return nil
+			}
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				// Symlink to dir was created don't check its contents.
+				return filepath.SkipDir
+			}
+			return nil
+		})
+}
+
+func runGenerator(fileToRun string, targetDir string) ([]byte, error) {
+	cmd := exec.Command(fileToRun)
+	env := os.Environ()
+	env = append(
+		env,
+		fmt.Sprintf("OUT=%s", targetDir),
+	)
+	cmd.Env = env
+	return cmd.CombinedOutput()
+}
+
+func getGenertorsDir() string {
+	return filepath.Join(RootDirectory(), "/usr/share/holo/generators")
+}
+
+func getGeneratorCacheDir() (string, error) {
+	path, err := prepareDir(RootDirectory(), "/var/tmp/holo/generated")
+	if err == nil {
+		return path, nil
+	}
+	path, err = prepareDir(
+		os.Getenv("HOLO_CACHE_DIR"), "holo/generated",
+	)
+	if err == nil {
+		return path, nil
+	}
+	return "", err
+}
+
+func prepareDir(pathParts ...string) (string, error) {
+	path := filepath.Join(pathParts...)
+	if err := os.MkdirAll(path, 0755); err != nil {
+		if os.IsExist(err) {
+			return path, nil
+		}
+		return "", err
+	}
+	return path, nil
+}
+
+func isExecutableFile(stat os.FileInfo) bool {
+	mode := stat.Mode()
+	if !mode.IsRegular() {
+		return false
+	}
+	if (mode & 0111) == 0 {
+		return false
+	}
+	return true
+}

cmd/holo/internal/plugin.go

@@ -43,7 +43,9 @@ var ErrPluginExecutableMissing = errors.New("ErrPluginExecutableMissing")
 type Plugin struct {
 	id             string
 	executablePath string
-	metadata       map[string]string //from "info" call
+	// Root path for the plugin specific resource dir
+	resourceRoot string
+	metadata     map[string]string //from "info" call
 }
 
 //NewPlugin creates a new Plugin.
@@ -56,7 +58,12 @@ func NewPlugin(id string) (*Plugin, error) {
 //a non-standard location. (This is used exclusively for testing plugins before
 //they are installed.)
 func NewPluginWithExecutablePath(id string, executablePath string) (*Plugin, error) {
-	p := &Plugin{id, executablePath, make(map[string]string)}
+	p := &Plugin{
+		id,
+		executablePath,
+		"usr/share/holo/",
+		make(map[string]string),
+	}
 
 	//check if the plugin executable exists
 	_, err := os.Stat(executablePath)
@@ -108,10 +115,17 @@ func (p *Plugin) ID() string {
 	return p.id
 }
 
+//SetResourceRoot changes the resource root to given path.
+//Future calls to ResourceDirectory will return a path relative to
+//given path.
+func (p *Plugin) SetResourceRoot(path string) {
+	p.resourceRoot = path
+}
+
 //ResourceDirectory returns the path to the directory where this plugin may
 //find its resources (entity definitions etc.).
 func (p *Plugin) ResourceDirectory() string {
-	return filepath.Join(RootDirectory(), "usr/share/holo/"+p.id)
+	return filepath.Join(RootDirectory(), p.resourceRoot, p.id)
 }
 
 //CacheDirectory returns the path to the directory where this plugin may

cmd/holo/main.go

@@ -104,6 +104,13 @@ func Main() (exitCode int) {
 			selectors = append(selectors, &Selector{String: arg, Used: false})
 		}
 
+		//run generators before scan phase
+		if err := impl.RunGenerators(config); err != nil {
+			impl.Errorf(impl.Stderr,
+				"Failed to process generators: %s", err.Error(),
+			)
+		}
+
 		//ask all plugins to scan for entities
 		var entities []*impl.Entity
 		for _, plugin := range config.Plugins {

doc/holo-generators.7.pod

@@ -0,0 +1,56 @@
+=encoding UTF-8
+
+=head1 NAME
+
+holo-generators - dynamically generate files for other plugins
+
+=head1 DESCRIPTION
+
+Generators are executable files placed under 
+F</usr/share/holo/generators>.
+Holo will execute generators in lexical order before asking plugins
+to scan their files.
+Upon execution of a generator the environment variable C<$OUT> is 
+passed to it.
+Unless for caching purposes generators MUST only write to the 
+directory specified by C<$OUT>.
+The structure under C<$OUT> is the same as for static files under
+F</usr/share/holo/>.
+For example to generate files for the holo-files plugin a generator 
+MAY place files at F<$OUT/files>.
+
+Generated files placed under C<$OUT> and static files under 
+F</usr/share/holo/> will be made available for plugins, 
+while generated files take precedence over static files.
+This means if a generator places a file at 
+F<$OUT/files/20-webserver/etc/nginx/nginx.conf>
+the plugin holo-files will only see this file insteadof the file at
+F</usr/share/holo/files/20-webserver/etc/nginx/nginx.conf>.
+
+Files that have been written to C<$OUT> will be preserved between 
+runs.
+A generator MAY decide wether to overwrite existing files or not
+this includes files written by other generators.
+It is RECOMMENDED that generators only regenerate files if an output
+different from the previous run is expected. 
+
+The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT",
+"SHOULD", "SHOULD NOT", "RECOMMENDED",  "MAY", and "OPTIONAL" in 
+this document are to be interpreted as described in 
+L<RFC 2119|https://tools.ietf.org/html/rfc2119>.
+
+=head1 SEE ALSO
+
+L<holorc(5)>
+
+L<holo-files(8)>.
+
+=head1 AUTHOR
+
+Peter Werner
+
+Further documentation is available at the project homepage: http://holocm.org
+
+Please report any issues and feature requests at GitHub: http://github.com/holocm/holo/issues
+
+=cut

test/generators/01-basic/README.md

@@ -0,0 +1,8 @@
+Basic tests for the generators feature.
+The following cases are covered:
+- Generator files are executed and generated files are deployed 
+  into the respective folder.
+- The folder is passed correctly to plugin.
+- Generators are executed in alphabetical order.
+- Generators with non-zero exit code are are mentioned in the output
+  and do not stall the execution.

test/generators/01-basic/expected-apply-output

@@ -0,0 +1,10 @@
+Ran generator 02-failing.sh
+     found at target/usr/share/holo/generators/02-failing.sh
+
+02-failing.sh: failing
+!! exit status 1
+
+Printing env:target/var/tmp/holo/generated/print
+found at HOLO_RESOURCE_DIR
+
+exit status 0

test/generators/01-basic/expected-diff-output

@@ -0,0 +1,7 @@
+Ran generator 02-failing.sh
+     found at target/usr/share/holo/generators/02-failing.sh
+
+02-failing.sh: failing
+!! exit status 1
+
+exit status 0

test/generators/01-basic/expected-scan-output

@@ -0,0 +1,10 @@
+Ran generator 02-failing.sh
+     found at target/usr/share/holo/generators/02-failing.sh
+
+02-failing.sh: failing
+!! exit status 1
+
+env:target/var/tmp/holo/generated/print
+    found at HOLO_RESOURCE_DIR
+
+exit status 0

test/generators/01-basic/expected-tree

@@ -0,0 +1,28 @@
+symlink   0777 ./etc/holorc
+../../../holorc
+----------------------------------------
+directory 0755 ./run/
+----------------------------------------
+directory 0755 ./tmp/
+----------------------------------------
+file      0755 ./usr/share/holo/generators/01-simple.sh
+#!/bin/sh
+mkdir -p "${OUT}/print/"
+echo "Simple generated file" > "${OUT}/print/file.txt"
+----------------------------------------
+file      0755 ./usr/share/holo/generators/02-failing.sh
+#!/bin/sh
+echo "02-failing.sh: failing"
+exit 1
+----------------------------------------
+directory 0755 ./usr/share/holo/print/
+----------------------------------------
+directory 0755 ./var/lib/holo/files/base/
+----------------------------------------
+directory 0755 ./var/lib/holo/files/provisioned/
+----------------------------------------
+directory 0755 ./var/lib/holo/print/
+----------------------------------------
+file      0644 ./var/tmp/holo/generated/print/file.txt
+Simple generated file
+----------------------------------------