diff --git a/Makefile b/Makefile index b9d63e3..91edbc8 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ test/cov.cov: clean-tests build/holo.test export HOLO_BINARY=../../../build/holo.test && \ export HOLO_TEST_COVERDIR=$(abspath test/cov) && \ export HOLO_TEST_SCRIPTPATH=../../../util && \ - $(foreach p,files run-scripts ssh-keys users-groups,\ + $(foreach p,files run-scripts ssh-keys users-groups generators,\ ln -sfT ../build/holo.test test/holo-$p && \ ./util/holo-test holo-$p $(sort $(wildcard test/$p/??-*)) && ) \ true diff --git a/README.md b/README.md index 3511498..3ff0b01 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ User documentation is available in man page form: * [holo-users-groups(8)](doc/holo-users-groups.8.pod) * [holorc(5)](doc/holorc.5.pod) * [holo-plugin-interface(7)](doc/holo-plugin-interface.7.pod) +* [holo-generators(7)](doc/holo-generators.7.pod) * [holo-test(7)](doc/holo-test.7.pod) (not a public interface) For further information, visit [holocm.org](http://holocm.org). diff --git a/cmd/holo/internal/generators.go b/cmd/holo/internal/generators.go new file mode 100644 index 0000000..ce9b216 --- /dev/null +++ b/cmd/holo/internal/generators.go @@ -0,0 +1,181 @@ +/******************************************************************************* +* +* Copyright 2020 Peter Werner +* +* This file is part of Holo. +* +* Holo is free software: you can redistribute it and/or modify it under the +* terms of the GNU General Public License as published by the Free Software +* Foundation, either version 3 of the License, or (at your option) any later +* version. +* +* Holo is distributed in the hope that it will be useful, but WITHOUT ANY +* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +* A PARTICULAR PURPOSE. See the GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License along with +* Holo. If not, see . +* +*******************************************************************************/ + +package impl + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" +) + +// RunGenerators executes all generators in the generator directory +// and changes the resource path of plugins for which files were +// generated to. +func RunGenerators(config *Configuration) error { + inputDir := getGenertorsDir() + if _, err := os.Stat(inputDir); err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + targetDir, err := getGeneratorCacheDir() + if err != nil { + return fmt.Errorf( + "couldn't access cache-dir ('%s') for generators: %s", + targetDir, err, + ) + } + runGenerators(inputDir, targetDir) + for _, plugin := range config.Plugins { + if err := updatePluginPaths(plugin, targetDir); err != nil { + Errorf(Stderr, + "Failed to perpare generated dir for plugin '%s': %s", + plugin.id, err.Error(), + ) + } + } + return nil +} + +func runGenerators(inputDir string, targetDir string) { + filepath.Walk(inputDir, + func(path string, info os.FileInfo, err error) error { + if err != nil { + Warnf(Stderr, "%s: %s", path, err.Error()) + return nil + } + if isExecutableFile(info) { + out, err := runGenerator(path, targetDir) + // Keep silent unless an error occurred or generator has + // printed output. + if err != nil || len(out) > 0 { + shortPath, _ := filepath.Rel(inputDir, path) + fmt.Fprintf(os.Stdout, "Ran generator %s\n", shortPath) + fmt.Fprintf(os.Stdout, " found at %s\n", path) + Stdout.Write(out) + if err != nil { + Errorf(Stderr, err.Error()) + } + } + } + return nil + }) +} + +func updatePluginPaths(plugin *Plugin, dir string) error { + pluginDir := plugin.ResourceDirectory() + newPluginDir := filepath.Join(dir, plugin.id) + if info, err := os.Stat(newPluginDir); err == nil && info.IsDir() { + // Files were generated for this plugin. + // Fill the plugins directory with existsing static files. + if err := symlinkFiles(pluginDir, newPluginDir); err != nil { + return err + } + // Change the plugin resource dir to point to the generated dir. + resource, _ := filepath.Rel(RootDirectory(), dir) + plugin.SetResourceRoot(resource) + } + return nil +} + +func symlinkFiles(oldDir string, newDir string) error { + return filepath.Walk(oldDir, + func(oldFile string, info os.FileInfo, err error) error { + if err != nil || oldFile == oldDir { + return err + } + relPath, _ := filepath.Rel(oldDir, oldFile) + newFile := filepath.Join(newDir, relPath) + err = os.Symlink(oldFile, newFile) + if os.IsExist(err) { + // newFile already exists. Examine it. + newFileInfo, err := os.Lstat(newFile) + if err == nil && info.IsDir() && !newFileInfo.IsDir() { + // newFile exists but is not a directory. + // If oldFile is a directory trying to symlink its contents + // will result in errors. Skip it. + return filepath.SkipDir + } + return nil + } + if err != nil { + return err + } + if info.IsDir() { + // Symlink to dir was created don't check its contents. + return filepath.SkipDir + } + return nil + }) +} + +func runGenerator(fileToRun string, targetDir string) ([]byte, error) { + cmd := exec.Command(fileToRun) + env := os.Environ() + env = append( + env, + fmt.Sprintf("OUT=%s", targetDir), + ) + cmd.Env = env + return cmd.CombinedOutput() +} + +func getGenertorsDir() string { + return filepath.Join(RootDirectory(), "/usr/share/holo/generators") +} + +func getGeneratorCacheDir() (string, error) { + path, err := prepareDir(RootDirectory(), "/var/tmp/holo/generated") + if err == nil { + return path, nil + } + path, err = prepareDir( + os.Getenv("HOLO_CACHE_DIR"), "holo/generated", + ) + if err == nil { + return path, nil + } + return "", err +} + +func prepareDir(pathParts ...string) (string, error) { + path := filepath.Join(pathParts...) + if err := os.MkdirAll(path, 0755); err != nil { + if os.IsExist(err) { + return path, nil + } + return "", err + } + return path, nil +} + +func isExecutableFile(stat os.FileInfo) bool { + mode := stat.Mode() + if !mode.IsRegular() { + return false + } + if (mode & 0111) == 0 { + return false + } + return true +} diff --git a/cmd/holo/internal/plugin.go b/cmd/holo/internal/plugin.go index 40e7f60..28e95b0 100644 --- a/cmd/holo/internal/plugin.go +++ b/cmd/holo/internal/plugin.go @@ -43,7 +43,9 @@ var ErrPluginExecutableMissing = errors.New("ErrPluginExecutableMissing") type Plugin struct { id string executablePath string - metadata map[string]string //from "info" call + // Root path for the plugin specific resource dir + resourceRoot string + metadata map[string]string //from "info" call } //NewPlugin creates a new Plugin. @@ -56,7 +58,12 @@ func NewPlugin(id string) (*Plugin, error) { //a non-standard location. (This is used exclusively for testing plugins before //they are installed.) func NewPluginWithExecutablePath(id string, executablePath string) (*Plugin, error) { - p := &Plugin{id, executablePath, make(map[string]string)} + p := &Plugin{ + id, + executablePath, + "usr/share/holo/", + make(map[string]string), + } //check if the plugin executable exists _, err := os.Stat(executablePath) @@ -108,10 +115,17 @@ func (p *Plugin) ID() string { return p.id } +//SetResourceRoot changes the resource root to given path. +//Future calls to ResourceDirectory will return a path relative to +//given path. +func (p *Plugin) SetResourceRoot(path string) { + p.resourceRoot = path +} + //ResourceDirectory returns the path to the directory where this plugin may //find its resources (entity definitions etc.). func (p *Plugin) ResourceDirectory() string { - return filepath.Join(RootDirectory(), "usr/share/holo/"+p.id) + return filepath.Join(RootDirectory(), p.resourceRoot, p.id) } //CacheDirectory returns the path to the directory where this plugin may diff --git a/cmd/holo/main.go b/cmd/holo/main.go index dfb594b..dcbf8cb 100644 --- a/cmd/holo/main.go +++ b/cmd/holo/main.go @@ -104,6 +104,13 @@ func Main() (exitCode int) { selectors = append(selectors, &Selector{String: arg, Used: false}) } + //run generators before scan phase + if err := impl.RunGenerators(config); err != nil { + impl.Errorf(impl.Stderr, + "Failed to process generators: %s", err.Error(), + ) + } + //ask all plugins to scan for entities var entities []*impl.Entity for _, plugin := range config.Plugins { diff --git a/doc/holo-generators.7.pod b/doc/holo-generators.7.pod new file mode 100644 index 0000000..a519fbd --- /dev/null +++ b/doc/holo-generators.7.pod @@ -0,0 +1,56 @@ +=encoding UTF-8 + +=head1 NAME + +holo-generators - dynamically generate files for other plugins + +=head1 DESCRIPTION + +Generators are executable files placed under +F. +Holo will execute generators in lexical order before asking plugins +to scan their files. +Upon execution of a generator the environment variable C<$OUT> is +passed to it. +Unless for caching purposes generators MUST only write to the +directory specified by C<$OUT>. +The structure under C<$OUT> is the same as for static files under +F. +For example to generate files for the holo-files plugin a generator +MAY place files at F<$OUT/files>. + +Generated files placed under C<$OUT> and static files under +F will be made available for plugins, +while generated files take precedence over static files. +This means if a generator places a file at +F<$OUT/files/20-webserver/etc/nginx/nginx.conf> +the plugin holo-files will only see this file insteadof the file at +F. + +Files that have been written to C<$OUT> will be preserved between +runs. +A generator MAY decide wether to overwrite existing files or not +this includes files written by other generators. +It is RECOMMENDED that generators only regenerate files if an output +different from the previous run is expected. + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", +"SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in +this document are to be interpreted as described in +L. + +=head1 SEE ALSO + +L + +L. + +=head1 AUTHOR + +Peter Werner + +Further documentation is available at the project homepage: http://holocm.org + +Please report any issues and feature requests at GitHub: http://github.com/holocm/holo/issues + +=cut diff --git a/test/generators/01-basic/README.md b/test/generators/01-basic/README.md new file mode 100644 index 0000000..f3ac829 --- /dev/null +++ b/test/generators/01-basic/README.md @@ -0,0 +1,8 @@ +Basic tests for the generators feature. +The following cases are covered: +- Generator files are executed and generated files are deployed + into the respective folder. +- The folder is passed correctly to plugin. +- Generators are executed in alphabetical order. +- Generators with non-zero exit code are are mentioned in the output + and do not stall the execution. \ No newline at end of file diff --git a/test/generators/01-basic/expected-apply-output b/test/generators/01-basic/expected-apply-output new file mode 100644 index 0000000..d5b3182 --- /dev/null +++ b/test/generators/01-basic/expected-apply-output @@ -0,0 +1,10 @@ +Ran generator 02-failing.sh + found at target/usr/share/holo/generators/02-failing.sh + +02-failing.sh: failing +!! exit status 1 + +Printing env:target/var/tmp/holo/generated/print +found at HOLO_RESOURCE_DIR + +exit status 0 diff --git a/test/generators/01-basic/expected-diff-output b/test/generators/01-basic/expected-diff-output new file mode 100644 index 0000000..ee2f8e7 --- /dev/null +++ b/test/generators/01-basic/expected-diff-output @@ -0,0 +1,7 @@ +Ran generator 02-failing.sh + found at target/usr/share/holo/generators/02-failing.sh + +02-failing.sh: failing +!! exit status 1 + +exit status 0 diff --git a/test/generators/01-basic/expected-scan-output b/test/generators/01-basic/expected-scan-output new file mode 100644 index 0000000..487cbcd --- /dev/null +++ b/test/generators/01-basic/expected-scan-output @@ -0,0 +1,10 @@ +Ran generator 02-failing.sh + found at target/usr/share/holo/generators/02-failing.sh + +02-failing.sh: failing +!! exit status 1 + +env:target/var/tmp/holo/generated/print + found at HOLO_RESOURCE_DIR + +exit status 0 diff --git a/test/generators/01-basic/expected-tree b/test/generators/01-basic/expected-tree new file mode 100644 index 0000000..45d2af0 --- /dev/null +++ b/test/generators/01-basic/expected-tree @@ -0,0 +1,28 @@ +symlink 0777 ./etc/holorc +../../../holorc +---------------------------------------- +directory 0755 ./run/ +---------------------------------------- +directory 0755 ./tmp/ +---------------------------------------- +file 0755 ./usr/share/holo/generators/01-simple.sh +#!/bin/sh +mkdir -p "${OUT}/print/" +echo "Simple generated file" > "${OUT}/print/file.txt" +---------------------------------------- +file 0755 ./usr/share/holo/generators/02-failing.sh +#!/bin/sh +echo "02-failing.sh: failing" +exit 1 +---------------------------------------- +directory 0755 ./usr/share/holo/print/ +---------------------------------------- +directory 0755 ./var/lib/holo/files/base/ +---------------------------------------- +directory 0755 ./var/lib/holo/files/provisioned/ +---------------------------------------- +directory 0755 ./var/lib/holo/print/ +---------------------------------------- +file 0644 ./var/tmp/holo/generated/print/file.txt +Simple generated file +---------------------------------------- diff --git a/test/generators/01-basic/source-tree b/test/generators/01-basic/source-tree new file mode 100644 index 0000000..159d491 --- /dev/null +++ b/test/generators/01-basic/source-tree @@ -0,0 +1,23 @@ +symlink 0777 ./etc/holorc +../../../holorc +---------------------------------------- +directory 0755 ./run/ +---------------------------------------- +directory 0755 ./tmp/ +---------------------------------------- +directory 0755 ./usr/share/holo/print +---------------------------------------- +file 0755 ./usr/share/holo/generators/01-simple.sh +#!/bin/sh +mkdir -p "${OUT}/print/" +echo "Simple generated file" > "${OUT}/print/file.txt" +---------------------------------------- +file 0755 ./usr/share/holo/generators/02-failing.sh +#!/bin/sh +echo "02-failing.sh: failing" +exit 1 +---------------------------------------- +directory 0755 ./var/lib/holo/files/base/ +---------------------------------------- +directory 0755 ./var/lib/holo/files/provisioned/ +---------------------------------------- diff --git a/test/generators/02-symlink-static/README.md b/test/generators/02-symlink-static/README.md new file mode 100644 index 0000000..af9f1b0 --- /dev/null +++ b/test/generators/02-symlink-static/README.md @@ -0,0 +1,2 @@ +Test whether existing static files are symlinked correctly into +the temporary dir passed to plugins. \ No newline at end of file diff --git a/test/generators/02-symlink-static/expected-apply-output b/test/generators/02-symlink-static/expected-apply-output new file mode 100644 index 0000000..a0052ea --- /dev/null +++ b/test/generators/02-symlink-static/expected-apply-output @@ -0,0 +1,5 @@ + +Printing env:target/var/tmp/holo/generated/print +found at HOLO_RESOURCE_DIR + +exit status 0 diff --git a/test/generators/02-symlink-static/expected-diff-output b/test/generators/02-symlink-static/expected-diff-output new file mode 100644 index 0000000..39a9383 --- /dev/null +++ b/test/generators/02-symlink-static/expected-diff-output @@ -0,0 +1 @@ +exit status 0 diff --git a/test/generators/02-symlink-static/expected-scan-output b/test/generators/02-symlink-static/expected-scan-output new file mode 100644 index 0000000..cb45210 --- /dev/null +++ b/test/generators/02-symlink-static/expected-scan-output @@ -0,0 +1,5 @@ + +env:target/var/tmp/holo/generated/print + found at HOLO_RESOURCE_DIR + +exit status 0 diff --git a/test/generators/02-symlink-static/expected-tree b/test/generators/02-symlink-static/expected-tree new file mode 100644 index 0000000..2e3f5a5 --- /dev/null +++ b/test/generators/02-symlink-static/expected-tree @@ -0,0 +1,36 @@ +symlink 0777 ./etc/holorc +../../../holorc +---------------------------------------- +directory 0755 ./run/ +---------------------------------------- +directory 0755 ./tmp/ +---------------------------------------- +file 0755 ./usr/share/holo/generators/01-simple.sh +#!/bin/sh +mkdir -p "${OUT}/print/" +echo "Simple generated file" > "${OUT}/print/file.txt" +---------------------------------------- +file 0755 ./usr/share/holo/print/dir/static1.txt +Simple static file 1 +---------------------------------------- +file 0755 ./usr/share/holo/print/dir/static2.txt +Simple static file 2 +---------------------------------------- +file 0755 ./usr/share/holo/print/static.txt +Simple static file +---------------------------------------- +directory 0755 ./var/lib/holo/files/base/ +---------------------------------------- +directory 0755 ./var/lib/holo/files/provisioned/ +---------------------------------------- +directory 0755 ./var/lib/holo/print/ +---------------------------------------- +symlink 0777 ./var/tmp/holo/generated/print/dir +target/usr/share/holo/print/dir +---------------------------------------- +file 0644 ./var/tmp/holo/generated/print/file.txt +Simple generated file +---------------------------------------- +symlink 0777 ./var/tmp/holo/generated/print/static.txt +target/usr/share/holo/print/static.txt +---------------------------------------- diff --git a/test/generators/02-symlink-static/source-tree b/test/generators/02-symlink-static/source-tree new file mode 100644 index 0000000..3b884ae --- /dev/null +++ b/test/generators/02-symlink-static/source-tree @@ -0,0 +1,27 @@ +symlink 0777 ./etc/holorc +../../../holorc +---------------------------------------- +directory 0755 ./run/ +---------------------------------------- +directory 0755 ./tmp/ +---------------------------------------- +file 0755 ./usr/share/holo/print/dir/static1.txt +Simple static file 1 +---------------------------------------- +file 0755 ./usr/share/holo/print/dir/static2.txt +Simple static file 2 +---------------------------------------- +file 0755 ./usr/share/holo/print/static.txt +Simple static file +---------------------------------------- +file 0755 ./usr/share/holo/generators/01-simple.sh +#!/bin/sh +mkdir -p "${OUT}/print/" +echo "Simple generated file" > "${OUT}/print/file.txt" +---------------------------------------- +directory 0755 ./var/lib/holo/files/base/ +---------------------------------------- +directory 0755 ./var/lib/holo/files/provisioned/ +---------------------------------------- +directory 0755 ./var/tmp/holo/generated/ +---------------------------------------- diff --git a/test/generators/03-conflict/README.md b/test/generators/03-conflict/README.md new file mode 100644 index 0000000..b5e880f --- /dev/null +++ b/test/generators/03-conflict/README.md @@ -0,0 +1,3 @@ +Test whether generated files take precedence over static files +as expected (if both a static and a generated version of the same +file exist the generated should be preferred). \ No newline at end of file diff --git a/test/generators/03-conflict/expected-apply-output b/test/generators/03-conflict/expected-apply-output new file mode 100644 index 0000000..a0052ea --- /dev/null +++ b/test/generators/03-conflict/expected-apply-output @@ -0,0 +1,5 @@ + +Printing env:target/var/tmp/holo/generated/print +found at HOLO_RESOURCE_DIR + +exit status 0 diff --git a/test/generators/03-conflict/expected-diff-output b/test/generators/03-conflict/expected-diff-output new file mode 100644 index 0000000..39a9383 --- /dev/null +++ b/test/generators/03-conflict/expected-diff-output @@ -0,0 +1 @@ +exit status 0 diff --git a/test/generators/03-conflict/expected-scan-output b/test/generators/03-conflict/expected-scan-output new file mode 100644 index 0000000..cb45210 --- /dev/null +++ b/test/generators/03-conflict/expected-scan-output @@ -0,0 +1,5 @@ + +env:target/var/tmp/holo/generated/print + found at HOLO_RESOURCE_DIR + +exit status 0 diff --git a/test/generators/03-conflict/expected-tree b/test/generators/03-conflict/expected-tree new file mode 100644 index 0000000..997d175 --- /dev/null +++ b/test/generators/03-conflict/expected-tree @@ -0,0 +1,24 @@ +symlink 0777 ./etc/holorc +../../../holorc +---------------------------------------- +directory 0755 ./run/ +---------------------------------------- +directory 0755 ./tmp/ +---------------------------------------- +file 0755 ./usr/share/holo/generators/01-simple.sh +#!/bin/sh +mkdir -p "${OUT}/print/" +echo "Simple generated file" > "${OUT}/print/file.txt" +---------------------------------------- +file 0755 ./usr/share/holo/print/file.txt +Simple static file +---------------------------------------- +directory 0755 ./var/lib/holo/files/base/ +---------------------------------------- +directory 0755 ./var/lib/holo/files/provisioned/ +---------------------------------------- +directory 0755 ./var/lib/holo/print/ +---------------------------------------- +file 0644 ./var/tmp/holo/generated/print/file.txt +Simple generated file +---------------------------------------- diff --git a/test/generators/03-conflict/source-tree b/test/generators/03-conflict/source-tree new file mode 100644 index 0000000..48447d0 --- /dev/null +++ b/test/generators/03-conflict/source-tree @@ -0,0 +1,19 @@ +symlink 0777 ./etc/holorc +../../../holorc +---------------------------------------- +directory 0755 ./run/ +---------------------------------------- +directory 0755 ./tmp/ +---------------------------------------- +file 0755 ./usr/share/holo/print/file.txt +Simple static file +---------------------------------------- +file 0755 ./usr/share/holo/generators/01-simple.sh +#!/bin/sh +mkdir -p "${OUT}/print/" +echo "Simple generated file" > "${OUT}/print/file.txt" +---------------------------------------- +directory 0755 ./var/lib/holo/files/base/ +---------------------------------------- +directory 0755 ./var/lib/holo/files/provisioned/ +---------------------------------------- diff --git a/test/generators/echo_plugin.sh b/test/generators/echo_plugin.sh new file mode 100755 index 0000000..2cf7840 --- /dev/null +++ b/test/generators/echo_plugin.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +case "$1" in + info) + echo MIN_API_VERSION=3 + echo MAX_API_VERSION=3 + ;; + scan) + # list executables in $HOLO_RESOURCE_DIR + echo "ENTITY: env:$HOLO_RESOURCE_DIR" + echo "ACTION: Printing" + echo "found at: HOLO_RESOURCE_DIR" + echo "SOURCE: $HOLO_RESOURCE_DIR/$FILENAME" + ;; + diff) + ;; + apply|force-apply) + ;; + *) + exit 1 + ;; +esac \ No newline at end of file diff --git a/test/generators/holorc b/test/generators/holorc new file mode 100644 index 0000000..87bb75c --- /dev/null +++ b/test/generators/holorc @@ -0,0 +1 @@ +plugin print=../echo_plugin.sh \ No newline at end of file