From 27c83b5256dc91d03ca8b68993342b618aa879e7 Mon Sep 17 00:00:00 2001 From: Peter Werner Date: Fri, 19 Jun 2020 08:58:15 +0200 Subject: [PATCH 1/5] test: basics of generator feature. --- Makefile | 2 +- test/generators/01-basic/README.md | 8 +++++ .../generators/01-basic/expected-apply-output | 10 +++++++ test/generators/01-basic/expected-diff-output | 7 +++++ test/generators/01-basic/expected-scan-output | 10 +++++++ test/generators/01-basic/expected-tree | 29 +++++++++++++++++++ test/generators/01-basic/source-tree | 24 +++++++++++++++ test/generators/echo_plugin.sh | 22 ++++++++++++++ test/generators/holorc | 1 + 9 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 test/generators/01-basic/README.md create mode 100644 test/generators/01-basic/expected-apply-output create mode 100644 test/generators/01-basic/expected-diff-output create mode 100644 test/generators/01-basic/expected-scan-output create mode 100644 test/generators/01-basic/expected-tree create mode 100644 test/generators/01-basic/source-tree create mode 100755 test/generators/echo_plugin.sh create mode 100644 test/generators/holorc diff --git a/Makefile b/Makefile index b9d63e3..91edbc8 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ test/cov.cov: clean-tests build/holo.test export HOLO_BINARY=../../../build/holo.test && \ export HOLO_TEST_COVERDIR=$(abspath test/cov) && \ export HOLO_TEST_SCRIPTPATH=../../../util && \ - $(foreach p,files run-scripts ssh-keys users-groups,\ + $(foreach p,files run-scripts ssh-keys users-groups generators,\ ln -sfT ../build/holo.test test/holo-$p && \ ./util/holo-test holo-$p $(sort $(wildcard test/$p/??-*)) && ) \ true diff --git a/test/generators/01-basic/README.md b/test/generators/01-basic/README.md new file mode 100644 index 0000000..f3ac829 --- /dev/null +++ b/test/generators/01-basic/README.md @@ -0,0 +1,8 @@ +Basic tests for the generators feature. +The following cases are covered: +- Generator files are executed and generated files are deployed + into the respective folder. +- The folder is passed correctly to plugin. +- Generators are executed in alphabetical order. +- Generators with non-zero exit code are are mentioned in the output + and do not stall the execution. \ No newline at end of file diff --git a/test/generators/01-basic/expected-apply-output b/test/generators/01-basic/expected-apply-output new file mode 100644 index 0000000..974633d --- /dev/null +++ b/test/generators/01-basic/expected-apply-output @@ -0,0 +1,10 @@ + +Processing generators... +target/usr/share/holo/generators/01-simple.sh +target/usr/share/holo/generators/02-failing.sh +!! Running generator failed: exit status 1 + +Printing env:target/var/tmp/holo/generated/print +found at HOLO_RESOURCE_DIR + +exit status 0 diff --git a/test/generators/01-basic/expected-diff-output b/test/generators/01-basic/expected-diff-output new file mode 100644 index 0000000..a3ebc0f --- /dev/null +++ b/test/generators/01-basic/expected-diff-output @@ -0,0 +1,7 @@ + +Processing generators... +target/usr/share/holo/generators/01-simple.sh +target/usr/share/holo/generators/02-failing.sh +!! Running generator failed: exit status 1 + +exit status 0 diff --git a/test/generators/01-basic/expected-scan-output b/test/generators/01-basic/expected-scan-output new file mode 100644 index 0000000..eee970a --- /dev/null +++ b/test/generators/01-basic/expected-scan-output @@ -0,0 +1,10 @@ + +Processing generators... +target/usr/share/holo/generators/01-simple.sh +target/usr/share/holo/generators/02-failing.sh +!! Running generator failed: exit status 1 + +env:target/var/tmp/holo/generated/print + found at HOLO_RESOURCE_DIR + +exit status 0 diff --git a/test/generators/01-basic/expected-tree b/test/generators/01-basic/expected-tree new file mode 100644 index 0000000..24412f0 --- /dev/null +++ b/test/generators/01-basic/expected-tree @@ -0,0 +1,29 @@ +symlink 0777 ./etc/holorc +../../../holorc +---------------------------------------- +directory 0755 ./run/ +---------------------------------------- +directory 0755 ./tmp/ +---------------------------------------- +file 0755 ./usr/share/holo/generators/01-simple.sh +#!/bin/sh +echo "Running generator 01-simple.sh" +mkdir "${OUT}/print/" +echo "Simple generated file" > "${OUT}/print/file.txt" +---------------------------------------- +file 0755 ./usr/share/holo/generators/02-failing.sh +#!/bin/sh +echo "Running generator 02-failing.sh" +exit 1 +---------------------------------------- +directory 0755 ./usr/share/holo/print/ +---------------------------------------- +directory 0755 ./var/lib/holo/files/base/ +---------------------------------------- +directory 0755 ./var/lib/holo/files/provisioned/ +---------------------------------------- +directory 0755 ./var/lib/holo/print/ +---------------------------------------- +file 0644 ./var/tmp/holo/generated/print/file.txt +Simple generated file +---------------------------------------- diff --git a/test/generators/01-basic/source-tree b/test/generators/01-basic/source-tree new file mode 100644 index 0000000..f1fd39f --- /dev/null +++ b/test/generators/01-basic/source-tree @@ -0,0 +1,24 @@ +symlink 0777 ./etc/holorc +../../../holorc +---------------------------------------- +directory 0755 ./run/ +---------------------------------------- +directory 0755 ./tmp/ +---------------------------------------- +directory 0755 ./usr/share/holo/print +---------------------------------------- +file 0755 ./usr/share/holo/generators/01-simple.sh +#!/bin/sh +echo "Running generator 01-simple.sh" +mkdir "${OUT}/print/" +echo "Simple generated file" > "${OUT}/print/file.txt" +---------------------------------------- +file 0755 ./usr/share/holo/generators/02-failing.sh +#!/bin/sh +echo "Running generator 02-failing.sh" +exit 1 +---------------------------------------- +directory 0755 ./var/lib/holo/files/base/ +---------------------------------------- +directory 0755 ./var/lib/holo/files/provisioned/ +---------------------------------------- diff --git a/test/generators/echo_plugin.sh b/test/generators/echo_plugin.sh new file mode 100755 index 0000000..2cf7840 --- /dev/null +++ b/test/generators/echo_plugin.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +case "$1" in + info) + echo MIN_API_VERSION=3 + echo MAX_API_VERSION=3 + ;; + scan) + # list executables in $HOLO_RESOURCE_DIR + echo "ENTITY: env:$HOLO_RESOURCE_DIR" + echo "ACTION: Printing" + echo "found at: HOLO_RESOURCE_DIR" + echo "SOURCE: $HOLO_RESOURCE_DIR/$FILENAME" + ;; + diff) + ;; + apply|force-apply) + ;; + *) + exit 1 + ;; +esac \ No newline at end of file diff --git a/test/generators/holorc b/test/generators/holorc new file mode 100644 index 0000000..87bb75c --- /dev/null +++ b/test/generators/holorc @@ -0,0 +1 @@ +plugin print=../echo_plugin.sh \ No newline at end of file From 0edaa071dc75d2e35965e0487f52a243ef2b9971 Mon Sep 17 00:00:00 2001 From: Peter Werner Date: Fri, 19 Jun 2020 08:59:24 +0200 Subject: [PATCH 2/5] test: symlinking static files to generated dir. --- test/generators/02-symlink-static/README.md | 2 + .../02-symlink-static/expected-apply-output | 8 ++++ .../02-symlink-static/expected-diff-output | 5 +++ .../02-symlink-static/expected-scan-output | 8 ++++ .../02-symlink-static/expected-tree | 37 +++++++++++++++++++ test/generators/02-symlink-static/source-tree | 28 ++++++++++++++ 6 files changed, 88 insertions(+) create mode 100644 test/generators/02-symlink-static/README.md create mode 100644 test/generators/02-symlink-static/expected-apply-output create mode 100644 test/generators/02-symlink-static/expected-diff-output create mode 100644 test/generators/02-symlink-static/expected-scan-output create mode 100644 test/generators/02-symlink-static/expected-tree create mode 100644 test/generators/02-symlink-static/source-tree diff --git a/test/generators/02-symlink-static/README.md b/test/generators/02-symlink-static/README.md new file mode 100644 index 0000000..af9f1b0 --- /dev/null +++ b/test/generators/02-symlink-static/README.md @@ -0,0 +1,2 @@ +Test whether existing static files are symlinked correctly into +the temporary dir passed to plugins. \ No newline at end of file diff --git a/test/generators/02-symlink-static/expected-apply-output b/test/generators/02-symlink-static/expected-apply-output new file mode 100644 index 0000000..ff8e200 --- /dev/null +++ b/test/generators/02-symlink-static/expected-apply-output @@ -0,0 +1,8 @@ + +Processing generators... +target/usr/share/holo/generators/01-simple.sh + +Printing env:target/var/tmp/holo/generated/print +found at HOLO_RESOURCE_DIR + +exit status 0 diff --git a/test/generators/02-symlink-static/expected-diff-output b/test/generators/02-symlink-static/expected-diff-output new file mode 100644 index 0000000..857b787 --- /dev/null +++ b/test/generators/02-symlink-static/expected-diff-output @@ -0,0 +1,5 @@ + +Processing generators... +target/usr/share/holo/generators/01-simple.sh + +exit status 0 diff --git a/test/generators/02-symlink-static/expected-scan-output b/test/generators/02-symlink-static/expected-scan-output new file mode 100644 index 0000000..3cc16b5 --- /dev/null +++ b/test/generators/02-symlink-static/expected-scan-output @@ -0,0 +1,8 @@ + +Processing generators... +target/usr/share/holo/generators/01-simple.sh + +env:target/var/tmp/holo/generated/print + found at HOLO_RESOURCE_DIR + +exit status 0 diff --git a/test/generators/02-symlink-static/expected-tree b/test/generators/02-symlink-static/expected-tree new file mode 100644 index 0000000..acbdb3d --- /dev/null +++ b/test/generators/02-symlink-static/expected-tree @@ -0,0 +1,37 @@ +symlink 0777 ./etc/holorc +../../../holorc +---------------------------------------- +directory 0755 ./run/ +---------------------------------------- +directory 0755 ./tmp/ +---------------------------------------- +file 0755 ./usr/share/holo/generators/01-simple.sh +#!/bin/sh +echo "Running generator 01-simple.sh" +mkdir "${OUT}/print/" +echo "Simple generated file" > "${OUT}/print/file.txt" +---------------------------------------- +file 0755 ./usr/share/holo/print/dir/static1.txt +Simple static file 1 +---------------------------------------- +file 0755 ./usr/share/holo/print/dir/static2.txt +Simple static file 2 +---------------------------------------- +file 0755 ./usr/share/holo/print/static.txt +Simple static file +---------------------------------------- +directory 0755 ./var/lib/holo/files/base/ +---------------------------------------- +directory 0755 ./var/lib/holo/files/provisioned/ +---------------------------------------- +directory 0755 ./var/lib/holo/print/ +---------------------------------------- +symlink 0777 ./var/tmp/holo/generated/print/dir +target/usr/share/holo/print/dir +---------------------------------------- +file 0644 ./var/tmp/holo/generated/print/file.txt +Simple generated file +---------------------------------------- +symlink 0777 ./var/tmp/holo/generated/print/static.txt +target/usr/share/holo/print/static.txt +---------------------------------------- diff --git a/test/generators/02-symlink-static/source-tree b/test/generators/02-symlink-static/source-tree new file mode 100644 index 0000000..23d2993 --- /dev/null +++ b/test/generators/02-symlink-static/source-tree @@ -0,0 +1,28 @@ +symlink 0777 ./etc/holorc +../../../holorc +---------------------------------------- +directory 0755 ./run/ +---------------------------------------- +directory 0755 ./tmp/ +---------------------------------------- +file 0755 ./usr/share/holo/print/dir/static1.txt +Simple static file 1 +---------------------------------------- +file 0755 ./usr/share/holo/print/dir/static2.txt +Simple static file 2 +---------------------------------------- +file 0755 ./usr/share/holo/print/static.txt +Simple static file +---------------------------------------- +file 0755 ./usr/share/holo/generators/01-simple.sh +#!/bin/sh +echo "Running generator 01-simple.sh" +mkdir "${OUT}/print/" +echo "Simple generated file" > "${OUT}/print/file.txt" +---------------------------------------- +directory 0755 ./var/lib/holo/files/base/ +---------------------------------------- +directory 0755 ./var/lib/holo/files/provisioned/ +---------------------------------------- +directory 0755 ./var/tmp/holo/generated/ +---------------------------------------- From 66a77dfbfce149fdf0e36a21abbb70e76de3cdb6 Mon Sep 17 00:00:00 2001 From: Peter Werner Date: Fri, 19 Jun 2020 09:00:41 +0200 Subject: [PATCH 3/5] test: conflicts static/generated files. --- test/generators/03-conflict/README.md | 3 +++ .../03-conflict/expected-apply-output | 8 ++++++ .../03-conflict/expected-diff-output | 5 ++++ .../03-conflict/expected-scan-output | 8 ++++++ test/generators/03-conflict/expected-tree | 25 +++++++++++++++++++ test/generators/03-conflict/source-tree | 20 +++++++++++++++ 6 files changed, 69 insertions(+) create mode 100644 test/generators/03-conflict/README.md create mode 100644 test/generators/03-conflict/expected-apply-output create mode 100644 test/generators/03-conflict/expected-diff-output create mode 100644 test/generators/03-conflict/expected-scan-output create mode 100644 test/generators/03-conflict/expected-tree create mode 100644 test/generators/03-conflict/source-tree diff --git a/test/generators/03-conflict/README.md b/test/generators/03-conflict/README.md new file mode 100644 index 0000000..b5e880f --- /dev/null +++ b/test/generators/03-conflict/README.md @@ -0,0 +1,3 @@ +Test whether generated files take precedence over static files +as expected (if both a static and a generated version of the same +file exist the generated should be preferred). \ No newline at end of file diff --git a/test/generators/03-conflict/expected-apply-output b/test/generators/03-conflict/expected-apply-output new file mode 100644 index 0000000..ff8e200 --- /dev/null +++ b/test/generators/03-conflict/expected-apply-output @@ -0,0 +1,8 @@ + +Processing generators... +target/usr/share/holo/generators/01-simple.sh + +Printing env:target/var/tmp/holo/generated/print +found at HOLO_RESOURCE_DIR + +exit status 0 diff --git a/test/generators/03-conflict/expected-diff-output b/test/generators/03-conflict/expected-diff-output new file mode 100644 index 0000000..857b787 --- /dev/null +++ b/test/generators/03-conflict/expected-diff-output @@ -0,0 +1,5 @@ + +Processing generators... +target/usr/share/holo/generators/01-simple.sh + +exit status 0 diff --git a/test/generators/03-conflict/expected-scan-output b/test/generators/03-conflict/expected-scan-output new file mode 100644 index 0000000..3cc16b5 --- /dev/null +++ b/test/generators/03-conflict/expected-scan-output @@ -0,0 +1,8 @@ + +Processing generators... +target/usr/share/holo/generators/01-simple.sh + +env:target/var/tmp/holo/generated/print + found at HOLO_RESOURCE_DIR + +exit status 0 diff --git a/test/generators/03-conflict/expected-tree b/test/generators/03-conflict/expected-tree new file mode 100644 index 0000000..0874943 --- /dev/null +++ b/test/generators/03-conflict/expected-tree @@ -0,0 +1,25 @@ +symlink 0777 ./etc/holorc +../../../holorc +---------------------------------------- +directory 0755 ./run/ +---------------------------------------- +directory 0755 ./tmp/ +---------------------------------------- +file 0755 ./usr/share/holo/generators/01-simple.sh +#!/bin/sh +echo "Running generator 01-simple.sh" +mkdir "${OUT}/print/" +echo "Simple generated file" > "${OUT}/print/file.txt" +---------------------------------------- +file 0755 ./usr/share/holo/print/file.txt +Simple static file +---------------------------------------- +directory 0755 ./var/lib/holo/files/base/ +---------------------------------------- +directory 0755 ./var/lib/holo/files/provisioned/ +---------------------------------------- +directory 0755 ./var/lib/holo/print/ +---------------------------------------- +file 0644 ./var/tmp/holo/generated/print/file.txt +Simple generated file +---------------------------------------- diff --git a/test/generators/03-conflict/source-tree b/test/generators/03-conflict/source-tree new file mode 100644 index 0000000..91a4d89 --- /dev/null +++ b/test/generators/03-conflict/source-tree @@ -0,0 +1,20 @@ +symlink 0777 ./etc/holorc +../../../holorc +---------------------------------------- +directory 0755 ./run/ +---------------------------------------- +directory 0755 ./tmp/ +---------------------------------------- +file 0755 ./usr/share/holo/print/file.txt +Simple static file +---------------------------------------- +file 0755 ./usr/share/holo/generators/01-simple.sh +#!/bin/sh +echo "Running generator 01-simple.sh" +mkdir "${OUT}/print/" +echo "Simple generated file" > "${OUT}/print/file.txt" +---------------------------------------- +directory 0755 ./var/lib/holo/files/base/ +---------------------------------------- +directory 0755 ./var/lib/holo/files/provisioned/ +---------------------------------------- From cca5c11b556bbfc0fe04e8842b883979256a8eef Mon Sep 17 00:00:00 2001 From: Peter Werner Date: Sat, 8 Aug 2020 12:57:30 +0200 Subject: [PATCH 4/5] feat: add generators Generators are executed before the scan phase and can generated file for other plugins. --- README.md | 1 + cmd/holo/internal/generators.go | 173 ++++++++++++++++++++++++++++++++ cmd/holo/internal/plugin.go | 20 +++- cmd/holo/main.go | 7 ++ doc/holo-generators.7.pod | 56 +++++++++++ 5 files changed, 254 insertions(+), 3 deletions(-) create mode 100644 cmd/holo/internal/generators.go create mode 100644 doc/holo-generators.7.pod diff --git a/README.md b/README.md index 3511498..3ff0b01 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ User documentation is available in man page form: * [holo-users-groups(8)](doc/holo-users-groups.8.pod) * [holorc(5)](doc/holorc.5.pod) * [holo-plugin-interface(7)](doc/holo-plugin-interface.7.pod) +* [holo-generators(7)](doc/holo-generators.7.pod) * [holo-test(7)](doc/holo-test.7.pod) (not a public interface) For further information, visit [holocm.org](http://holocm.org). diff --git a/cmd/holo/internal/generators.go b/cmd/holo/internal/generators.go new file mode 100644 index 0000000..a64dfe0 --- /dev/null +++ b/cmd/holo/internal/generators.go @@ -0,0 +1,173 @@ +/******************************************************************************* +* +* Copyright 2020 Peter Werner +* +* This file is part of Holo. +* +* Holo is free software: you can redistribute it and/or modify it under the +* terms of the GNU General Public License as published by the Free Software +* Foundation, either version 3 of the License, or (at your option) any later +* version. +* +* Holo is distributed in the hope that it will be useful, but WITHOUT ANY +* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +* A PARTICULAR PURPOSE. See the GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License along with +* Holo. If not, see . +* +*******************************************************************************/ + +package impl + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" +) + +// RunGenerators executes all generators in the generator directory +// and changes the resource path of plugins for which files were +// generated to. +func RunGenerators(config *Configuration) error { + inputDir := getGenertorsDir() + if _, err := os.Stat(inputDir); err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + targetDir, err := getGeneratorCacheDir() + if err != nil { + return fmt.Errorf( + "couldn't access cache-dir ('%s') for generators: %s", + targetDir, err, + ) + } + fmt.Fprintln(Stdout, "Processing generators...") + filepath.Walk(inputDir, + func(path string, info os.FileInfo, err error) error { + if err != nil { + Warnf(Stderr, "%s: %s", path, err.Error()) + return nil + } + if isExecutableFile(info) { + fmt.Fprintln(Stdout, path) + if err = runGenerator(path, targetDir); err != nil { + Errorf( + Stderr, + "Running generator failed: %s", err.Error(), + ) + } + } + return nil + }) + for _, plugin := range config.Plugins { + if err := updatePluginPaths(plugin, targetDir); err != nil { + Errorf(Stderr, + "Failed to perpare generated dir for plugin '%s': %s", + plugin.id, err.Error(), + ) + } + } + return nil +} + +func updatePluginPaths(plugin *Plugin, dir string) error { + pluginDir := plugin.ResourceDirectory() + newPluginDir := filepath.Join(dir, plugin.id) + if info, err := os.Stat(newPluginDir); err == nil && info.IsDir() { + // Files were generated for this plugin. + // Fill the plugins directory with existsing static files. + if err := symlinkFiles(pluginDir, newPluginDir); err != nil { + return err + } + // Change the plugin resource dir to point to the generated dir. + resource, _ := filepath.Rel(RootDirectory(), dir) + plugin.SetResourceRoot(resource) + } + return nil +} + +func symlinkFiles(oldDir string, newDir string) error { + return filepath.Walk(oldDir, + func(oldFile string, info os.FileInfo, err error) error { + if err != nil || oldFile == oldDir { + return err + } + relPath, _ := filepath.Rel(oldDir, oldFile) + newFile := filepath.Join(newDir, relPath) + err = os.Symlink(oldFile, newFile) + if os.IsExist(err) { + // newFile already exists. Examine it. + newFileInfo, err := os.Lstat(newFile) + if err == nil && info.IsDir() && !newFileInfo.IsDir() { + // newFile exists but is not a directory. + // If oldFile is a directory trying to symlink its contents + // will result in errors. Skip it. + return filepath.SkipDir + } + return nil + } + if err != nil { + return err + } + if info.IsDir() { + // Symlink to dir was created don't check its contents. + return filepath.SkipDir + } + return nil + }) +} + +func runGenerator(fileToRun string, targetDir string) error { + cmd := exec.Command(fileToRun) + env := os.Environ() + env = append( + env, + fmt.Sprintf("OUT=%s", targetDir), + ) + cmd.Env = env + return cmd.Run() +} + +func getGenertorsDir() string { + return filepath.Join(RootDirectory(), "/usr/share/holo/generators") +} + +func getGeneratorCacheDir() (string, error) { + path, err := prepareDir(RootDirectory(), "/var/tmp/holo/generated") + if err == nil { + return path, nil + } + path, err = prepareDir( + os.Getenv("HOLO_CACHE_DIR"), "holo/generated", + ) + if err == nil { + return path, nil + } + return "", err +} + +func prepareDir(pathParts ...string) (string, error) { + path := filepath.Join(pathParts...) + if err := os.MkdirAll(path, 0755); err != nil { + if os.IsExist(err) { + return path, nil + } + return "", err + } + return path, nil +} + +func isExecutableFile(stat os.FileInfo) bool { + mode := stat.Mode() + if !mode.IsRegular() { + return false + } + if (mode & 0111) == 0 { + return false + } + return true +} diff --git a/cmd/holo/internal/plugin.go b/cmd/holo/internal/plugin.go index 40e7f60..28e95b0 100644 --- a/cmd/holo/internal/plugin.go +++ b/cmd/holo/internal/plugin.go @@ -43,7 +43,9 @@ var ErrPluginExecutableMissing = errors.New("ErrPluginExecutableMissing") type Plugin struct { id string executablePath string - metadata map[string]string //from "info" call + // Root path for the plugin specific resource dir + resourceRoot string + metadata map[string]string //from "info" call } //NewPlugin creates a new Plugin. @@ -56,7 +58,12 @@ func NewPlugin(id string) (*Plugin, error) { //a non-standard location. (This is used exclusively for testing plugins before //they are installed.) func NewPluginWithExecutablePath(id string, executablePath string) (*Plugin, error) { - p := &Plugin{id, executablePath, make(map[string]string)} + p := &Plugin{ + id, + executablePath, + "usr/share/holo/", + make(map[string]string), + } //check if the plugin executable exists _, err := os.Stat(executablePath) @@ -108,10 +115,17 @@ func (p *Plugin) ID() string { return p.id } +//SetResourceRoot changes the resource root to given path. +//Future calls to ResourceDirectory will return a path relative to +//given path. +func (p *Plugin) SetResourceRoot(path string) { + p.resourceRoot = path +} + //ResourceDirectory returns the path to the directory where this plugin may //find its resources (entity definitions etc.). func (p *Plugin) ResourceDirectory() string { - return filepath.Join(RootDirectory(), "usr/share/holo/"+p.id) + return filepath.Join(RootDirectory(), p.resourceRoot, p.id) } //CacheDirectory returns the path to the directory where this plugin may diff --git a/cmd/holo/main.go b/cmd/holo/main.go index dfb594b..dcbf8cb 100644 --- a/cmd/holo/main.go +++ b/cmd/holo/main.go @@ -104,6 +104,13 @@ func Main() (exitCode int) { selectors = append(selectors, &Selector{String: arg, Used: false}) } + //run generators before scan phase + if err := impl.RunGenerators(config); err != nil { + impl.Errorf(impl.Stderr, + "Failed to process generators: %s", err.Error(), + ) + } + //ask all plugins to scan for entities var entities []*impl.Entity for _, plugin := range config.Plugins { diff --git a/doc/holo-generators.7.pod b/doc/holo-generators.7.pod new file mode 100644 index 0000000..a519fbd --- /dev/null +++ b/doc/holo-generators.7.pod @@ -0,0 +1,56 @@ +=encoding UTF-8 + +=head1 NAME + +holo-generators - dynamically generate files for other plugins + +=head1 DESCRIPTION + +Generators are executable files placed under +F. +Holo will execute generators in lexical order before asking plugins +to scan their files. +Upon execution of a generator the environment variable C<$OUT> is +passed to it. +Unless for caching purposes generators MUST only write to the +directory specified by C<$OUT>. +The structure under C<$OUT> is the same as for static files under +F. +For example to generate files for the holo-files plugin a generator +MAY place files at F<$OUT/files>. + +Generated files placed under C<$OUT> and static files under +F will be made available for plugins, +while generated files take precedence over static files. +This means if a generator places a file at +F<$OUT/files/20-webserver/etc/nginx/nginx.conf> +the plugin holo-files will only see this file insteadof the file at +F. + +Files that have been written to C<$OUT> will be preserved between +runs. +A generator MAY decide wether to overwrite existing files or not +this includes files written by other generators. +It is RECOMMENDED that generators only regenerate files if an output +different from the previous run is expected. + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", +"SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in +this document are to be interpreted as described in +L. + +=head1 SEE ALSO + +L + +L. + +=head1 AUTHOR + +Peter Werner + +Further documentation is available at the project homepage: http://holocm.org + +Please report any issues and feature requests at GitHub: http://github.com/holocm/holo/issues + +=cut From dc9965ee4914cef07d2ea4c3fb7fad074bef49a5 Mon Sep 17 00:00:00 2001 From: Peter Werner Date: Sat, 19 Sep 2020 20:28:41 +0200 Subject: [PATCH 5/5] fix: Use properly styled output for generators --- cmd/holo/internal/generators.go | 44 +++++++++++-------- .../generators/01-basic/expected-apply-output | 8 ++-- test/generators/01-basic/expected-diff-output | 8 ++-- test/generators/01-basic/expected-scan-output | 8 ++-- test/generators/01-basic/expected-tree | 5 +-- test/generators/01-basic/source-tree | 5 +-- .../02-symlink-static/expected-apply-output | 3 -- .../02-symlink-static/expected-diff-output | 4 -- .../02-symlink-static/expected-scan-output | 3 -- .../02-symlink-static/expected-tree | 3 +- test/generators/02-symlink-static/source-tree | 3 +- .../03-conflict/expected-apply-output | 3 -- .../03-conflict/expected-diff-output | 4 -- .../03-conflict/expected-scan-output | 3 -- test/generators/03-conflict/expected-tree | 3 +- test/generators/03-conflict/source-tree | 3 +- 16 files changed, 46 insertions(+), 64 deletions(-) diff --git a/cmd/holo/internal/generators.go b/cmd/holo/internal/generators.go index a64dfe0..ce9b216 100644 --- a/cmd/holo/internal/generators.go +++ b/cmd/holo/internal/generators.go @@ -45,7 +45,19 @@ func RunGenerators(config *Configuration) error { targetDir, err, ) } - fmt.Fprintln(Stdout, "Processing generators...") + runGenerators(inputDir, targetDir) + for _, plugin := range config.Plugins { + if err := updatePluginPaths(plugin, targetDir); err != nil { + Errorf(Stderr, + "Failed to perpare generated dir for plugin '%s': %s", + plugin.id, err.Error(), + ) + } + } + return nil +} + +func runGenerators(inputDir string, targetDir string) { filepath.Walk(inputDir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -53,25 +65,21 @@ func RunGenerators(config *Configuration) error { return nil } if isExecutableFile(info) { - fmt.Fprintln(Stdout, path) - if err = runGenerator(path, targetDir); err != nil { - Errorf( - Stderr, - "Running generator failed: %s", err.Error(), - ) + out, err := runGenerator(path, targetDir) + // Keep silent unless an error occurred or generator has + // printed output. + if err != nil || len(out) > 0 { + shortPath, _ := filepath.Rel(inputDir, path) + fmt.Fprintf(os.Stdout, "Ran generator %s\n", shortPath) + fmt.Fprintf(os.Stdout, " found at %s\n", path) + Stdout.Write(out) + if err != nil { + Errorf(Stderr, err.Error()) + } } } return nil }) - for _, plugin := range config.Plugins { - if err := updatePluginPaths(plugin, targetDir); err != nil { - Errorf(Stderr, - "Failed to perpare generated dir for plugin '%s': %s", - plugin.id, err.Error(), - ) - } - } - return nil } func updatePluginPaths(plugin *Plugin, dir string) error { @@ -121,7 +129,7 @@ func symlinkFiles(oldDir string, newDir string) error { }) } -func runGenerator(fileToRun string, targetDir string) error { +func runGenerator(fileToRun string, targetDir string) ([]byte, error) { cmd := exec.Command(fileToRun) env := os.Environ() env = append( @@ -129,7 +137,7 @@ func runGenerator(fileToRun string, targetDir string) error { fmt.Sprintf("OUT=%s", targetDir), ) cmd.Env = env - return cmd.Run() + return cmd.CombinedOutput() } func getGenertorsDir() string { diff --git a/test/generators/01-basic/expected-apply-output b/test/generators/01-basic/expected-apply-output index 974633d..d5b3182 100644 --- a/test/generators/01-basic/expected-apply-output +++ b/test/generators/01-basic/expected-apply-output @@ -1,8 +1,8 @@ +Ran generator 02-failing.sh + found at target/usr/share/holo/generators/02-failing.sh -Processing generators... -target/usr/share/holo/generators/01-simple.sh -target/usr/share/holo/generators/02-failing.sh -!! Running generator failed: exit status 1 +02-failing.sh: failing +!! exit status 1 Printing env:target/var/tmp/holo/generated/print found at HOLO_RESOURCE_DIR diff --git a/test/generators/01-basic/expected-diff-output b/test/generators/01-basic/expected-diff-output index a3ebc0f..ee2f8e7 100644 --- a/test/generators/01-basic/expected-diff-output +++ b/test/generators/01-basic/expected-diff-output @@ -1,7 +1,7 @@ +Ran generator 02-failing.sh + found at target/usr/share/holo/generators/02-failing.sh -Processing generators... -target/usr/share/holo/generators/01-simple.sh -target/usr/share/holo/generators/02-failing.sh -!! Running generator failed: exit status 1 +02-failing.sh: failing +!! exit status 1 exit status 0 diff --git a/test/generators/01-basic/expected-scan-output b/test/generators/01-basic/expected-scan-output index eee970a..487cbcd 100644 --- a/test/generators/01-basic/expected-scan-output +++ b/test/generators/01-basic/expected-scan-output @@ -1,8 +1,8 @@ +Ran generator 02-failing.sh + found at target/usr/share/holo/generators/02-failing.sh -Processing generators... -target/usr/share/holo/generators/01-simple.sh -target/usr/share/holo/generators/02-failing.sh -!! Running generator failed: exit status 1 +02-failing.sh: failing +!! exit status 1 env:target/var/tmp/holo/generated/print found at HOLO_RESOURCE_DIR diff --git a/test/generators/01-basic/expected-tree b/test/generators/01-basic/expected-tree index 24412f0..45d2af0 100644 --- a/test/generators/01-basic/expected-tree +++ b/test/generators/01-basic/expected-tree @@ -7,13 +7,12 @@ directory 0755 ./tmp/ ---------------------------------------- file 0755 ./usr/share/holo/generators/01-simple.sh #!/bin/sh -echo "Running generator 01-simple.sh" -mkdir "${OUT}/print/" +mkdir -p "${OUT}/print/" echo "Simple generated file" > "${OUT}/print/file.txt" ---------------------------------------- file 0755 ./usr/share/holo/generators/02-failing.sh #!/bin/sh -echo "Running generator 02-failing.sh" +echo "02-failing.sh: failing" exit 1 ---------------------------------------- directory 0755 ./usr/share/holo/print/ diff --git a/test/generators/01-basic/source-tree b/test/generators/01-basic/source-tree index f1fd39f..159d491 100644 --- a/test/generators/01-basic/source-tree +++ b/test/generators/01-basic/source-tree @@ -9,13 +9,12 @@ directory 0755 ./usr/share/holo/print ---------------------------------------- file 0755 ./usr/share/holo/generators/01-simple.sh #!/bin/sh -echo "Running generator 01-simple.sh" -mkdir "${OUT}/print/" +mkdir -p "${OUT}/print/" echo "Simple generated file" > "${OUT}/print/file.txt" ---------------------------------------- file 0755 ./usr/share/holo/generators/02-failing.sh #!/bin/sh -echo "Running generator 02-failing.sh" +echo "02-failing.sh: failing" exit 1 ---------------------------------------- directory 0755 ./var/lib/holo/files/base/ diff --git a/test/generators/02-symlink-static/expected-apply-output b/test/generators/02-symlink-static/expected-apply-output index ff8e200..a0052ea 100644 --- a/test/generators/02-symlink-static/expected-apply-output +++ b/test/generators/02-symlink-static/expected-apply-output @@ -1,7 +1,4 @@ -Processing generators... -target/usr/share/holo/generators/01-simple.sh - Printing env:target/var/tmp/holo/generated/print found at HOLO_RESOURCE_DIR diff --git a/test/generators/02-symlink-static/expected-diff-output b/test/generators/02-symlink-static/expected-diff-output index 857b787..39a9383 100644 --- a/test/generators/02-symlink-static/expected-diff-output +++ b/test/generators/02-symlink-static/expected-diff-output @@ -1,5 +1 @@ - -Processing generators... -target/usr/share/holo/generators/01-simple.sh - exit status 0 diff --git a/test/generators/02-symlink-static/expected-scan-output b/test/generators/02-symlink-static/expected-scan-output index 3cc16b5..cb45210 100644 --- a/test/generators/02-symlink-static/expected-scan-output +++ b/test/generators/02-symlink-static/expected-scan-output @@ -1,7 +1,4 @@ -Processing generators... -target/usr/share/holo/generators/01-simple.sh - env:target/var/tmp/holo/generated/print found at HOLO_RESOURCE_DIR diff --git a/test/generators/02-symlink-static/expected-tree b/test/generators/02-symlink-static/expected-tree index acbdb3d..2e3f5a5 100644 --- a/test/generators/02-symlink-static/expected-tree +++ b/test/generators/02-symlink-static/expected-tree @@ -7,8 +7,7 @@ directory 0755 ./tmp/ ---------------------------------------- file 0755 ./usr/share/holo/generators/01-simple.sh #!/bin/sh -echo "Running generator 01-simple.sh" -mkdir "${OUT}/print/" +mkdir -p "${OUT}/print/" echo "Simple generated file" > "${OUT}/print/file.txt" ---------------------------------------- file 0755 ./usr/share/holo/print/dir/static1.txt diff --git a/test/generators/02-symlink-static/source-tree b/test/generators/02-symlink-static/source-tree index 23d2993..3b884ae 100644 --- a/test/generators/02-symlink-static/source-tree +++ b/test/generators/02-symlink-static/source-tree @@ -16,8 +16,7 @@ Simple static file ---------------------------------------- file 0755 ./usr/share/holo/generators/01-simple.sh #!/bin/sh -echo "Running generator 01-simple.sh" -mkdir "${OUT}/print/" +mkdir -p "${OUT}/print/" echo "Simple generated file" > "${OUT}/print/file.txt" ---------------------------------------- directory 0755 ./var/lib/holo/files/base/ diff --git a/test/generators/03-conflict/expected-apply-output b/test/generators/03-conflict/expected-apply-output index ff8e200..a0052ea 100644 --- a/test/generators/03-conflict/expected-apply-output +++ b/test/generators/03-conflict/expected-apply-output @@ -1,7 +1,4 @@ -Processing generators... -target/usr/share/holo/generators/01-simple.sh - Printing env:target/var/tmp/holo/generated/print found at HOLO_RESOURCE_DIR diff --git a/test/generators/03-conflict/expected-diff-output b/test/generators/03-conflict/expected-diff-output index 857b787..39a9383 100644 --- a/test/generators/03-conflict/expected-diff-output +++ b/test/generators/03-conflict/expected-diff-output @@ -1,5 +1 @@ - -Processing generators... -target/usr/share/holo/generators/01-simple.sh - exit status 0 diff --git a/test/generators/03-conflict/expected-scan-output b/test/generators/03-conflict/expected-scan-output index 3cc16b5..cb45210 100644 --- a/test/generators/03-conflict/expected-scan-output +++ b/test/generators/03-conflict/expected-scan-output @@ -1,7 +1,4 @@ -Processing generators... -target/usr/share/holo/generators/01-simple.sh - env:target/var/tmp/holo/generated/print found at HOLO_RESOURCE_DIR diff --git a/test/generators/03-conflict/expected-tree b/test/generators/03-conflict/expected-tree index 0874943..997d175 100644 --- a/test/generators/03-conflict/expected-tree +++ b/test/generators/03-conflict/expected-tree @@ -7,8 +7,7 @@ directory 0755 ./tmp/ ---------------------------------------- file 0755 ./usr/share/holo/generators/01-simple.sh #!/bin/sh -echo "Running generator 01-simple.sh" -mkdir "${OUT}/print/" +mkdir -p "${OUT}/print/" echo "Simple generated file" > "${OUT}/print/file.txt" ---------------------------------------- file 0755 ./usr/share/holo/print/file.txt diff --git a/test/generators/03-conflict/source-tree b/test/generators/03-conflict/source-tree index 91a4d89..48447d0 100644 --- a/test/generators/03-conflict/source-tree +++ b/test/generators/03-conflict/source-tree @@ -10,8 +10,7 @@ Simple static file ---------------------------------------- file 0755 ./usr/share/holo/generators/01-simple.sh #!/bin/sh -echo "Running generator 01-simple.sh" -mkdir "${OUT}/print/" +mkdir -p "${OUT}/print/" echo "Simple generated file" > "${OUT}/print/file.txt" ---------------------------------------- directory 0755 ./var/lib/holo/files/base/