From 830a8ea47029dea5df2e8a6d5df35340724fc71f Mon Sep 17 00:00:00 2001 From: Marton Soos Date: Thu, 27 Feb 2020 17:56:08 +0100 Subject: [PATCH 01/17] Refactor env sources in the run command so that they return structs to delay fetching of secrets. This way the shadowed secret references will not be evaluated. Also updated templates to allow for checking whether or not they contain secrets. This will be used in the trace command, for example. --- internals/secrethub/env_source.go | 423 +++++++++++++++++++++++++++ internals/secrethub/run.go | 418 +------------------------- internals/secrethub/run_test.go | 12 +- internals/secrethub/secret_reader.go | 6 + internals/secrethub/tpl/template.go | 2 + internals/secrethub/tpl/v1.go | 4 + internals/secrethub/tpl/v2.go | 11 + 7 files changed, 462 insertions(+), 414 deletions(-) create mode 100644 internals/secrethub/env_source.go diff --git a/internals/secrethub/env_source.go b/internals/secrethub/env_source.go new file mode 100644 index 00000000..4a2f9a28 --- /dev/null +++ b/internals/secrethub/env_source.go @@ -0,0 +1,423 @@ +package secrethub + +import ( + "bufio" + "bytes" + "errors" + "io" + "io/ioutil" + "path/filepath" + "strings" + "unicode" + + "github.com/secrethub/secrethub-cli/internals/cli/validation" + "github.com/secrethub/secrethub-cli/internals/secrethub/tpl" + "github.com/secrethub/secrethub-go/internals/api" + "gopkg.in/yaml.v2" +) + +// EnvSource defines a method of reading environment variables from a source. +type EnvSource interface { + // Env returns a map of key value pairs. + Env() (map[string]value, error) +} + +type value interface { + resolve(tpl.SecretReader) (string, error) + containsSecret() bool +} + +type secretValue struct { + path string +} + +func (s *secretValue) resolve(sr tpl.SecretReader) (string, error) { + return sr.ReadSecret(s.path) +} + +func (s *secretValue) containsSecret() bool { + return true +} + +func newSecretValue(path string) value { + return &secretValue{path: path} +} + +// EnvFlags defines environment variables sourced from command-line flags. +type EnvFlags map[string]string + +// NewEnvFlags parses a map of flag values. +func NewEnvFlags(flags map[string]string) (EnvFlags, error) { + for name, path := range flags { + err := validation.ValidateEnvarName(name) + if err != nil { + return nil, err + } + + err = api.ValidateSecretPath(path) + if err != nil { + return nil, err + } + } + + return flags, nil +} + +// Env returns a map of environment variables sourced from +// command-line flags and set to their corresponding value. +func (ef EnvFlags) Env() (map[string]value, error) { + result := make(map[string]value) + for name, path := range ef { + result[name] = newSecretValue(path) + } + return result, nil +} + +// referenceEnv is an environment with secrets configured with the +// secrethub:// syntax in the os environment variables. +type referenceEnv struct { + envVars map[string]string +} + +// newReferenceEnv returns an environment with secrets configured in the +// os environment with the secrethub:// syntax. +func newReferenceEnv(osEnv map[string]string) *referenceEnv { + envVars := make(map[string]string) + for key, value := range osEnv { + if strings.HasPrefix(value, secretReferencePrefix) { + envVars[key] = strings.TrimPrefix(value, secretReferencePrefix) + } + } + return &referenceEnv{ + envVars: envVars, + } +} + +// Env returns a map of key value pairs with the secrets configured with the +// secrethub:// syntax. +func (env *referenceEnv) Env() (map[string]value, error) { + envVarsWithSecrets := make(map[string]value) + for key, path := range env.envVars { + envVarsWithSecrets[key] = newSecretValue(path) + } + return envVarsWithSecrets, nil +} + +type envDirSecretValue struct { + value string +} + +func (s *envDirSecretValue) resolve(_ tpl.SecretReader) (string, error) { + return s.value, nil +} + +func (s *envDirSecretValue) containsSecret() bool { + return true +} + +func newEnvDirSecretValue(value string) value { + return &envDirSecretValue{value: value} +} + +// EnvDir defines environment variables sourced from files in a directory. +type EnvDir map[string]value + +// NewEnvDir sources environment variables from files in a given directory, +// using the file name as key and contents as value. +func NewEnvDir(path string) (EnvDir, error) { + files, err := ioutil.ReadDir(path) + if err != nil { + return nil, ErrReadEnvDir(err) + } + + env := make(map[string]value) + for _, f := range files { + if !f.IsDir() { + filePath := filepath.Join(path, f.Name()) + fileContent, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, ErrReadEnvFile(f.Name(), err) + } + + env[f.Name()] = newEnvDirSecretValue(string(fileContent)) + } + } + + return env, nil +} + +// Env returns a map of environment variables sourced from the directory. +func (dir EnvDir) Env() (map[string]value, error) { + return dir, nil +} + +type templateValue struct { + filepath string + template tpl.Template + varReader tpl.VariableReader +} + +func (v *templateValue) resolve(sr tpl.SecretReader) (string, error) { + value, err := v.template.Evaluate(v.varReader, sr) + if err != nil { + return "", ErrParsingTemplate(v.filepath, err) + } + return value, nil +} + +func (v *templateValue) containsSecret() bool { + return v.template.ContainsSecrets() +} + +func newTemplateValue(filepath string, template tpl.Template, varReader tpl.VariableReader) value { + return &templateValue{ + filepath: filepath, + template: template, + varReader: varReader, + } +} + +type envTemplate struct { + filepath string + envVars []envvarTpls + templateVarReader tpl.VariableReader +} + +type envvarTpls struct { + key tpl.Template + value tpl.Template + lineNo int +} + +// Env injects the given secrets in the environment values and returns +// a map of the resulting environment. +func (t envTemplate) Env() (map[string]value, error) { + result := make(map[string]value) + for _, tpls := range t.envVars { + key, err := tpls.key.Evaluate(t.templateVarReader, secretReaderNotAllowed{}) + if err != nil { + return nil, err + } + + err = validation.ValidateEnvarName(key) + if err != nil { + return nil, templateError(tpls.lineNo, err) + } + + value := newTemplateValue(t.filepath, tpls.value, t.templateVarReader) + + result[key] = value + } + return result, nil +} + +func templateError(lineNo int, err error) error { + if lineNo > 0 { + return ErrTemplate(lineNo, err) + } + return err +} + +// ReadEnvFile reads and parses a .env file. +func ReadEnvFile(filepath string, reader io.Reader, varReader tpl.VariableReader, parser tpl.Parser) (EnvFile, error) { + env, err := NewEnv(filepath, reader, varReader, parser) + if err != nil { + return EnvFile{}, ErrParsingTemplate(filepath, err) + } + return EnvFile{ + path: filepath, + env: env, + }, nil +} + +// EnvFile contains an environment that is read from a file. +type EnvFile struct { + path string + env EnvSource +} + +// Env returns a map of key value pairs read from the environment file. +func (e EnvFile) Env() (map[string]value, error) { + env, err := e.env.Env() + if err != nil { + return nil, ErrParsingTemplate(e.path, err) + } + return env, nil +} + +// NewEnv loads an environment of key-value pairs from a string. +// The format of the string can be `key: value` or `key=value` pairs. +func NewEnv(filepath string, r io.Reader, varReader tpl.VariableReader, parser tpl.Parser) (EnvSource, error) { + env, err := parseEnvironment(r) + if err != nil { + return nil, err + } + + secretTemplates := make([]envvarTpls, len(env)) + for i, envvar := range env { + keyTpl, err := parser.Parse(envvar.key, envvar.lineNumber, envvar.columnNumberKey) + if err != nil { + return nil, err + } + + err = validation.ValidateEnvarName(envvar.key) + if err != nil { + return nil, err + } + + valTpl, err := parser.Parse(envvar.value, envvar.lineNumber, envvar.columnNumberValue) + if err != nil { + return nil, err + } + + secretTemplates[i] = envvarTpls{ + key: keyTpl, + value: valTpl, + lineNo: envvar.lineNumber, + } + } + + return envTemplate{ + filepath: filepath, + envVars: secretTemplates, + templateVarReader: varReader, + }, nil +} + +type envvar struct { + key string + value string + lineNumber int + columnNumberKey int + columnNumberValue int +} + +// parseEnvironment parses envvars from a string. +// It first tries the key=value format. When that returns an error, +// the yml format is tried. +// The default parser to be used with the format is also returned. +func parseEnvironment(r io.Reader) ([]envvar, error) { + var ymlReader bytes.Buffer + env, err := parseDotEnv(io.TeeReader(r, &ymlReader)) + if err != nil { + var ymlErr error + env, ymlErr = parseYML(&ymlReader) + if ymlErr != nil { + return nil, err + } + } + return env, nil +} + +// parseDotEnv parses key-value pairs in the .env syntax (key=value). +func parseDotEnv(r io.Reader) ([]envvar, error) { + vars := map[string]envvar{} + scanner := bufio.NewScanner(r) + + i := 0 + for scanner.Scan() { + i++ + line := scanner.Text() + + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 { + return nil, ErrTemplate(i, errors.New("template is not formatted as key=value pairs")) + } + + columnNumberValue := len(parts[0]) + 2 // the length of the key (including spaces and quotes) + one for the = sign and one for the current column. + for _, r := range parts[1] { + if !unicode.IsSpace(r) { + break + } + columnNumberValue++ + } + + columnNumberKey := 1 // one for the current column. + for _, r := range parts[0] { + if !unicode.IsSpace(r) { + break + } + columnNumberKey++ + } + + key := strings.TrimSpace(parts[0]) + + value, isTrimmed := trimQuotes(strings.TrimSpace(parts[1])) + if isTrimmed { + columnNumberValue++ + } + + vars[key] = envvar{ + key: key, + value: value, + lineNumber: i, + columnNumberValue: columnNumberValue, + columnNumberKey: columnNumberKey, + } + } + + i = 0 + res := make([]envvar, len(vars)) + for _, envvar := range vars { + res[i] = envvar + i++ + } + + return res, nil +} + +const ( + doubleQuoteChar = '\u0022' // " + singleQuoteChar = '\u0027' // ' +) + +// trimQuotes removes a leading and trailing quote from the given string value if +// it is wrapped in either single or double quotes. +// +// Rules: +// - Empty values become empty values (e.g. `''`and `""` both evaluate to the empty string ``). +// - Inner quotes are maintained (e.g. `{"foo":"bar"}` remains unchanged). +// - Single and double quoted values are escaped (e.g. `'foo'` and `"foo"` both evaluate to `foo`). +// - Single and double qouted values maintain whitespace from both ends (e.g. `" foo "` becomes ` foo `) +// - Inputs with either leading or trailing whitespace are considered unquoted, +// so make sure you sanitize your inputs before calling this function. +func trimQuotes(s string) (string, bool) { + n := len(s) + if n > 1 && + (s[0] == singleQuoteChar && s[n-1] == singleQuoteChar || + s[0] == doubleQuoteChar && s[n-1] == doubleQuoteChar) { + return s[1 : n-1], true + } + + return s, false +} + +func parseYML(r io.Reader) ([]envvar, error) { + contents, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + pairs := make(map[string]string) + err = yaml.Unmarshal(contents, pairs) + if err != nil { + return nil, err + } + + vars := make([]envvar, len(pairs)) + i := 0 + for key, value := range pairs { + vars[i] = envvar{ + key: key, + value: value, + lineNumber: -1, + } + i++ + } + return vars, nil +} diff --git a/internals/secrethub/run.go b/internals/secrethub/run.go index db860535..d8d13eb8 100644 --- a/internals/secrethub/run.go +++ b/internals/secrethub/run.go @@ -1,11 +1,8 @@ package secrethub import ( - "bufio" "bytes" - "errors" "fmt" - "io" "io/ioutil" "os" "os/exec" @@ -14,7 +11,6 @@ import ( "strings" "syscall" "time" - "unicode" "github.com/secrethub/secrethub-cli/internals/cli/ui" @@ -24,10 +20,7 @@ import ( "github.com/secrethub/secrethub-cli/internals/secrethub/tpl" "github.com/secrethub/secrethub-cli/internals/secretspec" - "github.com/secrethub/secrethub-go/internals/api" "github.com/secrethub/secrethub-go/internals/errio" - - "gopkg.in/yaml.v2" ) // Errors @@ -270,32 +263,16 @@ func (cmd *RunCommand) sourceEnvironment() ([]string, []string, error) { envSources = append(envSources, dirSource) } - // Collect all secrets - secrets := make(map[string]string) - for _, source := range envSources { - for _, path := range source.Secrets() { - secrets[path] = "" - } - } - var sr tpl.SecretReader = newSecretReader(cmd.newClient) if cmd.ignoreMissingSecrets { sr = newIgnoreMissingSecretReader(sr) } secretReader := newBufferedSecretReader(sr) - for path := range secrets { - secret, err := secretReader.ReadSecret(path) - if err != nil { - return nil, nil, err - } - secrets[path] = secret - } - // Construct the environment, sourcing variables from the configured sources. environment := make(map[string]string) for _, source := range envSources { - pairs, err := source.Env(secrets, secretReader) + pairs, err := source.Env() if err != nil { return nil, nil, err } @@ -304,7 +281,11 @@ func (cmd *RunCommand) sourceEnvironment() ([]string, []string, error) { // Only set a variable if it wasn't set by a previous source. _, found := environment[key] if !found { - environment[key] = value + resolvedValue, err := value.resolve(secretReader) + if err != nil { + return nil, nil, err + } + environment[key] = resolvedValue } } } @@ -360,390 +341,3 @@ func parseKeyValueStringsToMap(values []string) (map[string]string, []string) { return parsedLines, unparsableLines } - -// EnvSource defines a method of reading environment variables from a source. -type EnvSource interface { - // Env returns a map of key value pairs. - Env(secrets map[string]string, sr tpl.SecretReader) (map[string]string, error) - // Secrets returns a list of paths to secrets that are used in the environment. - Secrets() []string -} - -type envTemplate struct { - envVars []envvarTpls - templateVarReader tpl.VariableReader -} - -type envvarTpls struct { - key tpl.Template - value tpl.Template - lineNo int -} - -type secretReaderNotAllowed struct{} - -func (sr secretReaderNotAllowed) ReadSecret(path string) (string, error) { - return "", ErrSecretsNotAllowedInKey -} - -// Env injects the given secrets in the environment values and returns -// a map of the resulting environment. -func (t envTemplate) Env(secrets map[string]string, sr tpl.SecretReader) (map[string]string, error) { - result := make(map[string]string) - for _, tpls := range t.envVars { - key, err := tpls.key.Evaluate(t.templateVarReader, secretReaderNotAllowed{}) - if err != nil { - return nil, err - } - - err = validation.ValidateEnvarName(key) - if err != nil { - return nil, templateError(tpls.lineNo, err) - } - - value, err := tpls.value.Evaluate(t.templateVarReader, sr) - if err != nil { - return nil, err - } - - result[key] = value - } - return result, nil -} - -func templateError(lineNo int, err error) error { - if lineNo > 0 { - return ErrTemplate(lineNo, err) - } - return err -} - -// Secrets implements the EnvSource.Secrets function. -// The envTemplate fetches its secrets using a tpl.SecretReader. -func (t envTemplate) Secrets() []string { - return []string{} -} - -// ReadEnvFile reads and parses a .env file. -func ReadEnvFile(filepath string, reader io.Reader, varReader tpl.VariableReader, parser tpl.Parser) (EnvFile, error) { - env, err := NewEnv(reader, varReader, parser) - if err != nil { - return EnvFile{}, ErrParsingTemplate(filepath, err) - } - return EnvFile{ - path: filepath, - env: env, - }, nil -} - -// referenceEnv is an environment with secrets configured with the -// secrethub:// syntax in the os environment variables. -type referenceEnv struct { - envVars map[string]string -} - -// newReferenceEnv returns an environment with secrets configured in the -// os environment with the secrethub:// syntax. -func newReferenceEnv(osEnv map[string]string) *referenceEnv { - envVars := make(map[string]string) - for key, value := range osEnv { - if strings.HasPrefix(value, secretReferencePrefix) { - envVars[key] = strings.TrimPrefix(value, secretReferencePrefix) - } - } - return &referenceEnv{ - envVars: envVars, - } -} - -// Env returns a map of key value pairs with the secrets configured with the -// secrethub:// syntax. -func (env *referenceEnv) Env(_ map[string]string, secretReader tpl.SecretReader) (map[string]string, error) { - envVarsWithSecrets := make(map[string]string) - for key, path := range env.envVars { - secret, err := secretReader.ReadSecret(path) - if err != nil { - return nil, err - } - envVarsWithSecrets[key] = secret - } - return envVarsWithSecrets, nil -} - -func (env *referenceEnv) Secrets() []string { - return nil -} - -// EnvFile contains an environment that is read from a file. -type EnvFile struct { - path string - env EnvSource -} - -// Env returns a map of key value pairs read from the environment file. -func (e EnvFile) Env(secrets map[string]string, sr tpl.SecretReader) (map[string]string, error) { - env, err := e.env.Env(secrets, sr) - if err != nil { - return nil, ErrParsingTemplate(e.path, err) - } - return env, nil -} - -// Secrets returns a list of paths to secrets that are used in the environment. -func (e EnvFile) Secrets() []string { - return e.env.Secrets() -} - -// NewEnv loads an environment of key-value pairs from a string. -// The format of the string can be `key: value` or `key=value` pairs. -func NewEnv(r io.Reader, varReader tpl.VariableReader, parser tpl.Parser) (EnvSource, error) { - env, err := parseEnvironment(r) - if err != nil { - return nil, err - } - - secretTemplates := make([]envvarTpls, len(env)) - for i, envvar := range env { - keyTpl, err := parser.Parse(envvar.key, envvar.lineNumber, envvar.columnNumberKey) - if err != nil { - return nil, err - } - - err = validation.ValidateEnvarName(envvar.key) - if err != nil { - return nil, err - } - - valTpl, err := parser.Parse(envvar.value, envvar.lineNumber, envvar.columnNumberValue) - if err != nil { - return nil, err - } - - secretTemplates[i] = envvarTpls{ - key: keyTpl, - value: valTpl, - lineNo: envvar.lineNumber, - } - } - - return envTemplate{ - envVars: secretTemplates, - templateVarReader: varReader, - }, nil -} - -type envvar struct { - key string - value string - lineNumber int - columnNumberKey int - columnNumberValue int -} - -// parseEnvironment parses envvars from a string. -// It first tries the key=value format. When that returns an error, -// the yml format is tried. -// The default parser to be used with the format is also returned. -func parseEnvironment(r io.Reader) ([]envvar, error) { - var ymlReader bytes.Buffer - env, err := parseDotEnv(io.TeeReader(r, &ymlReader)) - if err != nil { - var ymlErr error - env, ymlErr = parseYML(&ymlReader) - if ymlErr != nil { - return nil, err - } - } - return env, nil -} - -// parseDotEnv parses key-value pairs in the .env syntax (key=value). -func parseDotEnv(r io.Reader) ([]envvar, error) { - vars := map[string]envvar{} - scanner := bufio.NewScanner(r) - - i := 0 - for scanner.Scan() { - i++ - line := scanner.Text() - - trimmed := strings.TrimSpace(line) - if trimmed == "" || strings.HasPrefix(trimmed, "#") { - continue - } - - parts := strings.SplitN(line, "=", 2) - if len(parts) != 2 { - return nil, ErrTemplate(i, errors.New("template is not formatted as key=value pairs")) - } - - columnNumberValue := len(parts[0]) + 2 // the length of the key (including spaces and quotes) + one for the = sign and one for the current column. - for _, r := range parts[1] { - if !unicode.IsSpace(r) { - break - } - columnNumberValue++ - } - - columnNumberKey := 1 // one for the current column. - for _, r := range parts[0] { - if !unicode.IsSpace(r) { - break - } - columnNumberKey++ - } - - key := strings.TrimSpace(parts[0]) - - value, isTrimmed := trimQuotes(strings.TrimSpace(parts[1])) - if isTrimmed { - columnNumberValue++ - } - - vars[key] = envvar{ - key: key, - value: value, - lineNumber: i, - columnNumberValue: columnNumberValue, - columnNumberKey: columnNumberKey, - } - } - - i = 0 - res := make([]envvar, len(vars)) - for _, envvar := range vars { - res[i] = envvar - i++ - } - - return res, nil -} - -const ( - doubleQuoteChar = '\u0022' // " - singleQuoteChar = '\u0027' // ' -) - -// trimQuotes removes a leading and trailing quote from the given string value if -// it is wrapped in either single or double quotes. -// -// Rules: -// - Empty values become empty values (e.g. `''`and `""` both evaluate to the empty string ``). -// - Inner quotes are maintained (e.g. `{"foo":"bar"}` remains unchanged). -// - Single and double quoted values are escaped (e.g. `'foo'` and `"foo"` both evaluate to `foo`). -// - Single and double qouted values maintain whitespace from both ends (e.g. `" foo "` becomes ` foo `) -// - Inputs with either leading or trailing whitespace are considered unquoted, -// so make sure you sanitize your inputs before calling this function. -func trimQuotes(s string) (string, bool) { - n := len(s) - if n > 1 && - (s[0] == singleQuoteChar && s[n-1] == singleQuoteChar || - s[0] == doubleQuoteChar && s[n-1] == doubleQuoteChar) { - return s[1 : n-1], true - } - - return s, false -} - -func parseYML(r io.Reader) ([]envvar, error) { - contents, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - - pairs := make(map[string]string) - err = yaml.Unmarshal(contents, pairs) - if err != nil { - return nil, err - } - - vars := make([]envvar, len(pairs)) - i := 0 - for key, value := range pairs { - vars[i] = envvar{ - key: key, - value: value, - lineNumber: -1, - } - i++ - } - return vars, nil -} - -// EnvDir defines environment variables sourced from files in a directory. -type EnvDir map[string]string - -// NewEnvDir sources environment variables from files in a given directory, -// using the file name as key and contents as value. -func NewEnvDir(path string) (EnvDir, error) { - files, err := ioutil.ReadDir(path) - if err != nil { - return nil, ErrReadEnvDir(err) - } - - env := make(map[string]string) - for _, f := range files { - if !f.IsDir() { - filePath := filepath.Join(path, f.Name()) - fileContent, err := ioutil.ReadFile(filePath) - if err != nil { - return nil, ErrReadEnvFile(f.Name(), err) - } - - env[f.Name()] = string(fileContent) - } - } - - return env, nil -} - -// Env returns a map of environment variables sourced from the directory. -func (dir EnvDir) Env(secrets map[string]string, _ tpl.SecretReader) (map[string]string, error) { - return dir, nil -} - -// Secrets returns a list of paths to secrets that are used in the environment. -func (dir EnvDir) Secrets() []string { - return []string{} -} - -// EnvFlags defines environment variables sourced from command-line flags. -type EnvFlags map[string]string - -// NewEnvFlags parses a map of flag values. -func NewEnvFlags(flags map[string]string) (EnvFlags, error) { - for name, path := range flags { - err := validation.ValidateEnvarName(name) - if err != nil { - return nil, err - } - - err = api.ValidateSecretPath(path) - if err != nil { - return nil, err - } - } - - return flags, nil -} - -// Env returns a map of environment variables sourced from -// command-line flags and set to their corresponding value. -func (ef EnvFlags) Env(secrets map[string]string, _ tpl.SecretReader) (map[string]string, error) { - result := make(map[string]string) - for name, path := range ef { - result[name] = secrets[path] - } - return result, nil -} - -// Secrets returns the paths to the secrets that are used in the flags. -func (ef EnvFlags) Secrets() []string { - result := make([]string, len(ef)) - i := 0 - for _, v := range ef { - result[i] = v - i++ - } - return result -} diff --git a/internals/secrethub/run_test.go b/internals/secrethub/run_test.go index 7ac4f2f4..89f00a5c 100644 --- a/internals/secrethub/run_test.go +++ b/internals/secrethub/run_test.go @@ -445,13 +445,21 @@ func TestNewEnv(t *testing.T) { parser, err := getTemplateParser([]byte(tc.raw), "auto") assert.OK(t, err) - env, err := NewEnv(strings.NewReader(tc.raw), tc.templateVarReader, parser) + env, err := NewEnv("secrethub.env", strings.NewReader(tc.raw), tc.templateVarReader, parser) if err != nil { assert.Equal(t, err, tc.err) } else { - actual, err := env.Env(map[string]string{}, fakes.FakeSecretReader{Secrets: tc.replacements}) + actualValues, err := env.Env() assert.Equal(t, err, tc.err) + // resolve values + actual := make(map[string]string, len(actualValues)) + for name, value := range actualValues { + actual[name], err = value.resolve(fakes.FakeSecretReader{Secrets: tc.replacements}) + if err != nil { + t.Fail() + } + } assert.Equal(t, actual, tc.expected) } }) diff --git a/internals/secrethub/secret_reader.go b/internals/secrethub/secret_reader.go index 5c4d82c6..e2de81a8 100644 --- a/internals/secrethub/secret_reader.go +++ b/internals/secrethub/secret_reader.go @@ -57,6 +57,12 @@ func (sr *bufferedSecretReader) ReadSecret(path string) (string, error) { return secret, err } +type secretReaderNotAllowed struct{} + +func (sr secretReaderNotAllowed) ReadSecret(path string) (string, error) { + return "", ErrSecretsNotAllowedInKey +} + // Values returns a list of values read with this secret reader. func (sr bufferedSecretReader) Values() []string { return sr.secretsRead diff --git a/internals/secrethub/tpl/template.go b/internals/secrethub/tpl/template.go index e79dc78e..7ab21ead 100644 --- a/internals/secrethub/tpl/template.go +++ b/internals/secrethub/tpl/template.go @@ -21,6 +21,8 @@ type Template interface { // Evaluate renders a template. It replaces all variable- and secret tags in the template. // The supplied variables should have lowercase keys. Evaluate(varReader VariableReader, sr SecretReader) (string, error) + + ContainsSecrets() bool } // NewParser returns a parser for the latest template syntax. diff --git a/internals/secrethub/tpl/v1.go b/internals/secrethub/tpl/v1.go index ee659b0c..7db2315f 100644 --- a/internals/secrethub/tpl/v1.go +++ b/internals/secrethub/tpl/v1.go @@ -48,3 +48,7 @@ func (t templateV1) Evaluate(_ VariableReader, sr SecretReader) (string, error) return t.template.Inject(secrets) } + +func (t templateV1) ContainsSecrets() bool { + return len(t.template.Keys()) > 0 +} diff --git a/internals/secrethub/tpl/v2.go b/internals/secrethub/tpl/v2.go index 5789505c..15b7ad94 100644 --- a/internals/secrethub/tpl/v2.go +++ b/internals/secrethub/tpl/v2.go @@ -459,3 +459,14 @@ func (t templateV2) Evaluate(varReader VariableReader, sr SecretReader) (string, return buffer.String(), nil } + +func (t templateV2) ContainsSecrets() bool { + for _, node := range t.nodes { + _, ok := node.(secret) + if ok { + return true + } + } + + return false +} From e58fd8483ff665de2d248bf27bdc96f943af42ad Mon Sep 17 00:00:00 2001 From: Marton Soos Date: Thu, 27 Feb 2020 19:33:56 +0100 Subject: [PATCH 02/17] Factor out environment struct It contains the logic of sourcing secrets to the command executed by the run commmand --- internals/secrethub/env_source.go | 140 +++++++++++++++ internals/secrethub/run.go | 142 ++++----------- internals/secrethub/run_test.go | 288 ++++++++++++++++++------------ 3 files changed, 346 insertions(+), 224 deletions(-) diff --git a/internals/secrethub/env_source.go b/internals/secrethub/env_source.go index 4a2f9a28..460bb363 100644 --- a/internals/secrethub/env_source.go +++ b/internals/secrethub/env_source.go @@ -6,16 +6,128 @@ import ( "errors" "io" "io/ioutil" + "os" "path/filepath" "strings" "unicode" + "github.com/secrethub/secrethub-cli/internals/cli" + + "github.com/secrethub/secrethub-cli/internals/cli/ui" + "github.com/secrethub/secrethub-cli/internals/cli/validation" "github.com/secrethub/secrethub-cli/internals/secrethub/tpl" "github.com/secrethub/secrethub-go/internals/api" "gopkg.in/yaml.v2" ) +type environment struct { + io ui.IO + osEnv []string + readFile func(filename string) ([]byte, error) + osStat func(filename string) (os.FileInfo, error) + envar map[string]string + envFile string + templateVars map[string]string + templateVersion string + dontPromptMissingTemplateVar bool +} + +func newEnvironment(io ui.IO) *environment { + return &environment{ + io: io, + osEnv: os.Environ(), + readFile: ioutil.ReadFile, + osStat: os.Stat, + templateVars: make(map[string]string), + envar: make(map[string]string), + } +} + +func (env *environment) register(clause *cli.CommandClause) { + clause.Flag("envar", "Source an environment variable from a secret at a given path with `NAME=`").Short('e').StringMapVar(&env.envar) + clause.Flag("env-file", "The path to a file with environment variable mappings of the form `NAME=value`. Template syntax can be used to inject secrets.").StringVar(&env.envFile) + clause.Flag("template", "").Hidden().StringVar(&env.envFile) + clause.Flag("var", "Define the value for a template variable with `VAR=VALUE`, e.g. --var env=prod").Short('v').StringMapVar(&env.templateVars) + clause.Flag("template-version", "The template syntax version to be used. The options are v1, v2, latest or auto to automatically detect the version.").Default("auto").StringVar(&env.templateVersion) + clause.Flag("no-prompt", "Do not prompt when a template variable is missing and return an error instead.").BoolVar(&env.dontPromptMissingTemplateVar) +} + +func (env *environment) Env() (map[string]value, error) { + osEnv, _ := parseKeyValueStringsToMap(env.osEnv) + var sources []EnvSource + + //secrethub.env file + if env.envFile == "" { + _, err := env.osStat(defaultEnvFile) + if err == nil { + env.envFile = defaultEnvFile + } else if !os.IsNotExist(err) { + return nil, ErrReadDefaultEnvFile(defaultEnvFile, err) + } + } + + if env.envFile != "" { + templateVariableReader, err := newVariableReader(osEnv, env.templateVars) + if err != nil { + return nil, err + } + + if !env.dontPromptMissingTemplateVar { + templateVariableReader = newPromptMissingVariableReader(templateVariableReader, env.io) + } + + raw, err := env.readFile(env.envFile) + if err != nil { + return nil, ErrCannotReadFile(env.envFile, err) + } + + parser, err := getTemplateParser(raw, env.templateVersion) + if err != nil { + return nil, err + } + + envFile, err := ReadEnvFile(env.envFile, bytes.NewReader(raw), templateVariableReader, parser) + if err != nil { + return nil, err + } + sources = append(sources, envFile) + } + + // secret references (secrethub://) + referenceEnv := newReferenceEnv(osEnv) + sources = append(sources, referenceEnv) + + // --envar flag + // TODO: Validate the flags when parsing by implementing the Flag interface for EnvFlags. + flagEnv, err := NewEnvFlags(env.envar) + if err != nil { + return nil, err + } + sources = append(sources, flagEnv) + + var envs []map[string]value + for _, source := range sources { + env, err := source.Env() + if err != nil { + return nil, err + } + envs = append(envs, env) + } + + return mergeEnvs(envs...), nil +} + +func mergeEnvs(envs ...map[string]value) map[string]value { + result := map[string]value{} + for _, env := range envs { + for name, value := range env { + result[name] = value + } + } + return result +} + // EnvSource defines a method of reading environment variables from a source. type EnvSource interface { // Env returns a map of key value pairs. @@ -421,3 +533,31 @@ func parseYML(r io.Reader) ([]envvar, error) { } return vars, nil } + +type plaintextValue struct { + value string +} + +func newPlaintextValue(value string) *plaintextValue { + return &plaintextValue{value: value} +} + +func (v *plaintextValue) resolve(_ tpl.SecretReader) (string, error) { + return v.value, nil +} + +func (v *plaintextValue) containsSecret() bool { + return false +} + +type osEnv struct { + osEnv map[string]string +} + +func (o *osEnv) Env() (map[string]value, error) { + res := map[string]value{} + for name, value := range o.osEnv { + res[name] = newPlaintextValue(value) + } + return res, nil +} diff --git a/internals/secrethub/run.go b/internals/secrethub/run.go index d8d13eb8..70650577 100644 --- a/internals/secrethub/run.go +++ b/internals/secrethub/run.go @@ -1,9 +1,7 @@ package secrethub import ( - "bytes" "fmt" - "io/ioutil" "os" "os/exec" "os/signal" @@ -12,14 +10,15 @@ import ( "syscall" "time" + "github.com/secrethub/secrethub-cli/internals/secrethub/tpl" + + "github.com/secrethub/secrethub-cli/internals/secretspec" + "github.com/secrethub/secrethub-cli/internals/cli/ui" "github.com/secrethub/secrethub-cli/internals/cli/masker" "github.com/secrethub/secrethub-cli/internals/cli/validation" "github.com/secrethub/secrethub-cli/internals/secrethub/command" - "github.com/secrethub/secrethub-cli/internals/secrethub/tpl" - "github.com/secrethub/secrethub-cli/internals/secretspec" - "github.com/secrethub/secrethub-go/internals/errio" ) @@ -52,33 +51,24 @@ const ( // defined with --envar or --env-file flags and secrets.yml files. // The yml files write to .secretsenv/ when running the set command. type RunCommand struct { - io ui.IO - osEnv []string - readFile func(filename string) ([]byte, error) - osStat func(filename string) (os.FileInfo, error) - command []string - envar map[string]string - envFile string - templateVars map[string]string - templateVersion string - env string - noMasking bool - maskingTimeout time.Duration - newClient newClientFunc - ignoreMissingSecrets bool - dontPromptMissingTemplateVar bool + io ui.IO + osEnv []string + command []string + env string + environment *environment + noMasking bool + maskingTimeout time.Duration + newClient newClientFunc + ignoreMissingSecrets bool } // NewRunCommand creates a new RunCommand. func NewRunCommand(io ui.IO, newClient newClientFunc) *RunCommand { return &RunCommand{ - io: io, - osEnv: os.Environ(), - readFile: ioutil.ReadFile, - osStat: os.Stat, - envar: make(map[string]string), - templateVars: make(map[string]string), - newClient: newClient, + io: io, + osEnv: os.Environ(), + environment: newEnvironment(io), + newClient: newClient, } } @@ -93,17 +83,11 @@ func (cmd *RunCommand) Register(r command.Registerer) { clause.HelpLong(helpLong) clause.Alias("exec") clause.Arg("command", "The command to execute").Required().StringsVar(&cmd.command) - clause.Flag("envar", "Source an environment variable from a secret at a given path with `NAME=`").Short('e').StringMapVar(&cmd.envar) - clause.Flag("env-file", "The path to a file with environment variable mappings of the form `NAME=value`. Template syntax can be used to inject secrets.").StringVar(&cmd.envFile) - clause.Flag("template", "").Hidden().StringVar(&cmd.envFile) - clause.Flag("var", "Define the value for a template variable with `VAR=VALUE`, e.g. --var env=prod").Short('v').StringMapVar(&cmd.templateVars) clause.Flag("env", "The name of the environment prepared by the set command (default is `default`)").Default("default").Hidden().StringVar(&cmd.env) clause.Flag("no-masking", "Disable masking of secrets on stdout and stderr").BoolVar(&cmd.noMasking) clause.Flag("masking-timeout", "The maximum time output is buffered. Warning: lowering this value increases the chance of secrets not being masked.").Default("1s").DurationVar(&cmd.maskingTimeout) - clause.Flag("template-version", "The template syntax version to be used. The options are v1, v2, latest or auto to automatically detect the version.").Default("auto").StringVar(&cmd.templateVersion) clause.Flag("ignore-missing-secrets", "Do not return an error when a secret does not exist and use an empty value instead.").BoolVar(&cmd.ignoreMissingSecrets) - clause.Flag("no-prompt", "Do not prompt when a template variable is missing and return an error instead.").BoolVar(&cmd.dontPromptMissingTemplateVar) - + cmd.environment.register(clause) command.BindAction(clause, cmd.Run) } @@ -205,63 +189,32 @@ func (cmd *RunCommand) Run() error { func (cmd *RunCommand) sourceEnvironment() ([]string, []string, error) { osEnv, passthroughEnv := parseKeyValueStringsToMap(cmd.osEnv) - envSources := []EnvSource{} - - referenceEnv := newReferenceEnv(osEnv) - envSources = append(envSources, referenceEnv) - - // TODO: Validate the flags when parsing by implementing the Flag interface for EnvFlags. - flagSource, err := NewEnvFlags(cmd.envar) - if err != nil { - return nil, nil, err - } - envSources = append(envSources, flagSource) - - if cmd.envFile == "" { - _, err := cmd.osStat(defaultEnvFile) - if err == nil { - cmd.envFile = defaultEnvFile - } else if !os.IsNotExist(err) { - return nil, nil, ErrReadDefaultEnvFile(defaultEnvFile, err) - } + // add os envs + newEnv := map[string]string{} + for name, value := range osEnv { + newEnv[name] = value } - if cmd.envFile != "" { - templateVariableReader, err := newVariableReader(osEnv, cmd.templateVars) - if err != nil { - return nil, nil, err - } - - if !cmd.dontPromptMissingTemplateVar { - templateVariableReader = newPromptMissingVariableReader(templateVariableReader, cmd.io) - } - - raw, err := cmd.readFile(cmd.envFile) - if err != nil { - return nil, nil, ErrCannotReadFile(cmd.envFile, err) - } - - parser, err := getTemplateParser(raw, cmd.templateVersion) + // add values from .secretsenv and other sources + var dirValues map[string]value + envDir := filepath.Join(secretspec.SecretEnvPath, cmd.env) + _, err := os.Stat(envDir) + if err == nil { + dirSource, err := NewEnvDir(envDir) if err != nil { return nil, nil, err } - - envFile, err := ReadEnvFile(cmd.envFile, bytes.NewReader(raw), templateVariableReader, parser) + dirValues, err = dirSource.Env() if err != nil { return nil, nil, err } - envSources = append(envSources, envFile) } - envDir := filepath.Join(secretspec.SecretEnvPath, cmd.env) - _, err = cmd.osStat(envDir) - if err == nil { - dirSource, err := NewEnvDir(envDir) - if err != nil { - return nil, nil, err - } - envSources = append(envSources, dirSource) + envValues, err := cmd.environment.Env() + if err != nil { + return nil, nil, err } + envValues = mergeEnvs(dirValues, envValues) var sr tpl.SecretReader = newSecretReader(cmd.newClient) if cmd.ignoreMissingSecrets { @@ -269,38 +222,15 @@ func (cmd *RunCommand) sourceEnvironment() ([]string, []string, error) { } secretReader := newBufferedSecretReader(sr) - // Construct the environment, sourcing variables from the configured sources. - environment := make(map[string]string) - for _, source := range envSources { - pairs, err := source.Env() + for name, value := range envValues { + newEnv[name], err = value.resolve(secretReader) if err != nil { return nil, nil, err } - - for key, value := range pairs { - // Only set a variable if it wasn't set by a previous source. - _, found := environment[key] - if !found { - resolvedValue, err := value.resolve(secretReader) - if err != nil { - return nil, nil, err - } - environment[key] = resolvedValue - } - } - } - - // Source the remaining envars from the OS environment. - for key, value := range osEnv { - // Only set a variable if it wasn't set by a configured source. - _, found := environment[key] - if !found { - environment[key] = value - } } // Finally add the unparsed variables - processedOsEnv := append(passthroughEnv, mapToKeyValueStrings(environment)...) + processedOsEnv := append(passthroughEnv, mapToKeyValueStrings(newEnv)...) return processedOsEnv, secretReader.Values(), nil } diff --git a/internals/secrethub/run_test.go b/internals/secrethub/run_test.go index 89f00a5c..cf131e43 100644 --- a/internals/secrethub/run_test.go +++ b/internals/secrethub/run_test.go @@ -477,18 +477,22 @@ func TestRunCommand_Run(t *testing.T) { }{ "success, no secrets": { command: RunCommand{ - io: ui.NewFakeIO(), - osStat: osStatNotExist, + io: ui.NewFakeIO(), + environment: &environment{ + osStat: osStatNotExist, + }, command: []string{"echo", "test"}, }, }, "missing secret": { command: RunCommand{ command: []string{"echo", "test"}, - envar: map[string]string{ - "missing": "path/to/unexisting/secret", + environment: &environment{ + envar: map[string]string{ + "missing": "path/to/unexisting/secret", + }, + osStat: osStatNotExist, }, - osStat: osStatNotExist, newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ SecretService: &fakeclient.SecretService{ @@ -507,9 +511,11 @@ func TestRunCommand_Run(t *testing.T) { "missing secret ignored": { command: RunCommand{ command: []string{"echo", "test"}, - osStat: osStatNotExist, - envar: map[string]string{ - "missing": "path/to/unexisting/secret", + environment: &environment{ + osStat: osStatNotExist, + envar: map[string]string{ + "missing": "path/to/unexisting/secret", + }, }, io: ui.NewFakeIO(), newClient: func() (secrethub.ClientInterface, error) { @@ -530,11 +536,13 @@ func TestRunCommand_Run(t *testing.T) { "repo does not exist ignored": { command: RunCommand{ command: []string{"echo", "test"}, - envar: map[string]string{ - "missing": "unexisting/repo/secret", + environment: &environment{ + envar: map[string]string{ + "missing": "unexisting/repo/secret", + }, + osStat: osStatNotExist, }, - io: ui.NewFakeIO(), - osStat: osStatNotExist, + io: ui.NewFakeIO(), newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ SecretService: &fakeclient.SecretService{ @@ -552,32 +560,38 @@ func TestRunCommand_Run(t *testing.T) { }, "invalid template var: start with a number": { command: RunCommand{ - envFile: "secrethub.env", - osStat: osStatNotExist, - templateVars: map[string]string{ - "0foo": "value", + environment: &environment{ + osStat: osStatNotExist, + envFile: "secrethub.env", + templateVars: map[string]string{ + "0foo": "value", + }, + envar: map[string]string{}, }, - envar: map[string]string{}, }, err: ErrInvalidTemplateVar("0foo"), }, "invalid template var: illegal character": { command: RunCommand{ - envFile: "secrethub.env", - osStat: osStatNotExist, - templateVars: map[string]string{ - "foo@bar": "value", + environment: &environment{ + osStat: osStatNotExist, + envFile: "secrethub.env", + templateVars: map[string]string{ + "foo@bar": "value", + }, + envar: map[string]string{}, }, - envar: map[string]string{}, }, err: ErrInvalidTemplateVar("foo@bar"), }, "os env secret not found": { command: RunCommand{ - osEnv: []string{"TEST=secrethub://nonexistent/secret/path"}, command: []string{"echo", "test"}, io: ui.NewFakeIO(), - osStat: osStatNotExist, + environment: &environment{ + osEnv: []string{"TEST=secrethub://nonexistent/secret/path"}, + osStat: osStatNotExist, + }, newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ SecretService: &fakeclient.SecretService{ @@ -594,11 +608,13 @@ func TestRunCommand_Run(t *testing.T) { }, "os env secret not found ignored": { command: RunCommand{ - osEnv: []string{"TEST=secrethub://nonexistent/secret/path"}, ignoreMissingSecrets: true, command: []string{"echo", "test"}, io: ui.NewFakeIO(), - osStat: osStatNotExist, + environment: &environment{ + osEnv: []string{"TEST=secrethub://nonexistent/secret/path"}, + osStat: osStatNotExist, + }, newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ SecretService: &fakeclient.SecretService{ @@ -649,53 +665,65 @@ func TestRunCommand_environment(t *testing.T) { }{ "invalid template syntax": { command: RunCommand{ - command: []string{"echo", "test"}, - readFile: readFileFunc("secrethub.env", "TEST={{path/to/secret}"), - osStat: osStatFunc("secrethub.env", nil), - envFile: "secrethub.env", - templateVersion: "2", + command: []string{"echo", "test"}, + environment: &environment{ + osStat: osStatFunc("secrethub.env", nil), + readFile: readFileFunc("secrethub.env", "TEST={{path/to/secret}"), + envFile: "secrethub.env", + templateVersion: "2", + }, }, err: ErrParsingTemplate("secrethub.env", "template syntax error at 1:23: expected the closing of a secret tag `}}`, but reached the end of the template. (template.secret_tag_not_closed) "), }, "default env file does not exist": { command: RunCommand{ - osStat: osStatFunc("secrethub.env", os.ErrNotExist), + environment: &environment{ + osStat: osStatFunc("secrethub.env", os.ErrNotExist), + }, }, }, "default env file exists but cannot be read": { command: RunCommand{ - osStat: osStatFunc("secrethub.env", os.ErrPermission), + environment: &environment{ + osStat: osStatFunc("secrethub.env", os.ErrPermission), + }, }, err: ErrReadDefaultEnvFile(defaultEnvFile, os.ErrPermission), }, "custom env file does not exist": { command: RunCommand{ - envFile: "foo.env", - readFile: func(filename string) ([]byte, error) { - if filename == "foo.env" { - return nil, &os.PathError{Op: "open", Path: "foo.env", Err: os.ErrNotExist} - } - return nil, nil + environment: &environment{ + envFile: "foo.env", + readFile: func(filename string) ([]byte, error) { + if filename == "foo.env" { + return nil, &os.PathError{Op: "open", Path: "foo.env", Err: os.ErrNotExist} + } + return nil, nil + }, }, }, err: ErrCannotReadFile("foo.env", &os.PathError{Op: "open", Path: "foo.env", Err: os.ErrNotExist}), }, "custom env file success": { command: RunCommand{ - envFile: "foo.env", - templateVersion: "2", - osStat: osStatFunc("foo.env", nil), - readFile: readFileFunc("foo.env", "TEST=test"), + environment: &environment{ + osStat: osStatFunc("foo.env", nil), + envFile: "foo.env", + templateVersion: "2", + readFile: readFileFunc("foo.env", "TEST=test"), + }, }, expectedEnv: []string{"TEST=test"}, }, "env file secret does not exist": { command: RunCommand{ - command: []string{"echo", "test"}, - readFile: readFileFunc("secrethub.env", "TEST= {{ unexistent/secret/path }}"), - osStat: osStatFunc("secrethub.env", nil), - envFile: "secrethub.env", - templateVersion: "2", + command: []string{"echo", "test"}, + environment: &environment{ + osStat: osStatFunc("secrethub.env", nil), + readFile: readFileFunc("secrethub.env", "TEST= {{ unexistent/secret/path }}"), + envFile: "secrethub.env", + templateVersion: "2", + }, newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ SecretService: &fakeclient.SecretService{ @@ -712,13 +740,15 @@ func TestRunCommand_environment(t *testing.T) { }, "envar flag has precedence over env file": { command: RunCommand{ - readFile: readFileFunc("secrethub.env", "TEST=aaa"), - osStat: osStatFunc("secrethub.env", nil), - envFile: "secrethub.env", - envar: map[string]string{ - "TEST": "test/test/test", + environment: &environment{ + osStat: osStatFunc("secrethub.env", nil), + readFile: readFileFunc("secrethub.env", "TEST=aaa"), + envFile: "secrethub.env", + envar: map[string]string{ + "TEST": "test/test/test", + }, + templateVersion: "2", }, - templateVersion: "2", newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ SecretService: &fakeclient.SecretService{ @@ -737,11 +767,13 @@ func TestRunCommand_environment(t *testing.T) { // TODO Add test case for: envar flag has precedence over secret reference - requires refactoring of fakeclient "secret reference has precedence over .env file": { command: RunCommand{ - readFile: readFileFunc("secrethub.env", "TEST=aaa"), - osStat: osStatFunc("secrethub.env", nil), - dontPromptMissingTemplateVar: true, - templateVersion: "2", - osEnv: []string{"TEST=secrethub://test/test/test"}, + environment: &environment{ + osStat: osStatFunc("secrethub.env", nil), + readFile: readFileFunc("secrethub.env", "TEST=aaa"), + dontPromptMissingTemplateVar: true, + templateVersion: "2", + osEnv: []string{"TEST=secrethub://test/test/test"}, + }, newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ SecretService: &fakeclient.SecretService{ @@ -759,22 +791,26 @@ func TestRunCommand_environment(t *testing.T) { }, ".env file has precedence over other os variables": { command: RunCommand{ - readFile: readFileFunc("secrethub.env", "TEST=aaa"), - osStat: osStatFunc("secrethub.env", nil), - dontPromptMissingTemplateVar: true, - templateVersion: "2", - osEnv: []string{"TEST=bbb"}, + environment: &environment{ + osStat: osStatFunc("secrethub.env", nil), + readFile: readFileFunc("secrethub.env", "TEST=aaa"), + dontPromptMissingTemplateVar: true, + templateVersion: "2", + }, + osEnv: []string{"TEST=bbb"}, }, expectedSecrets: []string{}, expectedEnv: []string{"TEST=aaa"}, }, ".env file secret has precedence over other os variables": { command: RunCommand{ - readFile: readFileFunc("secrethub.env", "TEST={{path/to/secret}}"), - osStat: osStatFunc("secrethub.env", nil), - dontPromptMissingTemplateVar: true, - templateVersion: "2", - osEnv: []string{"TEST=bbb"}, + environment: &environment{ + osStat: osStatFunc("secrethub.env", nil), + readFile: readFileFunc("secrethub.env", "TEST={{path/to/secret}}"), + dontPromptMissingTemplateVar: true, + templateVersion: "2", + }, + osEnv: []string{"TEST=bbb"}, newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ SecretService: &fakeclient.SecretService{ @@ -793,13 +829,15 @@ func TestRunCommand_environment(t *testing.T) { "ignore missing secrets": { command: RunCommand{ ignoreMissingSecrets: true, - envFile: "secrethub.env", - readFile: readFileFunc("secrethub.env", ""), - osStat: osStatFunc("secrethub.env", nil), - envar: map[string]string{ - "TEST": "test/test/test", + environment: &environment{ + osStat: osStatFunc("secrethub.env", nil), + envFile: "secrethub.env", + readFile: readFileFunc("secrethub.env", ""), + envar: map[string]string{ + "TEST": "test/test/test", + }, + templateVersion: "2", }, - templateVersion: "2", newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ SecretService: &fakeclient.SecretService{ @@ -817,12 +855,14 @@ func TestRunCommand_environment(t *testing.T) { }, "--no-prompt": { command: RunCommand{ - readFile: readFileFunc("secrethub.env", "TEST = {{ test/$variable/test }}"), - osStat: osStatFunc("secrethub.env", nil), - noMasking: true, - dontPromptMissingTemplateVar: true, - envFile: "secrethub.env", - templateVersion: "2", + noMasking: true, + environment: &environment{ + osStat: osStatFunc("secrethub.env", nil), + readFile: readFileFunc("secrethub.env", "TEST = {{ test/$variable/test }}"), + dontPromptMissingTemplateVar: true, + envFile: "secrethub.env", + templateVersion: "2", + }, newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ SecretService: &fakeclient.SecretService{ @@ -839,12 +879,14 @@ func TestRunCommand_environment(t *testing.T) { }, "template var set in os environment": { command: RunCommand{ - readFile: readFileFunc("secrethub.env", "TEST = {{ test/$variable/test }}"), - osStat: osStatFunc("secrethub.env", nil), - noMasking: true, - dontPromptMissingTemplateVar: true, - templateVersion: "2", - osEnv: []string{"SECRETHUB_VAR_VARIABLE=test"}, + noMasking: true, + environment: &environment{ + osStat: osStatFunc("secrethub.env", nil), + readFile: readFileFunc("secrethub.env", "TEST = {{ test/$variable/test }}"), + dontPromptMissingTemplateVar: true, + templateVersion: "2", + osEnv: []string{"SECRETHUB_VAR_VARIABLE=test"}, + }, newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ SecretService: &fakeclient.SecretService{ @@ -861,12 +903,14 @@ func TestRunCommand_environment(t *testing.T) { }, "template var set by flag": { command: RunCommand{ - command: []string{"/bin/sh", "./test.sh"}, - readFile: readFileFunc("secrethub.env", "TEST = {{ test/$variable/test }}"), - osStat: osStatFunc("secrethub.env", nil), - dontPromptMissingTemplateVar: true, - templateVersion: "2", - templateVars: map[string]string{"variable": "test"}, + command: []string{"/bin/sh", "./test.sh"}, + environment: &environment{ + osStat: osStatFunc("secrethub.env", nil), + readFile: readFileFunc("secrethub.env", "TEST = {{ test/$variable/test }}"), + dontPromptMissingTemplateVar: true, + templateVersion: "2", + templateVars: map[string]string{"variable": "test"}, + }, newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ SecretService: &fakeclient.SecretService{ @@ -883,13 +927,15 @@ func TestRunCommand_environment(t *testing.T) { }, "template var set by flag has precedence over var set by environment": { command: RunCommand{ - command: []string{"/bin/sh", "./test.sh"}, - readFile: readFileFunc("secrethub.env", "TEST=$variable"), - osStat: osStatFunc("secrethub.env", nil), - dontPromptMissingTemplateVar: true, - templateVersion: "2", - templateVars: map[string]string{"variable": "foo"}, - osEnv: []string{"SECRETHUB_VAR_VARIABLE=bar"}, + command: []string{"/bin/sh", "./test.sh"}, + environment: &environment{ + osStat: osStatFunc("secrethub.env", nil), + readFile: readFileFunc("secrethub.env", "TEST=$variable"), + dontPromptMissingTemplateVar: true, + templateVersion: "2", + templateVars: map[string]string{"variable": "foo"}, + }, + osEnv: []string{"SECRETHUB_VAR_VARIABLE=bar"}, newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ SecretService: &fakeclient.SecretService{ @@ -906,10 +952,12 @@ func TestRunCommand_environment(t *testing.T) { }, "v1 template syntax success": { command: RunCommand{ - command: []string{"/bin/sh", "./test.sh"}, - readFile: readFileFunc("secrethub.env", "TEST= ${path/to/secret}"), - osStat: osStatFunc("secrethub.env", nil), - templateVersion: "1", + command: []string{"/bin/sh", "./test.sh"}, + environment: &environment{ + osStat: osStatFunc("secrethub.env", nil), + readFile: readFileFunc("secrethub.env", "TEST= ${path/to/secret}"), + templateVersion: "1", + }, newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ SecretService: &fakeclient.SecretService{ @@ -968,13 +1016,15 @@ func TestRunCommand_RunWithFile(t *testing.T) { command: RunCommand{ command: []string{"/bin/sh", "./test.sh"}, noMasking: true, - osStat: osStatOnlySecretHubEnv, - readFile: readFileWithContent(""), - envFile: "secrethub.env", - envar: map[string]string{ - "TEST": "test/test/test", + environment: &environment{ + osStat: osStatOnlySecretHubEnv, + readFile: readFileWithContent(""), + envFile: "secrethub.env", + envar: map[string]string{ + "TEST": "test/test/test", + }, + templateVersion: "2", }, - templateVersion: "2", newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ SecretService: &fakeclient.SecretService{ @@ -992,14 +1042,16 @@ func TestRunCommand_RunWithFile(t *testing.T) { "secret masking": { script: "echo $TEST", command: RunCommand{ - command: []string{"/bin/sh", "./test.sh"}, - envFile: "secrethub.env", - readFile: readFileWithContent(""), - osStat: osStatOnlySecretHubEnv, - envar: map[string]string{ - "TEST": "test/test/test", - }, - templateVersion: "2", + command: []string{"/bin/sh", "./test.sh"}, + environment: &environment{ + osStat: osStatOnlySecretHubEnv, + envFile: "secrethub.env", + readFile: readFileWithContent(""), + envar: map[string]string{ + "TEST": "test/test/test", + }, + templateVersion: "2", + }, newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ SecretService: &fakeclient.SecretService{ From d3b47ab0ca9a0a2d6dce1363188d3cafb6e31182 Mon Sep 17 00:00:00 2001 From: Simon Barendse Date: Fri, 28 Feb 2020 11:33:18 +0100 Subject: [PATCH 03/17] Make envSource.Env private --- internals/secrethub/env_source.go | 28 ++++++++++++++-------------- internals/secrethub/run.go | 4 ++-- internals/secrethub/run_test.go | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/internals/secrethub/env_source.go b/internals/secrethub/env_source.go index 460bb363..a8a138fd 100644 --- a/internals/secrethub/env_source.go +++ b/internals/secrethub/env_source.go @@ -53,7 +53,7 @@ func (env *environment) register(clause *cli.CommandClause) { clause.Flag("no-prompt", "Do not prompt when a template variable is missing and return an error instead.").BoolVar(&env.dontPromptMissingTemplateVar) } -func (env *environment) Env() (map[string]value, error) { +func (env *environment) env() (map[string]value, error) { osEnv, _ := parseKeyValueStringsToMap(env.osEnv) var sources []EnvSource @@ -108,7 +108,7 @@ func (env *environment) Env() (map[string]value, error) { var envs []map[string]value for _, source := range sources { - env, err := source.Env() + env, err := source.env() if err != nil { return nil, err } @@ -131,7 +131,7 @@ func mergeEnvs(envs ...map[string]value) map[string]value { // EnvSource defines a method of reading environment variables from a source. type EnvSource interface { // Env returns a map of key value pairs. - Env() (map[string]value, error) + env() (map[string]value, error) } type value interface { @@ -177,7 +177,7 @@ func NewEnvFlags(flags map[string]string) (EnvFlags, error) { // Env returns a map of environment variables sourced from // command-line flags and set to their corresponding value. -func (ef EnvFlags) Env() (map[string]value, error) { +func (ef EnvFlags) env() (map[string]value, error) { result := make(map[string]value) for name, path := range ef { result[name] = newSecretValue(path) @@ -207,7 +207,7 @@ func newReferenceEnv(osEnv map[string]string) *referenceEnv { // Env returns a map of key value pairs with the secrets configured with the // secrethub:// syntax. -func (env *referenceEnv) Env() (map[string]value, error) { +func (env *referenceEnv) env() (map[string]value, error) { envVarsWithSecrets := make(map[string]value) for key, path := range env.envVars { envVarsWithSecrets[key] = newSecretValue(path) @@ -259,7 +259,7 @@ func NewEnvDir(path string) (EnvDir, error) { } // Env returns a map of environment variables sourced from the directory. -func (dir EnvDir) Env() (map[string]value, error) { +func (dir EnvDir) env() (map[string]value, error) { return dir, nil } @@ -303,7 +303,7 @@ type envvarTpls struct { // Env injects the given secrets in the environment values and returns // a map of the resulting environment. -func (t envTemplate) Env() (map[string]value, error) { +func (t envTemplate) env() (map[string]value, error) { result := make(map[string]value) for _, tpls := range t.envVars { key, err := tpls.key.Evaluate(t.templateVarReader, secretReaderNotAllowed{}) @@ -337,20 +337,20 @@ func ReadEnvFile(filepath string, reader io.Reader, varReader tpl.VariableReader return EnvFile{}, ErrParsingTemplate(filepath, err) } return EnvFile{ - path: filepath, - env: env, + path: filepath, + envSource: env, }, nil } // EnvFile contains an environment that is read from a file. type EnvFile struct { - path string - env EnvSource + path string + envSource EnvSource } // Env returns a map of key value pairs read from the environment file. -func (e EnvFile) Env() (map[string]value, error) { - env, err := e.env.Env() +func (e EnvFile) env() (map[string]value, error) { + env, err := e.envSource.env() if err != nil { return nil, ErrParsingTemplate(e.path, err) } @@ -554,7 +554,7 @@ type osEnv struct { osEnv map[string]string } -func (o *osEnv) Env() (map[string]value, error) { +func (o *osEnv) env() (map[string]value, error) { res := map[string]value{} for name, value := range o.osEnv { res[name] = newPlaintextValue(value) diff --git a/internals/secrethub/run.go b/internals/secrethub/run.go index 70650577..88367738 100644 --- a/internals/secrethub/run.go +++ b/internals/secrethub/run.go @@ -204,13 +204,13 @@ func (cmd *RunCommand) sourceEnvironment() ([]string, []string, error) { if err != nil { return nil, nil, err } - dirValues, err = dirSource.Env() + dirValues, err = dirSource.env() if err != nil { return nil, nil, err } } - envValues, err := cmd.environment.Env() + envValues, err := cmd.environment.env() if err != nil { return nil, nil, err } diff --git a/internals/secrethub/run_test.go b/internals/secrethub/run_test.go index cf131e43..a0c6e991 100644 --- a/internals/secrethub/run_test.go +++ b/internals/secrethub/run_test.go @@ -449,7 +449,7 @@ func TestNewEnv(t *testing.T) { if err != nil { assert.Equal(t, err, tc.err) } else { - actualValues, err := env.Env() + actualValues, err := env.env() assert.Equal(t, err, tc.err) // resolve values From 2eaa005773175aeffc76d0ffddb2860fb32eb355 Mon Sep 17 00:00:00 2001 From: Simon Barendse Date: Fri, 28 Feb 2020 11:44:13 +0100 Subject: [PATCH 04/17] Move .seretsenv dir source to environment struct This also fixes the order of precedence, which was briefly changed some commits back. --- internals/secrethub/env_source.go | 14 ++++++++++++++ internals/secrethub/run.go | 21 --------------------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/internals/secrethub/env_source.go b/internals/secrethub/env_source.go index a8a138fd..aad3e7c3 100644 --- a/internals/secrethub/env_source.go +++ b/internals/secrethub/env_source.go @@ -12,6 +12,7 @@ import ( "unicode" "github.com/secrethub/secrethub-cli/internals/cli" + "github.com/secrethub/secrethub-cli/internals/secretspec" "github.com/secrethub/secrethub-cli/internals/cli/ui" @@ -31,6 +32,7 @@ type environment struct { templateVars map[string]string templateVersion string dontPromptMissingTemplateVar bool + secretsEnvDir string } func newEnvironment(io ui.IO) *environment { @@ -51,12 +53,24 @@ func (env *environment) register(clause *cli.CommandClause) { clause.Flag("var", "Define the value for a template variable with `VAR=VALUE`, e.g. --var env=prod").Short('v').StringMapVar(&env.templateVars) clause.Flag("template-version", "The template syntax version to be used. The options are v1, v2, latest or auto to automatically detect the version.").Default("auto").StringVar(&env.templateVersion) clause.Flag("no-prompt", "Do not prompt when a template variable is missing and return an error instead.").BoolVar(&env.dontPromptMissingTemplateVar) + clause.Flag("env", "The name of the environment prepared by the set command (default is `default`)").Default("default").Hidden().StringVar(&env.secretsEnvDir) } func (env *environment) env() (map[string]value, error) { osEnv, _ := parseKeyValueStringsToMap(env.osEnv) var sources []EnvSource + // .secretsenv dir (for backwards compatibility) + envDir := filepath.Join(secretspec.SecretEnvPath, env.secretsEnvDir) + _, err := os.Stat(envDir) + if err == nil { + dirSource, err := NewEnvDir(envDir) + if err != nil { + return nil, err + } + sources = append(sources, dirSource) + } + //secrethub.env file if env.envFile == "" { _, err := env.osStat(defaultEnvFile) diff --git a/internals/secrethub/run.go b/internals/secrethub/run.go index 88367738..4d3e48f9 100644 --- a/internals/secrethub/run.go +++ b/internals/secrethub/run.go @@ -5,15 +5,12 @@ import ( "os" "os/exec" "os/signal" - "path/filepath" "strings" "syscall" "time" "github.com/secrethub/secrethub-cli/internals/secrethub/tpl" - "github.com/secrethub/secrethub-cli/internals/secretspec" - "github.com/secrethub/secrethub-cli/internals/cli/ui" "github.com/secrethub/secrethub-cli/internals/cli/masker" @@ -54,7 +51,6 @@ type RunCommand struct { io ui.IO osEnv []string command []string - env string environment *environment noMasking bool maskingTimeout time.Duration @@ -83,7 +79,6 @@ func (cmd *RunCommand) Register(r command.Registerer) { clause.HelpLong(helpLong) clause.Alias("exec") clause.Arg("command", "The command to execute").Required().StringsVar(&cmd.command) - clause.Flag("env", "The name of the environment prepared by the set command (default is `default`)").Default("default").Hidden().StringVar(&cmd.env) clause.Flag("no-masking", "Disable masking of secrets on stdout and stderr").BoolVar(&cmd.noMasking) clause.Flag("masking-timeout", "The maximum time output is buffered. Warning: lowering this value increases the chance of secrets not being masked.").Default("1s").DurationVar(&cmd.maskingTimeout) clause.Flag("ignore-missing-secrets", "Do not return an error when a secret does not exist and use an empty value instead.").BoolVar(&cmd.ignoreMissingSecrets) @@ -195,26 +190,10 @@ func (cmd *RunCommand) sourceEnvironment() ([]string, []string, error) { newEnv[name] = value } - // add values from .secretsenv and other sources - var dirValues map[string]value - envDir := filepath.Join(secretspec.SecretEnvPath, cmd.env) - _, err := os.Stat(envDir) - if err == nil { - dirSource, err := NewEnvDir(envDir) - if err != nil { - return nil, nil, err - } - dirValues, err = dirSource.env() - if err != nil { - return nil, nil, err - } - } - envValues, err := cmd.environment.env() if err != nil { return nil, nil, err } - envValues = mergeEnvs(dirValues, envValues) var sr tpl.SecretReader = newSecretReader(cmd.newClient) if cmd.ignoreMissingSecrets { From 9d35b31ffafc952bcbfd2962f542720498f2a633 Mon Sep 17 00:00:00 2001 From: Simon Barendse Date: Fri, 28 Feb 2020 11:50:03 +0100 Subject: [PATCH 05/17] Preallocate envs variable --- internals/secrethub/env_source.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internals/secrethub/env_source.go b/internals/secrethub/env_source.go index aad3e7c3..200b52aa 100644 --- a/internals/secrethub/env_source.go +++ b/internals/secrethub/env_source.go @@ -120,7 +120,7 @@ func (env *environment) env() (map[string]value, error) { } sources = append(sources, flagEnv) - var envs []map[string]value + envs := make([]map[string]value, len(sources)) for _, source := range sources { env, err := source.env() if err != nil { From 708dc4799f5a33fec3f9522b59d754a048ea5aca Mon Sep 17 00:00:00 2001 From: Simon Barendse Date: Fri, 28 Feb 2020 12:59:03 +0100 Subject: [PATCH 06/17] Move os environment to environment struct --- internals/secrethub/env_source.go | 14 +++++++++----- internals/secrethub/run.go | 6 +----- internals/secrethub/run_test.go | 16 ++++++++++++---- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/internals/secrethub/env_source.go b/internals/secrethub/env_source.go index 200b52aa..db8502ca 100644 --- a/internals/secrethub/env_source.go +++ b/internals/secrethub/env_source.go @@ -24,7 +24,7 @@ import ( type environment struct { io ui.IO - osEnv []string + osEnv func() []string readFile func(filename string) ([]byte, error) osStat func(filename string) (os.FileInfo, error) envar map[string]string @@ -38,7 +38,7 @@ type environment struct { func newEnvironment(io ui.IO) *environment { return &environment{ io: io, - osEnv: os.Environ(), + osEnv: os.Environ, readFile: ioutil.ReadFile, osStat: os.Stat, templateVars: make(map[string]string), @@ -57,9 +57,13 @@ func (env *environment) register(clause *cli.CommandClause) { } func (env *environment) env() (map[string]value, error) { - osEnv, _ := parseKeyValueStringsToMap(env.osEnv) + osEnvMap, _ := parseKeyValueStringsToMap(env.osEnv()) var sources []EnvSource + sources = append(sources, &osEnv{ + osEnv: osEnvMap, + }) + // .secretsenv dir (for backwards compatibility) envDir := filepath.Join(secretspec.SecretEnvPath, env.secretsEnvDir) _, err := os.Stat(envDir) @@ -82,7 +86,7 @@ func (env *environment) env() (map[string]value, error) { } if env.envFile != "" { - templateVariableReader, err := newVariableReader(osEnv, env.templateVars) + templateVariableReader, err := newVariableReader(osEnvMap, env.templateVars) if err != nil { return nil, err } @@ -109,7 +113,7 @@ func (env *environment) env() (map[string]value, error) { } // secret references (secrethub://) - referenceEnv := newReferenceEnv(osEnv) + referenceEnv := newReferenceEnv(osEnvMap) sources = append(sources, referenceEnv) // --envar flag diff --git a/internals/secrethub/run.go b/internals/secrethub/run.go index 4d3e48f9..5be8b2e9 100644 --- a/internals/secrethub/run.go +++ b/internals/secrethub/run.go @@ -182,13 +182,9 @@ func (cmd *RunCommand) Run() error { // sourceEnvironment returns the environment of the subcommand, with all the secrets sourced // and the secret values that need to be masked. func (cmd *RunCommand) sourceEnvironment() ([]string, []string, error) { - osEnv, passthroughEnv := parseKeyValueStringsToMap(cmd.osEnv) + _, passthroughEnv := parseKeyValueStringsToMap(cmd.osEnv) - // add os envs newEnv := map[string]string{} - for name, value := range osEnv { - newEnv[name] = value - } envValues, err := cmd.environment.env() if err != nil { diff --git a/internals/secrethub/run_test.go b/internals/secrethub/run_test.go index a0c6e991..f044e47c 100644 --- a/internals/secrethub/run_test.go +++ b/internals/secrethub/run_test.go @@ -589,7 +589,9 @@ func TestRunCommand_Run(t *testing.T) { command: []string{"echo", "test"}, io: ui.NewFakeIO(), environment: &environment{ - osEnv: []string{"TEST=secrethub://nonexistent/secret/path"}, + osEnv: func() []string { + return []string{"TEST=secrethub://nonexistent/secret/path"} + }, osStat: osStatNotExist, }, newClient: func() (secrethub.ClientInterface, error) { @@ -612,7 +614,9 @@ func TestRunCommand_Run(t *testing.T) { command: []string{"echo", "test"}, io: ui.NewFakeIO(), environment: &environment{ - osEnv: []string{"TEST=secrethub://nonexistent/secret/path"}, + osEnv: func() []string { + return []string{"TEST=secrethub://nonexistent/secret/path"} + }, osStat: osStatNotExist, }, newClient: func() (secrethub.ClientInterface, error) { @@ -772,7 +776,9 @@ func TestRunCommand_environment(t *testing.T) { readFile: readFileFunc("secrethub.env", "TEST=aaa"), dontPromptMissingTemplateVar: true, templateVersion: "2", - osEnv: []string{"TEST=secrethub://test/test/test"}, + osEnv: func() []string { + return []string{"TEST=secrethub://test/test/test"} + }, }, newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ @@ -885,7 +891,9 @@ func TestRunCommand_environment(t *testing.T) { readFile: readFileFunc("secrethub.env", "TEST = {{ test/$variable/test }}"), dontPromptMissingTemplateVar: true, templateVersion: "2", - osEnv: []string{"SECRETHUB_VAR_VARIABLE=test"}, + osEnv: func() []string { + return []string{"SECRETHUB_VAR_VARIABLE=test"} + }, }, newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ From abacb0507032be42174fe8361da878609db161ab Mon Sep 17 00:00:00 2001 From: Simon Barendse Date: Fri, 28 Feb 2020 13:17:24 +0100 Subject: [PATCH 07/17] Undo making osEnv a function --- internals/secrethub/env_source.go | 6 +++--- internals/secrethub/run_test.go | 16 ++++------------ 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/internals/secrethub/env_source.go b/internals/secrethub/env_source.go index db8502ca..607d3e6f 100644 --- a/internals/secrethub/env_source.go +++ b/internals/secrethub/env_source.go @@ -24,7 +24,7 @@ import ( type environment struct { io ui.IO - osEnv func() []string + osEnv []string readFile func(filename string) ([]byte, error) osStat func(filename string) (os.FileInfo, error) envar map[string]string @@ -38,7 +38,7 @@ type environment struct { func newEnvironment(io ui.IO) *environment { return &environment{ io: io, - osEnv: os.Environ, + osEnv: os.Environ(), readFile: ioutil.ReadFile, osStat: os.Stat, templateVars: make(map[string]string), @@ -57,7 +57,7 @@ func (env *environment) register(clause *cli.CommandClause) { } func (env *environment) env() (map[string]value, error) { - osEnvMap, _ := parseKeyValueStringsToMap(env.osEnv()) + osEnvMap, _ := parseKeyValueStringsToMap(env.osEnv) var sources []EnvSource sources = append(sources, &osEnv{ diff --git a/internals/secrethub/run_test.go b/internals/secrethub/run_test.go index f044e47c..a0c6e991 100644 --- a/internals/secrethub/run_test.go +++ b/internals/secrethub/run_test.go @@ -589,9 +589,7 @@ func TestRunCommand_Run(t *testing.T) { command: []string{"echo", "test"}, io: ui.NewFakeIO(), environment: &environment{ - osEnv: func() []string { - return []string{"TEST=secrethub://nonexistent/secret/path"} - }, + osEnv: []string{"TEST=secrethub://nonexistent/secret/path"}, osStat: osStatNotExist, }, newClient: func() (secrethub.ClientInterface, error) { @@ -614,9 +612,7 @@ func TestRunCommand_Run(t *testing.T) { command: []string{"echo", "test"}, io: ui.NewFakeIO(), environment: &environment{ - osEnv: func() []string { - return []string{"TEST=secrethub://nonexistent/secret/path"} - }, + osEnv: []string{"TEST=secrethub://nonexistent/secret/path"}, osStat: osStatNotExist, }, newClient: func() (secrethub.ClientInterface, error) { @@ -776,9 +772,7 @@ func TestRunCommand_environment(t *testing.T) { readFile: readFileFunc("secrethub.env", "TEST=aaa"), dontPromptMissingTemplateVar: true, templateVersion: "2", - osEnv: func() []string { - return []string{"TEST=secrethub://test/test/test"} - }, + osEnv: []string{"TEST=secrethub://test/test/test"}, }, newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ @@ -891,9 +885,7 @@ func TestRunCommand_environment(t *testing.T) { readFile: readFileFunc("secrethub.env", "TEST = {{ test/$variable/test }}"), dontPromptMissingTemplateVar: true, templateVersion: "2", - osEnv: func() []string { - return []string{"SECRETHUB_VAR_VARIABLE=test"} - }, + osEnv: []string{"SECRETHUB_VAR_VARIABLE=test"}, }, newClient: func() (secrethub.ClientInterface, error) { return fakeclient.Client{ From 46ad1c2534c01f5a31fbe07cc1d2caf8fb165d79 Mon Sep 17 00:00:00 2001 From: Simon Barendse Date: Fri, 28 Feb 2020 13:21:32 +0100 Subject: [PATCH 08/17] Also set the osEnv in the environment struct --- internals/secrethub/run_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internals/secrethub/run_test.go b/internals/secrethub/run_test.go index a0c6e991..ea83422c 100644 --- a/internals/secrethub/run_test.go +++ b/internals/secrethub/run_test.go @@ -929,6 +929,7 @@ func TestRunCommand_environment(t *testing.T) { command: RunCommand{ command: []string{"/bin/sh", "./test.sh"}, environment: &environment{ + osEnv: []string{"SECRETHUB_VAR_VARIABLE=bar"}, osStat: osStatFunc("secrethub.env", nil), readFile: readFileFunc("secrethub.env", "TEST=$variable"), dontPromptMissingTemplateVar: true, From 0fcd5662354868607dc2cf051fad3f138e79096f Mon Sep 17 00:00:00 2001 From: Simon Barendse Date: Fri, 28 Feb 2020 13:31:56 +0100 Subject: [PATCH 09/17] Add env read command (doesn't do anything yet) --- internals/secrethub/app.go | 1 + internals/secrethub/env.go | 26 ++++++++++++++++++++++++++ internals/secrethub/env_read.go | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 internals/secrethub/env.go create mode 100644 internals/secrethub/env_read.go diff --git a/internals/secrethub/app.go b/internals/secrethub/app.go index e8755fdd..346cc69c 100644 --- a/internals/secrethub/app.go +++ b/internals/secrethub/app.go @@ -163,6 +163,7 @@ func (app *App) registerCommands() { NewAccountCommand(app.io, app.clientFactory.NewClient, app.credentialStore).Register(app.cli) NewCredentialCommand(app.io, app.clientFactory, app.credentialStore).Register(app.cli) NewConfigCommand(app.io, app.credentialStore).Register(app.cli) + NewEnvCommand(app.io, app.clientFactory.NewClient).Register(app.cli) // Commands NewInitCommand(app.io, app.clientFactory.NewUnauthenticatedClient, app.clientFactory.NewClientWithCredentials, app.credentialStore).Register(app.cli) diff --git a/internals/secrethub/env.go b/internals/secrethub/env.go new file mode 100644 index 00000000..9968a361 --- /dev/null +++ b/internals/secrethub/env.go @@ -0,0 +1,26 @@ +package secrethub + +import ( + "github.com/secrethub/secrethub-cli/internals/cli/ui" + "github.com/secrethub/secrethub-cli/internals/secrethub/command" +) + +// EnvCommand handles operations regarding environment variables. +type EnvCommand struct { + io ui.IO + newClient newClientFunc +} + +// NewEnvCommand creates a new EnvCommand. +func NewEnvCommand(io ui.IO, newClient newClientFunc) *EnvCommand { + return &EnvCommand{ + io: io, + newClient: newClient, + } +} + +// Register registers the command and its sub-commands on the provided Registerer. +func (cmd *EnvCommand) Register(r command.Registerer) { + clause := r.Command("env", "Manage environment variables.").Hidden() // The command is hidden, because it's still in beta. + NewEnvReadCommand(cmd.io, cmd.newClient).Register(clause) +} diff --git a/internals/secrethub/env_read.go b/internals/secrethub/env_read.go new file mode 100644 index 00000000..eb9d0ba3 --- /dev/null +++ b/internals/secrethub/env_read.go @@ -0,0 +1,32 @@ +package secrethub + +import ( + "github.com/secrethub/secrethub-cli/internals/cli/ui" + "github.com/secrethub/secrethub-cli/internals/secrethub/command" +) + +// EnvReadCommand is a command to read the value of a single environment variable. +type EnvReadCommand struct { + io ui.IO + newClient newClientFunc +} + +// NewEnvReadCommand creates a new EnvReadCommand. +func NewEnvReadCommand(io ui.IO, newClient newClientFunc) *EnvReadCommand { + return &EnvReadCommand{ + io: io, + newClient: newClient, + } +} + +// Register adds a CommandClause and it's args and flags to a Registerer. +func (cmd *EnvReadCommand) Register(r command.Registerer) { + clause := r.Command("read", "Read the value of a single environment variable.") + + command.BindAction(clause, cmd.Run) +} + +// Run handles the command with the options as specified in the command. +func (cmd *EnvReadCommand) Run() error { + return nil +} From 83244a8d69eafae2f84b68578e7457ffd13a64f0 Mon Sep 17 00:00:00 2001 From: Simon Barendse Date: Fri, 28 Feb 2020 13:34:34 +0100 Subject: [PATCH 10/17] Add environment to env read command --- internals/secrethub/env_read.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/internals/secrethub/env_read.go b/internals/secrethub/env_read.go index eb9d0ba3..b80e5dc0 100644 --- a/internals/secrethub/env_read.go +++ b/internals/secrethub/env_read.go @@ -7,15 +7,17 @@ import ( // EnvReadCommand is a command to read the value of a single environment variable. type EnvReadCommand struct { - io ui.IO - newClient newClientFunc + io ui.IO + newClient newClientFunc + environment *environment } // NewEnvReadCommand creates a new EnvReadCommand. func NewEnvReadCommand(io ui.IO, newClient newClientFunc) *EnvReadCommand { return &EnvReadCommand{ - io: io, - newClient: newClient, + io: io, + newClient: newClient, + environment: newEnvironment(io), } } @@ -23,6 +25,8 @@ func NewEnvReadCommand(io ui.IO, newClient newClientFunc) *EnvReadCommand { func (cmd *EnvReadCommand) Register(r command.Registerer) { clause := r.Command("read", "Read the value of a single environment variable.") + cmd.environment.register(clause) + command.BindAction(clause, cmd.Run) } From 3be71fc042580c72708e22bc9b266772e79b5ce8 Mon Sep 17 00:00:00 2001 From: Simon Barendse Date: Fri, 28 Feb 2020 13:36:52 +0100 Subject: [PATCH 11/17] Add key arg to env read command --- internals/secrethub/env_read.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internals/secrethub/env_read.go b/internals/secrethub/env_read.go index b80e5dc0..67417077 100644 --- a/internals/secrethub/env_read.go +++ b/internals/secrethub/env_read.go @@ -10,6 +10,7 @@ type EnvReadCommand struct { io ui.IO newClient newClientFunc environment *environment + key string } // NewEnvReadCommand creates a new EnvReadCommand. @@ -24,6 +25,7 @@ func NewEnvReadCommand(io ui.IO, newClient newClientFunc) *EnvReadCommand { // Register adds a CommandClause and it's args and flags to a Registerer. func (cmd *EnvReadCommand) Register(r command.Registerer) { clause := r.Command("read", "Read the value of a single environment variable.") + clause.Arg("key", "the key of the environment variable to read").StringVar(&cmd.key) cmd.environment.register(clause) From 1b26c30a19bdb913a4980b3e90b882950dc928ee Mon Sep 17 00:00:00 2001 From: Simon Barendse Date: Fri, 28 Feb 2020 13:40:59 +0100 Subject: [PATCH 12/17] Implement env read command --- internals/secrethub/env_read.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/internals/secrethub/env_read.go b/internals/secrethub/env_read.go index 67417077..56b037fb 100644 --- a/internals/secrethub/env_read.go +++ b/internals/secrethub/env_read.go @@ -1,6 +1,8 @@ package secrethub import ( + "fmt" + "github.com/secrethub/secrethub-cli/internals/cli/ui" "github.com/secrethub/secrethub-cli/internals/secrethub/command" ) @@ -34,5 +36,24 @@ func (cmd *EnvReadCommand) Register(r command.Registerer) { // Run handles the command with the options as specified in the command. func (cmd *EnvReadCommand) Run() error { + env, err := cmd.environment.env() + if err != nil { + return err + } + + value, found := env[cmd.key] + if !found { + return fmt.Errorf("no environment variable with that key is set") + } + + secretReader := newSecretReader(cmd.newClient) + + res, err := value.resolve(secretReader) + if err != nil { + return err + } + + fmt.Fprintln(cmd.io.Stdout(), res) + return nil } From 40bf7218e7a79b6583903aaa7d4f669fbc653d1e Mon Sep 17 00:00:00 2001 From: Simon Barendse Date: Fri, 28 Feb 2020 13:49:14 +0100 Subject: [PATCH 13/17] Add env ls command --- internals/secrethub/env.go | 1 + internals/secrethub/env_ls.go | 50 +++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 internals/secrethub/env_ls.go diff --git a/internals/secrethub/env.go b/internals/secrethub/env.go index 9968a361..b6e3fca6 100644 --- a/internals/secrethub/env.go +++ b/internals/secrethub/env.go @@ -23,4 +23,5 @@ func NewEnvCommand(io ui.IO, newClient newClientFunc) *EnvCommand { func (cmd *EnvCommand) Register(r command.Registerer) { clause := r.Command("env", "Manage environment variables.").Hidden() // The command is hidden, because it's still in beta. NewEnvReadCommand(cmd.io, cmd.newClient).Register(clause) + NewEnvListCommand(cmd.io).Register(clause) } diff --git a/internals/secrethub/env_ls.go b/internals/secrethub/env_ls.go new file mode 100644 index 00000000..36971125 --- /dev/null +++ b/internals/secrethub/env_ls.go @@ -0,0 +1,50 @@ +package secrethub + +import ( + "fmt" + + "github.com/secrethub/secrethub-cli/internals/cli/ui" + "github.com/secrethub/secrethub-cli/internals/secrethub/command" +) + +// EnvListCommand is a command to list all environment variable keys set in the process of `secrethub run`. +type EnvListCommand struct { + io ui.IO + environment *environment +} + +// NewEnvListCommand creates a new EnvListCommand. +func NewEnvListCommand(io ui.IO) *EnvListCommand { + return &EnvListCommand{ + io: io, + environment: newEnvironment(io), + } +} + +// Register adds a CommandClause and it's args and flags to a Registerer. +func (cmd *EnvListCommand) Register(r command.Registerer) { + clause := r.Command("ls", "Read the value of a single environment variable.") + clause.Alias("list") + + cmd.environment.register(clause) + + command.BindAction(clause, cmd.Run) +} + +// Run handles the command with the options as specified in the command. +func (cmd *EnvListCommand) Run() error { + env, err := cmd.environment.env() + if err != nil { + return err + } + + for key, value := range env { + // For now only environment variables in which a secret is loaded are printed. + // TODO: Make this behavior configurable. + if value.containsSecret() { + fmt.Fprintln(cmd.io.Stdout(), key) + } + } + + return nil +} From d0ac787c2829d514641a1c48eec7c4828fbea358 Mon Sep 17 00:00:00 2001 From: Simon Barendse Date: Fri, 28 Feb 2020 14:19:31 +0100 Subject: [PATCH 14/17] Change Run godoc --- internals/secrethub/env_ls.go | 2 +- internals/secrethub/env_read.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internals/secrethub/env_ls.go b/internals/secrethub/env_ls.go index 36971125..229be99c 100644 --- a/internals/secrethub/env_ls.go +++ b/internals/secrethub/env_ls.go @@ -31,7 +31,7 @@ func (cmd *EnvListCommand) Register(r command.Registerer) { command.BindAction(clause, cmd.Run) } -// Run handles the command with the options as specified in the command. +// Run executes the command. func (cmd *EnvListCommand) Run() error { env, err := cmd.environment.env() if err != nil { diff --git a/internals/secrethub/env_read.go b/internals/secrethub/env_read.go index 56b037fb..edce65c5 100644 --- a/internals/secrethub/env_read.go +++ b/internals/secrethub/env_read.go @@ -34,7 +34,7 @@ func (cmd *EnvReadCommand) Register(r command.Registerer) { command.BindAction(clause, cmd.Run) } -// Run handles the command with the options as specified in the command. +// Run executes the command. func (cmd *EnvReadCommand) Run() error { env, err := cmd.environment.env() if err != nil { From e5c5aead796ebeda234a0af53abe07459f81a69b Mon Sep 17 00:00:00 2001 From: Simon Barendse Date: Mon, 2 Mar 2020 14:12:37 +0100 Subject: [PATCH 15/17] Fix env ls help text --- internals/secrethub/env_ls.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internals/secrethub/env_ls.go b/internals/secrethub/env_ls.go index 229be99c..2d68991b 100644 --- a/internals/secrethub/env_ls.go +++ b/internals/secrethub/env_ls.go @@ -23,7 +23,7 @@ func NewEnvListCommand(io ui.IO) *EnvListCommand { // Register adds a CommandClause and it's args and flags to a Registerer. func (cmd *EnvListCommand) Register(r command.Registerer) { - clause := r.Command("ls", "Read the value of a single environment variable.") + clause := r.Command("ls", "List environment variable keys that will be injected with secrets.") clause.Alias("list") cmd.environment.register(clause) From b535ac9f9d9be282013b1a7471c7d2d96581a28a Mon Sep 17 00:00:00 2001 From: Simon Barendse Date: Mon, 2 Mar 2020 14:18:43 +0100 Subject: [PATCH 16/17] Add to help text of env commands that they are in beta --- internals/secrethub/env.go | 3 ++- internals/secrethub/env_ls.go | 3 ++- internals/secrethub/env_read.go | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/internals/secrethub/env.go b/internals/secrethub/env.go index b6e3fca6..845d50f3 100644 --- a/internals/secrethub/env.go +++ b/internals/secrethub/env.go @@ -21,7 +21,8 @@ func NewEnvCommand(io ui.IO, newClient newClientFunc) *EnvCommand { // Register registers the command and its sub-commands on the provided Registerer. func (cmd *EnvCommand) Register(r command.Registerer) { - clause := r.Command("env", "Manage environment variables.").Hidden() // The command is hidden, because it's still in beta. + clause := r.Command("env", "[BETA] Manage environment variables.").Hidden() + clause.HelpLong("This command is hidden because it is still in beta. Future versions may break.") NewEnvReadCommand(cmd.io, cmd.newClient).Register(clause) NewEnvListCommand(cmd.io).Register(clause) } diff --git a/internals/secrethub/env_ls.go b/internals/secrethub/env_ls.go index 2d68991b..7202a36d 100644 --- a/internals/secrethub/env_ls.go +++ b/internals/secrethub/env_ls.go @@ -23,7 +23,8 @@ func NewEnvListCommand(io ui.IO) *EnvListCommand { // Register adds a CommandClause and it's args and flags to a Registerer. func (cmd *EnvListCommand) Register(r command.Registerer) { - clause := r.Command("ls", "List environment variable keys that will be injected with secrets.") + clause := r.Command("ls", "[BETA] List environment variable keys that will be injected with secrets.") + clause.HelpLong("This command is hidden because it is still in beta. Future versions may break.") clause.Alias("list") cmd.environment.register(clause) diff --git a/internals/secrethub/env_read.go b/internals/secrethub/env_read.go index edce65c5..3f1059bb 100644 --- a/internals/secrethub/env_read.go +++ b/internals/secrethub/env_read.go @@ -26,7 +26,8 @@ func NewEnvReadCommand(io ui.IO, newClient newClientFunc) *EnvReadCommand { // Register adds a CommandClause and it's args and flags to a Registerer. func (cmd *EnvReadCommand) Register(r command.Registerer) { - clause := r.Command("read", "Read the value of a single environment variable.") + clause := r.Command("read", "[BETA] Read the value of a single environment variable.") + clause.HelpLong("This command is hidden because it is still in beta. Future versions may break.") clause.Arg("key", "the key of the environment variable to read").StringVar(&cmd.key) cmd.environment.register(clause) From fdb745511cb771ae1841d61771a990e33218f911 Mon Sep 17 00:00:00 2001 From: Simon Barendse Date: Mon, 2 Mar 2020 14:36:22 +0100 Subject: [PATCH 17/17] Rephrase env ls help text Co-authored-by: Floris van der Grinten --- internals/secrethub/env_ls.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internals/secrethub/env_ls.go b/internals/secrethub/env_ls.go index 7202a36d..2f4908be 100644 --- a/internals/secrethub/env_ls.go +++ b/internals/secrethub/env_ls.go @@ -23,7 +23,7 @@ func NewEnvListCommand(io ui.IO) *EnvListCommand { // Register adds a CommandClause and it's args and flags to a Registerer. func (cmd *EnvListCommand) Register(r command.Registerer) { - clause := r.Command("ls", "[BETA] List environment variable keys that will be injected with secrets.") + clause := r.Command("ls", "[BETA] List environment variable names that will be populated with secrets.") clause.HelpLong("This command is hidden because it is still in beta. Future versions may break.") clause.Alias("list")