Params by path (#12)

* save

* Working-ish sketch

* Remove debug logging, add some tests

* log.Fatal -> log.Fatalf

* Trim trailing slash, refactor/regroup GetValue stuff

* Update tests

* Address #14 by removing log.Fatalf

Author: cmattoon

README.md

@@ -129,6 +129,22 @@ data:
 ```
 
 
+```
+apiVersion: v1
+kind: Secret
+metadata:
+  name: my-secret
+  annotations:
+    "alpha.ssm.cmattoon.com/k8s-secret-name": app-secrets
+    "alpha.ssm.cmattoon.com/aws-param-name": /path/to/env
+    "alpha.ssm.cmattoon.com/aws-param-type": Directory
+    "alpha.ssm.cmattoon.com/aws-param-key": "alias/aws/ssm"
+data:
+  file1: value1
+  file2: value2
+```
+
+
 Build
 -----
 

example.yaml examples/01-basic-SecureString.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: multi-secret
+  annotations:
+    alpha.ssm.cmattoon.com/aws-param-type: Directory
+    alpha.ssm.cmattoon.com/aws-param-name: /path/to/test
+    alpha.ssm.cmattoon.com/aws-param-key: alias/aws/ssm

multi-secret.yaml

@@ -72,6 +72,7 @@ func (c *Controller) HandleSecrets(cli kubernetes.Interface) error {
 		_, err = obj.UpdateObject(cli)
 		if err != nil {
 			log.Warnf("Failed to update object %s/%s", obj.Namespace, obj.Name)
+			log.Warn(err.Error())
 			continue
 		}
 		log.Infof("Successfully updated %s/%s", obj.Namespace, obj.Name)

pkg/controller/controller.go

@@ -16,13 +16,14 @@
 package provider
 
 import (
-	log "github.com/sirupsen/logrus"
+	"path"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ssm"
 	"github.com/cmattoon/aws-ssm/pkg/config"
+	log "github.com/sirupsen/logrus"
 )
 
 type AWSProvider struct {
@@ -47,16 +48,37 @@ func NewAWSProvider(cfg *config.Config) (Provider, error) {
 }
 
 func (p AWSProvider) GetParameterValue(name string, decrypt bool) (string, error) {
-	log.Debugf("GetParameterValue(%v, %v)", name, decrypt)
 	param, err := p.Service.GetParameter(&ssm.GetParameterInput{
 		Name:           aws.String(name),
 		WithDecryption: aws.Bool(decrypt),
 	})
 
 	if err != nil {
-		log.Debug("Failed to get value. Returning ''")
+		log.Errorf("Failed to GetParameterValue: %s", err)
 		return "", err
 	}
 
 	return *param.Parameter.Value, nil
 }
+
+func (p AWSProvider) GetParameterDataByPath(ppath string, decrypt bool) (map[string]string, error) {
+	// ppath is something like /path/to/env
+	params, err := p.Service.GetParametersByPath(&ssm.GetParametersByPathInput{
+		Path:           aws.String(ppath),
+		Recursive:      aws.Bool(true),
+		WithDecryption: aws.Bool(decrypt),
+	})
+
+	if err != nil {
+		log.Errorf("Failed to GetParameterDataByPath: %s", err)
+		return nil, err
+	}
+
+	results := make(map[string]string)
+	// '/path/to/env/foo' -> 'foo': *pa.Value
+	for _, pa := range params.Parameters {
+		_, basename := path.Split(*pa.Name)
+		results[basename] = *pa.Value
+	}
+	return results, nil
+}

pkg/provider/aws.go

@@ -16,28 +16,40 @@
 package provider
 
 import (
+	"errors"
 	//log "github.com/sirupsen/logrus"
 	"github.com/cmattoon/aws-ssm/pkg/config"
 )
 
 type Provider interface {
 	GetParameterValue(string, bool) (string, error)
+	GetParameterDataByPath(string, bool) (map[string]string, error)
 }
 
 func NewProvider(cfg *config.Config) (Provider, error) {
 	p, err := NewAWSProvider(cfg)
 	return p, err
 }
 
+// Mock an error with {"(error)", "error message"}
 type MockProvider struct {
-	Value          string
-	DecryptedValue string
+	Value             string
+	DecryptedValue    string
+	DirectoryContents map[string]string
 }
 
 func (mp MockProvider) GetParameterValue(s string, b bool) (string, error) {
+	if mp.Value == "(error)" {
+		return "", errors.New(mp.DecryptedValue)
+	}
+
 	if b {
 		// Decrypt flag
 		return mp.DecryptedValue, nil
 	}
 	return mp.Value, nil
 }
+
+func (mp MockProvider) GetParameterDataByPath(s string, b bool) (map[string]string, error) {
+	return mp.DirectoryContents, nil
+}

pkg/provider/provider.go

@@ -59,23 +59,51 @@ func NewSecret(sec v1.Secret, p provider.Provider, secret_name string, secret_na
 		Data:       map[string]string{},
 	}
 
-	log.Infof("Getting value for '%s/%s'", s.Namespace, s.Name)
+	log.Debugf("Getting value for '%s/%s'", s.Namespace, s.Name)
 
 	decrypt := false
 	if s.ParamKey != "" {
 		decrypt = true
 	}
 
-	value, err := p.GetParameterValue(s.ParamName, decrypt)
-
-	if err != nil {
-		log.Infof("Couldn't get value for %s/%s: %s",
-			s.Namespace, s.Name, err)
-		return nil, err
-	} else {
+	if s.ParamType == "String" || s.ParamType == "SecureString" {
+		value, err := p.GetParameterValue(s.ParamName, decrypt)
+		if err != nil {
+			return nil, err
+		}
 		s.ParamValue = value
+	} else if s.ParamType == "StringList" {
+		value, err := p.GetParameterValue(s.ParamName, decrypt)
+		if err != nil {
+			return nil, err
+		}
+		s.ParamValue = value
+		// StringList: Also set each key
+		values := s.ParseStringList()
+		for k, v := range values {
+			s.Set(k, v)
+		}
+	} else if s.ParamType == "Directory" {
+		// Directory: Set each sub-key
+		all_params, err := p.GetParameterDataByPath(s.ParamName, decrypt)
+		if err != nil {
+			return nil, err
+		}
+
+		for k, v := range all_params {
+			s.Set(safeKeyName(k), v)
+		}
+		s.ParamValue = "true" // Reads "Directory": "true"
+		return s, nil
 	}
 
+	// Always set the "$ParamType" key:
+	//   String: Value
+	//   SecureString: Value
+	//   StringList: Value
+	//   Directory: <ssm-path>
+	s.Set(s.ParamType, s.ParamValue)
+
 	return s, nil
 }
 
@@ -147,6 +175,7 @@ func (s *Secret) ParseStringList() (values map[string]string) {
 }
 
 func (s *Secret) Set(key string, val string) (err error) {
+	log.Debugf("Setting key=%s", key)
 	if s.Secret.StringData == nil {
 		s.Secret.StringData = make(map[string]string)
 	}
@@ -161,20 +190,13 @@ func (s *Secret) Set(key string, val string) (err error) {
 
 func (s *Secret) UpdateObject(cli kubernetes.Interface) (result *v1.Secret, err error) {
 	log.Info("Updating Kubernetes Secret...")
+	return cli.CoreV1().Secrets(s.Namespace).Update(&s.Secret)
+}
 
-	// Always set the "$ParamType" key:
-	//   String: Value
-	//   SecureString: Value
-	//   StringList: Value
-	s.Set(s.ParamType, s.ParamValue)
-
-	// If it's a StringList, attempt to set each k/v pair separately
-	if s.ParamType == "StringList" {
-		values := s.ParseStringList()
-		for k, v := range values {
-			s.Set(k, v)
-		}
+func safeKeyName(key string) string {
+	key = strings.TrimRight(key, "/")
+	if strings.HasPrefix(key, "/") {
+		key = strings.Replace(key, "/", "", 1)
 	}
-
-	return cli.CoreV1().Secrets(s.Namespace).Update(&s.Secret)
+	return strings.Replace(key, "/", "_", -1)
 }

pkg/secret/secret.go

@@ -21,7 +21,7 @@ import (
 
 	"github.com/cmattoon/aws-ssm/pkg/provider"
 	"github.com/stretchr/testify/assert"
-	//"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/require"
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -137,10 +137,8 @@ func TestSet(t *testing.T) {
 		ParamValue: "key1=val1,key2=val2,key3=val3,key4=val4=true",
 		Data:       map[string]string{},
 	}
-	s.Set("foo", "bar")
-	if s.Secret.StringData["foo"] != "bar" {
-		t.Fail()
-	}
+	require.NoError(t, s.Set("foo", "bar"))
+	assert.Equal(t, s.Secret.StringData["foo"], "bar")
 }
 
 func TestSetRefusesToOverwriteKey(t *testing.T) {
@@ -151,48 +149,58 @@ func TestSetRefusesToOverwriteKey(t *testing.T) {
 		ParamType:  "StringList",
 		ParamKey:   "foo-param",
 		ParamValue: "key1=val1,key2=val2,key3=val3,key4=val4=true",
-		Data:       map[string]string{},
-	}
-	err := s.Set("foo", "bar")
-	if err != nil {
-		t.Fail()
-	}
-	err = s.Set("foo", "baz")
-	if err != nil {
-		t.Fail()
-	}
-	if s.Secret.StringData["foo"] != "baz" {
-		t.Fail()
+		Data: map[string]string{
+			"foo": "bar",
+		},
 	}
+	require.Error(t, s.Set("foo", "baz"))
+
+	assert.Equal(t, "bar", s.Data["foo"])
+	// "foo=bar" was specified initially, not via Set
+	// so strictly speaking, it shouldn't be set here yet
+	// (because s.Set() should fail)
+	assert.Equal(t, "", s.Secret.StringData["foo"])
 }
 
 func TestSetsValue(t *testing.T) {
-	p := provider.MockProvider{"FooBar123", "PlaintextIsAnError"}
+	p := provider.MockProvider{"FooBar123", "PlaintextIsAnError", make(map[string]string)}
 	s := v1.Secret{}
 	testSecret, err := NewSecret(s, p, "foo-secret", "namespace", "foo-param", "String", "")
-	if err != nil {
-		t.Fail()
-	}
-	if testSecret.ParamValue != "FooBar123" {
-		t.Fail()
-	}
+
+	assert.Equal(t, err, nil)
+	assert.Equal(t, testSecret.ParamValue, "FooBar123")
 }
 
 // When the encryption key is defined, the decrypted value should be returned
 func TestNewSecretDecryptsIfKeyIsSet(t *testing.T) {
-	p := provider.MockProvider{"$@#*$(@)*$", "FooBar123"}
+	p := provider.MockProvider{"$@#*$(@)*$", "FooBar123", make(map[string]string)}
 	s := v1.Secret{}
 	testSecret, err := NewSecret(s, p, "foo-secret", "namespace", "foo-param", "String", "my/test/key")
-	if err != nil {
-		t.Fail()
+	assert.Equal(t, nil, err)
+	assert.Equal(t, testSecret.ParamValue, p.DecryptedValue)
+}
+
+func TestNewSecretHandlesStringList(t *testing.T) {
+	p := provider.MockProvider{"$@#*$(@)*$", "key1=val1,key2=val2,key3=val3", make(map[string]string)}
+	s := v1.Secret{}
+	ts, err := NewSecret(s, p, "foo-secret", "namespace", "foo-param", "StringList", "my/test/key")
+	assert.True(t, err == nil)
+	exp := map[string]string{
+		"key1": "val1",
+		"key2": "val2",
+		"key3": "val3",
 	}
-	if testSecret.ParamValue != p.DecryptedValue {
-		t.Fail()
+
+	for k, v := range exp {
+		v3, ok := ts.Secret.StringData[k]
+		assert.Equal(t, ok, true)
+		assert.Equal(t, v, v3)
 	}
+
 }
 
 func TestFromKubernetesSecretReturnsErrorIfIrrelevant(t *testing.T) {
-	p := provider.MockProvider{"$@#*$(@)*$", "FooBar123"}
+	p := provider.MockProvider{"$@#*$(@)*$", "FooBar123", make(map[string]string)}
 	s := v1.Secret{} // No annotations, so no params
 
 	_, err := FromKubernetesSecret(p, s)
@@ -204,7 +212,7 @@ func TestFromKubernetesSecretReturnsErrorIfIrrelevant(t *testing.T) {
 // If the parameter is of Type=SecureString, and no key is supplied,
 // attempt to use the default key.
 func TestFromKubernetesSecretUsesDefaultEncryptionKey(t *testing.T) {
-	p := provider.MockProvider{"$@#*$(@)*$", "FooBar123"}
+	p := provider.MockProvider{"$@#*$(@)*$", "FooBar123", make(map[string]string)}
 
 	s := v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
@@ -223,7 +231,7 @@ func TestFromKubernetesSecretUsesDefaultEncryptionKey(t *testing.T) {
 }
 
 func TestFromKubernetesSecretUsesSpecifiedEncryptionKey(t *testing.T) {
-	p := provider.MockProvider{"$@#*$(@)*$", "FooBar123"}
+	p := provider.MockProvider{"$@#*$(@)*$", "FooBar123", make(map[string]string)}
 
 	s := v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
@@ -241,3 +249,16 @@ func TestFromKubernetesSecretUsesSpecifiedEncryptionKey(t *testing.T) {
 		t.Fail()
 	}
 }
+
+func TestSafeKeyName(t *testing.T) {
+	keys := map[string]string{
+		"/foo/bar":     "foo_bar",
+		"/foo/bar/":    "foo_bar",
+		"//foo/bar":    "_foo_bar",
+		"//foo/bar/":   "_foo_bar",
+		"/foo/bar/baz": "foo_bar_baz",
+	}
+	for path, exp := range keys {
+		assert.Equal(t, safeKeyName(path), exp)
+	}
+}