Merge pull request #53 from armon/master

Adding support for Consul backend

README.md

@@ -4,7 +4,8 @@
 
 `confd` is a lightweight configuration management tool focused on:
 
-* keeping local configuration files up-to-date by polling [etcd](https://github.com/coreos/etcd) and processing [template resources](docs/template-resources.md).
+* keeping local configuration files up-to-date by polling [etcd](https://github.com/coreos/etcd) or
+  [Consul](http://consul.io) and processing [template resources](docs/template-resources.md).
 * reloading applications to pick up new config file changes
 
 ## Community
@@ -15,7 +16,7 @@
 
 ## Quick Start
 
-Before we begin be sure to [download and install confd](docs/installation.md). 
+Before we begin be sure to [download and install confd](docs/installation.md).
 
 ### Add keys to etcd
 
@@ -26,6 +27,8 @@ etcdctl set /myapp/database/url db.example.com
 etcdctl set /myapp/database/user rob
 ```
 
+Alternatively, Consul can be used a Key/Value store.
+
 ### Create the confdir
 
 The confdir is where template resource configs and source templates are stored. The default confdir is `/etc/confd`. Create the confdir by executing the following command:
@@ -74,7 +77,7 @@ confd supports two modes of operation, daemon and onetime mode. In daemon mode,
 Assuming your etcd server is running at http://127.0.0.1:4001 you can run the following command to process the `~/confd/conf.d/myconfig.toml` template resource:
 
 ```
-confd -verbose -onetime -node 'http://127.0.0.1:4001' -confdir ~/confd 
+confd -verbose -onetime -node 'http://127.0.0.1:4001' -confdir ~/confd
 ```
 Output:
 ```
@@ -91,7 +94,7 @@ cat /tmp/myconfig.conf
 ```
 
 Output:
-``` 
+```
 # This a comment
 [myconfig]
 database_url = db.example.com

confd.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/kelseyhightower/confd/config"
+	"github.com/kelseyhightower/confd/consul"
 	"github.com/kelseyhightower/confd/etcd/etcdutil"
 	"github.com/kelseyhightower/confd/log"
 	"github.com/kelseyhightower/confd/resource/template"
@@ -47,15 +48,15 @@ func main() {
 	log.SetVerbose(config.Verbose())
 	log.SetDebug(config.Debug())
 	log.Notice("Starting confd")
-	// Create the etcd client upfront and use it for the life of the process.
-	// The etcdClient is an http.Client and designed to be reused.
-	log.Notice("etcd nodes set to " + strings.Join(config.EtcdNodes(), ", "))
-	etcdClient, err := etcdutil.NewEtcdClient(config.EtcdNodes(), config.ClientCert(), config.ClientKey(), config.ClientCaKeys())
+
+	// Create the storage client
+	store, err := createStoreClient()
 	if err != nil {
 		log.Fatal(err.Error())
 	}
+
 	for {
-		runErrors := template.ProcessTemplateResources(etcdClient)
+		runErrors := template.ProcessTemplateResources(store)
 		// If the -onetime flag is passed on the command line we immediately exit
 		// after processing the template config files.
 		if onetime {
@@ -68,6 +69,20 @@ func main() {
 	}
 }
 
+// createStoreClient is used to create a storage client based
+// on our configuration. Either an etcd or Consul client.
+func createStoreClient() (template.StoreClient, error) {
+	if config.Consul() {
+		log.Notice("Consul address set to " + config.ConsulAddr())
+		return consul.NewConsulClient(config.ConsulAddr())
+	} else {
+		// Create the etcd client upfront and use it for the life of the process.
+		// The etcdClient is an http.Client and designed to be reused.
+		log.Notice("etcd nodes set to " + strings.Join(config.EtcdNodes(), ", "))
+		return etcdutil.NewEtcdClient(config.EtcdNodes(), config.ClientCert(), config.ClientKey(), config.ClientCaKeys())
+	}
+}
+
 // IsFileExist reports whether path exits.
 func IsFileExist(fpath string) bool {
 	if _, err := os.Stat(fpath); os.IsNotExist(err) {

config/config.go

@@ -21,6 +21,8 @@ var (
 	clientCaKeys string
 	config       Config // holds the global confd config.
 	confdir      string
+	consul       bool
+	consulAddr   string
 	debug        bool
 	etcdNodes    Nodes
 	etcdScheme   string
@@ -44,6 +46,8 @@ type confd struct {
 	ClientKey    string   `toml:"client_key"`
 	ClientCaKeys string   `toml:"client_cakeys"`
 	ConfDir      string   `toml:"confdir"`
+	Consul       bool     `toml:"consul"`
+	ConsulAddr   string   `toml:"consul_addr"`
 	EtcdNodes    []string `toml:"etcd_nodes"`
 	EtcdScheme   string   `toml:"etcd_scheme"`
 	Interval     int      `toml:"interval"`
@@ -60,6 +64,8 @@ func init() {
 	flag.StringVar(&clientKey, "client-key", "", "the client key")
 	flag.StringVar(&clientCaKeys, "client-ca-keys", "", "client ca keys")
 	flag.StringVar(&confdir, "confdir", "/etc/confd", "confd conf directory")
+	flag.BoolVar(&consul, "consul", false, "specified to enable use of Consul")
+	flag.StringVar(&consulAddr, "consul-addr", "", "address of Consul HTTP interface")
 	flag.Var(&etcdNodes, "node", "list of etcd nodes")
 	flag.StringVar(&etcdScheme, "etcd-scheme", "http", "the etcd URI scheme. (http or https)")
 	flag.IntVar(&interval, "interval", 600, "etcd polling interval")
@@ -113,7 +119,7 @@ func ClientKey() string {
 
 // ClientCaKeys returns the client CA certificates
 func ClientCaKeys() string {
-        return config.Confd.ClientCaKeys
+	return config.Confd.ClientCaKeys
 }
 
 // ConfDir returns the path to the confd config dir.
@@ -126,6 +132,16 @@ func ConfigDir() string {
 	return filepath.Join(config.Confd.ConfDir, "conf.d")
 }
 
+// Consul returns if we should use Consul
+func Consul() bool {
+	return config.Confd.Consul
+}
+
+// ConsulAddr returns the address of the consul node
+func ConsulAddr() string {
+	return config.Confd.ConsulAddr
+}
+
 // EtcdNodes returns a list of etcd node url strings.
 // For example: ["http://203.0.113.30:4001"]
 func EtcdNodes() []string {
@@ -186,6 +202,7 @@ func setDefaults() {
 	config = Config{
 		Confd: confd{
 			ConfDir:    "/etc/confd",
+			ConsulAddr: "127.0.0.1:8500",
 			Interval:   600,
 			Prefix:     "/",
 			EtcdNodes:  []string{"127.0.0.1:4001"},
@@ -262,6 +279,10 @@ func setConfigFromFlag(f *flag.Flag) {
 		config.Confd.ClientCaKeys = clientCaKeys
 	case "confdir":
 		config.Confd.ConfDir = confdir
+	case "consul":
+		config.Confd.Consul = consul
+	case "consul-addr":
+		config.Confd.ConsulAddr = consulAddr
 	case "node":
 		config.Confd.EtcdNodes = etcdNodes
 	case "etcd-scheme":

consul/client.go

@@ -0,0 +1,39 @@
+package consul
+
+import (
+	"github.com/armon/consul-kv"
+)
+
+// Client provides a wrapper around the consulkv client
+type Client struct {
+	client *consulkv.Client
+}
+
+// NewConsulClient returns a new client to Consul for the given address
+func NewConsulClient(addr string) (*Client, error) {
+	conf := consulkv.DefaultConfig()
+	conf.Address = addr
+	client, err := consulkv.NewClient(conf)
+	if err != nil {
+		return nil, err
+	}
+	c := &Client{
+		client: client,
+	}
+	return c, nil
+}
+
+// GetValues queries Consul for keys
+func (c *Client) GetValues(keys []string) (map[string]interface{}, error) {
+	vars := make(map[string]interface{})
+	for _, key := range keys {
+		_, pairs, err := c.client.List(key)
+		if err != nil {
+			return vars, err
+		}
+		for _, p := range pairs {
+			vars[p.Key] = string(p.Value)
+		}
+	}
+	return vars, nil
+}

docs/command-line-flags.md

@@ -13,6 +13,8 @@ Usage of confd:
   -client-key="": the client key
   -confdir="/etc/confd": confd conf directory
   -config-file="": the confd config file
+  -consul=false: specified to enable use of Consul
+  -consul-addr="": address of Consul HTTP interface
   -debug=false: enable debug logging
   -etcd-scheme="http": the etcd URI scheme. (http or https)
   -interval=600: etcd polling interval

docs/configuration-guide.md

@@ -15,6 +15,8 @@ Optional:
 * `client_cert` (string) The cert file of the client.
 * `client_key` (string) The key file of the client.
 * `confdir` (string) - The path to confd configs. The default is /etc/confd.
+* `consul` (bool) - Should Consul be used instead of etcd for Key/Value storage.
+* `consul_addr` (string) - Address of consul. The default is "127.0.0.1:8500".
 * `etcd_nodes` (array of strings) - An array of etcd cluster nodes. The default
   is ["http://127.0.0.1:4001"].
 * `etcd_scheme` (string) - The etcd scheme to use. Must be 'http' or 'https'

etcd/etcdutil/client.go

@@ -9,29 +9,32 @@ import (
 	"strings"
 )
 
-var replacer = strings.NewReplacer("/", "_")
+// Client is a wrapper around the etcd client
+type Client struct {
+	client EtcdClient
+}
+
+type EtcdClient interface {
+	Get(key string, sort, recurse bool) (*etcd.Response, error)
+}
 
 // NewEtcdClient returns an *etcd.Client with a connection to named machines.
 // It returns an error if a connection to the cluster cannot be made.
-func NewEtcdClient(machines []string, cert, key string, caCert string) (*etcd.Client, error) {
+func NewEtcdClient(machines []string, cert, key string, caCert string) (*Client, error) {
 	var c *etcd.Client
 	if cert != "" && key != "" {
 		c, err := etcd.NewTLSClient(machines, cert, key, caCert)
 		if err != nil {
-			return c, err
+			return &Client{c}, err
 		}
 	} else {
 		c = etcd.NewClient(machines)
 	}
 	success := c.SetCluster(machines)
 	if !success {
-		return c, errors.New("cannot connect to etcd cluster: " + strings.Join(machines, ","))
+		return &Client{c}, errors.New("cannot connect to etcd cluster: " + strings.Join(machines, ","))
 	}
-	return c, nil
-}
-
-type EtcdClient interface {
-	Get(key string, sort bool, recursive bool) (*etcd.Response, error)
+	return &Client{c}, nil
 }
 
 // GetValues queries etcd for keys prefixed by prefix.
@@ -40,14 +43,14 @@ type EtcdClient interface {
 // keys were '/nginx/port'; the prefixed '/production/nginx/port' key would
 // be queried for. If the value for the prefixed key where 80, the returned map
 // would contain the entry vars["nginx_port"] = "80".
-func GetValues(c EtcdClient, prefix string, keys []string) (map[string]interface{}, error) {
+func (c *Client) GetValues(keys []string) (map[string]interface{}, error) {
 	vars := make(map[string]interface{})
 	for _, key := range keys {
-		resp, err := c.Get(key, false, true)
+		resp, err := c.client.Get(key, false, true)
 		if err != nil {
 			return vars, err
 		}
-		err = nodeWalk(resp.Node, prefix, vars)
+		err = nodeWalk(resp.Node, vars)
 		if err != nil {
 			return vars, err
 		}
@@ -56,25 +59,17 @@ func GetValues(c EtcdClient, prefix string, keys []string) (map[string]interface
 }
 
 // nodeWalk recursively descends nodes, updating vars.
-func nodeWalk(node *etcd.Node, prefix string, vars map[string]interface{}) error {
+func nodeWalk(node *etcd.Node, vars map[string]interface{}) error {
 	if node != nil {
-		key := pathToKey(node.Key, prefix)
+		key := node.Key
 		if !node.Dir {
 			vars[key] = node.Value
 		} else {
 			vars[key] = node.Nodes
 			for _, node := range node.Nodes {
-				nodeWalk(&node, prefix, vars)
+				nodeWalk(&node, vars)
 			}
 		}
 	}
 	return nil
 }
-
-// pathToKey translates etcd key paths into something more suitable for use
-// in Golang templates. Turn /prefix/key/subkey into key_subkey.
-func pathToKey(key, prefix string) string {
-	key = strings.TrimPrefix(key, prefix)
-	key = strings.TrimPrefix(key, "/")
-	return replacer.Replace(key)
-}