NET-10195 - allow consul dataplane to be run as a DNS proxy without E…

…nvoy or XDS (#571) * set up flags * disable envoy and xds when flags dictate. change validation for dns bind address to be relaxed when running outside of sidecar mode. add validation that xds and envoy must both be either enabled or disabled. * fix conditional around xDS * change flag approach to use top level mode and get rid of enabled flags * rearrange some code to make it easier to read. * consolidate * get back to baseline * get back to baseline * fixing issues with conditional logic and consul-k8s tests not working. * fixing lint problem * Apply suggestions from code review Co-authored-by: Dhia Ayachi <[email protected]> * added changelog * fix ci errors * fix logic to prevent bootstrap config from getting called when in dns-proxy mode * adding unit tests for validation when in dns proxy mode --------- Co-authored-by: Dhia Ayachi <[email protected]>
hashicorp · Jul 11, 2024 · 6961d4e · 6961d4e
1 parent c30b874
commit 6961d4e
Show file tree

Hide file tree

Showing 8 changed files with 242 additions and 40 deletions.
diff --git a/.changelog/571.txt b/.changelog/571.txt
@@ -0,0 +1,9 @@
+```release-note:feature
+Added the ability to set the `-mode` flag. Options available are `sidecar` and `dns-proxy`. The system defaults to `sidecar`.
+When set to `sidecar`:
+- DNS Server, xDS Server, and Envoy are enabled.
+- The system validates that `-consul-dns-bind-addr` and equivalent environment variable must be set to the loopback address.
+When set to `dns-proxy`:
+- Only DNS Server is enabled. xDS Server and Envoy are disabled.
+- `consul-dns-bind-addr` and equivalent environment variable can be set to other values besides the loopback address.
+```
diff --git a/Makefile b/Makefile
@@ -69,6 +69,7 @@ docker-run: docker ## run the image of $(TAG)
 
 .PHONY: dev-docker
 dev-docker: docker ## build docker image and tag the image to local
+	echo '$(ARCH)'
 	docker tag '$(PRODUCT_NAME):$(VERSION)'  '$(PRODUCT_NAME):local'
 
 ##@ Testing

diff --git a/cmd/consul-dataplane/config.go b/cmd/consul-dataplane/config.go
@@ -21,6 +21,7 @@ type FlagOpts struct {
 }
 
 type DataplaneConfigFlags struct {
+	Mode      *string        `json:"mode,omitempty"`
 	Consul    ConsulFlags    `json:"consul,omitempty"`
 	Service   ServiceFlags   `json:"service,omitempty"`
 	Proxy     ProxyFlags     `json:"proxy,omitempty"`
@@ -209,6 +210,7 @@ func (f *FlagOpts) buildConfigFromFile() (DataplaneConfigFlags, error) {
 func buildDefaultConsulDPFlags() (DataplaneConfigFlags, error) {
 	data := `
 	{
+		"mode": "sidecar",
 		"consul": {
 			"grpcPort": 8502,
 			"serverWatchDisabled": false,
@@ -316,6 +318,7 @@ func constructRuntimeConfig(cfg DataplaneConfigFlags, extraArgs []string) (*cons
 				InsecureSkipVerify: boolVal(cfg.Consul.TLS.InsecureSkipVerify),
 			},
 		},
+		Mode:  consuldp.ModeType(stringVal(cfg.Mode)),
 		Proxy: &proxyCfg,
 		Logging: &consuldp.LoggingConfig{
 			Name:     DefaultLogName,

diff --git a/cmd/consul-dataplane/config_test.go b/cmd/consul-dataplane/config_test.go
@@ -33,6 +33,7 @@ func TestConfigGeneration(t *testing.T) {
 			},
 			makeExpectedCfg: func(flagOpts *FlagOpts) *consuldp.Config {
 				return &consuldp.Config{
+					Mode: consuldp.ModeTypeSidecar,
 					Consul: &consuldp.ConsulConfig{
 						Addresses:           stringVal(flagOpts.dataplaneConfig.Consul.Addresses),
 						GRPCPort:            intVal(flagOpts.dataplaneConfig.Consul.GRPCPort),
@@ -111,6 +112,7 @@ func TestConfigGeneration(t *testing.T) {
 			},
 			makeExpectedCfg: func(flagOpts *FlagOpts) *consuldp.Config {
 				return &consuldp.Config{
+					Mode: consuldp.ModeTypeSidecar,
 					Consul: &consuldp.ConsulConfig{
 						Addresses:           stringVal(flagOpts.dataplaneConfig.Consul.Addresses),
 						GRPCPort:            intVal(flagOpts.dataplaneConfig.Consul.GRPCPort),
@@ -189,6 +191,7 @@ func TestConfigGeneration(t *testing.T) {
 				if err != nil {
 					return nil, err
 				}
+				opts.dataplaneConfig.Mode = strReference("dns-proxy")
 				opts.dataplaneConfig.Consul.Credentials.Login.BearerTokenPath = strReference("/consul/bearertokenpath/")
 				opts.dataplaneConfig.Consul.Credentials.Login.Datacenter = strReference("dc100")
 				opts.dataplaneConfig.Consul.Credentials.Login.Meta = map[string]string{
@@ -206,6 +209,7 @@ func TestConfigGeneration(t *testing.T) {
 			},
 			makeExpectedCfg: func(flagOpts *FlagOpts) *consuldp.Config {
 				return &consuldp.Config{
+					Mode: consuldp.ModeTypeDNSProxy,
 					Consul: &consuldp.ConsulConfig{
 						Addresses:           stringVal(flagOpts.dataplaneConfig.Consul.Addresses),
 						GRPCPort:            intVal(flagOpts.dataplaneConfig.Consul.GRPCPort),
@@ -310,6 +314,7 @@ func TestConfigGeneration(t *testing.T) {
 			},
 			makeExpectedCfg: func(flagOpts *FlagOpts) *consuldp.Config {
 				return &consuldp.Config{
+					Mode: consuldp.ModeTypeSidecar,
 					Consul: &consuldp.ConsulConfig{
 						Addresses:           stringVal(flagOpts.dataplaneConfig.Consul.Addresses),
 						GRPCPort:            intVal(flagOpts.dataplaneConfig.Consul.GRPCPort),
@@ -408,6 +413,7 @@ func TestConfigGeneration(t *testing.T) {
 			},
 			makeExpectedCfg: func(flagOpts *FlagOpts) *consuldp.Config {
 				return &consuldp.Config{
+					Mode: consuldp.ModeTypeSidecar,
 					Consul: &consuldp.ConsulConfig{
 						Addresses:           stringVal(flagOpts.dataplaneConfig.Consul.Addresses),
 						GRPCPort:            intVal(flagOpts.dataplaneConfig.Consul.GRPCPort),
@@ -521,6 +527,7 @@ func TestConfigGeneration(t *testing.T) {
 			},
 			makeExpectedCfg: func(flagOpts *FlagOpts) *consuldp.Config {
 				return &consuldp.Config{
+					Mode: consuldp.ModeTypeSidecar,
 					Consul: &consuldp.ConsulConfig{
 						Addresses:           "consul_server.dc1",
 						GRPCPort:            8502,
@@ -627,6 +634,7 @@ func TestConfigGeneration(t *testing.T) {
 			},
 			makeExpectedCfg: func(flagOpts *FlagOpts) *consuldp.Config {
 				return &consuldp.Config{
+					Mode: consuldp.ModeTypeSidecar,
 					Consul: &consuldp.ConsulConfig{
 						Addresses:           stringVal(flagOpts.dataplaneConfig.Consul.Addresses),
 						GRPCPort:            intVal(flagOpts.dataplaneConfig.Consul.GRPCPort),
@@ -706,6 +714,7 @@ func TestConfigGeneration(t *testing.T) {
 			desc: "test whether CLI flag values override the file values with proxy flags",
 			flagOpts: func() (*FlagOpts, error) {
 				opts, err := generateFlagOptsWithProxyFlags()
+				opts.dataplaneConfig.Mode = strReference("dns-proxy")
 				if err != nil {
 					return nil, err
 				}
@@ -754,6 +763,7 @@ func TestConfigGeneration(t *testing.T) {
 			},
 			makeExpectedCfg: func(flagOpts *FlagOpts) *consuldp.Config {
 				return &consuldp.Config{
+					Mode: consuldp.ModeTypeDNSProxy,
 					Consul: &consuldp.ConsulConfig{
 						Addresses:           stringVal(flagOpts.dataplaneConfig.Consul.Addresses),
 						GRPCPort:            intVal(flagOpts.dataplaneConfig.Consul.GRPCPort),

diff --git a/cmd/consul-dataplane/main.go b/cmd/consul-dataplane/main.go
@@ -27,6 +27,10 @@ func init() {
 	flagOpts = &FlagOpts{}
 	flags.BoolVar(&flagOpts.printVersion, "version", false, "Prints the current version of consul-dataplane.")
 
+	StringVar(flags, &flagOpts.dataplaneConfig.Mode, "mode", "DP_MODE", "dataplane mode. Value can be:\n"+
+		"1. sidecar - used when running as a sidecar to Consul services with xDS Server, Envoy, and DNS Server running; OR\n"+
+		"2. dns-proxy - used when running as a standalone application where DNS Server runs, but Envoy and xDS Server are enabled.\n")
+
 	StringVar(flags, &flagOpts.dataplaneConfig.Consul.Addresses, "addresses", "DP_CONSUL_ADDRESSES", "Consul server gRPC addresses. Value can be:\n"+
 		"1. A DNS name that resolves to server addresses or the DNS name of a load balancer in front of the Consul servers; OR\n"+
 		"2. An executable command in the format, 'exec=<executable with optional args>'. The executable\n"+
@@ -107,8 +111,8 @@ func init() {
 	StringVar(flags, &flagOpts.dataplaneConfig.Consul.TLS.ServerName, "tls-server-name", "DP_TLS_SERVER_NAME", "The hostname to expect in the server certificate's subject. This is required if -addresses is not a DNS name.")
 	BoolVar(flags, &flagOpts.dataplaneConfig.Consul.TLS.InsecureSkipVerify, "tls-insecure-skip-verify", "DP_TLS_INSECURE_SKIP_VERIFY", "Do not verify the server's certificate. Useful for testing, but not recommended for production.")
 
-	StringVar(flags, &flagOpts.dataplaneConfig.DNSServer.BindAddr, "consul-dns-bind-addr", "DP_CONSUL_DNS_BIND_ADDR", "The address that will be bound to the consul dns proxy.")
-	IntVar(flags, &flagOpts.dataplaneConfig.DNSServer.BindPort, "consul-dns-bind-port", "DP_CONSUL_DNS_BIND_PORT", "The port the consul dns proxy will listen on. By default -1 disables the dns proxy")
+	StringVar(flags, &flagOpts.dataplaneConfig.DNSServer.BindAddr, "consul-dns-bind-addr", "DP_CONSUL_DNS_BIND_ADDR", "The address that will be bound to the consul dns listener.")
+	IntVar(flags, &flagOpts.dataplaneConfig.DNSServer.BindPort, "consul-dns-bind-port", "DP_CONSUL_DNS_BIND_PORT", "The port the consul dns listener will listen on. By default -1 disables the dns listener.")
 
 	// Default is false because it will generally be configured appropriately by Helm
 	// configuration or pod annotation.

diff --git a/pkg/consuldp/config.go b/pkg/consuldp/config.go
@@ -135,6 +135,18 @@ const (
 	CredentialsTypeLogin CredentialsType = "login"
 )
 
+// CredentialsType identifies the type of credentials provided.
+type ModeType string
+
+const (
+	// ModeTypeSidecar indicates that consul-dataplane is running in sidecar
+	// mode where DNS Server, xDS Server, and Envoy are all enabled.
+	ModeTypeSidecar ModeType = "sidecar"
+	// ModeTypeDNSProxy indicates that consul-dataplane is running in DNS Proxy
+	// mode where DNS Server is running but xDSServer and Envoy are disabled.
+	ModeTypeDNSProxy ModeType = "dns-proxy"
+)
+
 // StaticCredentialsConfig contains the static ACL token that will be used to
 // authenticate requests and streams to the Consul servers.
 type StaticCredentialsConfig struct {
@@ -316,6 +328,7 @@ type XDSServer struct {
 // Config is the configuration used by consul-dataplane, consolidated
 // from various sources - CLI flags, env vars, config file settings.
 type Config struct {
+	Mode      ModeType
 	DNSServer *DNSServerConfig
 	Consul    *ConsulConfig
 	Proxy     *ProxyConfig

diff --git a/pkg/consuldp/consul_dataplane.go b/pkg/consuldp/consul_dataplane.go
@@ -79,24 +79,24 @@ func validateConfig(cfg *Config) error {
 		return errors.New("consul addresses not specified")
 	case cfg.Consul.GRPCPort == 0:
 		return errors.New("consul server gRPC port not specified")
-	case cfg.Proxy == nil:
+	case cfg.Mode == ModeTypeSidecar && cfg.Proxy == nil:
 		return errors.New("proxy details not specified")
-	case cfg.Proxy.ProxyID == "":
+	case cfg.Mode == ModeTypeSidecar && cfg.Proxy.ProxyID == "":
 		return errors.New("proxy ID not specified")
-	case cfg.Envoy == nil:
+	case cfg.Mode == ModeTypeSidecar && cfg.Envoy == nil:
 		return errors.New("envoy settings not specified")
-	case cfg.Envoy.AdminBindAddress == "":
+	case cfg.Mode == ModeTypeSidecar && cfg.Envoy.AdminBindAddress == "":
 		return errors.New("envoy admin bind address not specified")
-	case cfg.Envoy.AdminBindPort == 0:
+	case cfg.Mode == ModeTypeSidecar && cfg.Envoy.AdminBindPort == 0:
 		return errors.New("envoy admin bind port not specified")
 	case cfg.Logging == nil:
 		return errors.New("logging settings not specified")
-	case cfg.XDSServer.BindAddress == "":
+	case cfg.Mode == ModeTypeSidecar && cfg.XDSServer.BindAddress == "":
 		return errors.New("envoy xDS bind address not specified")
-	case !strings.HasPrefix(cfg.XDSServer.BindAddress, "unix://") && !net.ParseIP(cfg.XDSServer.BindAddress).IsLoopback():
+	case cfg.Mode == ModeTypeSidecar && !strings.HasPrefix(cfg.XDSServer.BindAddress, "unix://") && !net.ParseIP(cfg.XDSServer.BindAddress).IsLoopback():
 		return errors.New("non-local xDS bind address not allowed")
-	case cfg.DNSServer.Port != -1 && !net.ParseIP(cfg.DNSServer.BindAddr).IsLoopback():
-		return errors.New("non-local DNS proxy bind address not allowed")
+	case cfg.Mode == ModeTypeSidecar && cfg.DNSServer.Port != -1 && !net.ParseIP(cfg.DNSServer.BindAddr).IsLoopback():
+		return errors.New("non-local DNS proxy bind address not allowed when running as a sidecar")
 	}
 
 	creds := cfg.Consul.Credentials
@@ -129,6 +129,7 @@ func validateConfig(cfg *Config) error {
 func (cdp *ConsulDataplane) Run(ctx context.Context) error {
 	ctx = hclog.WithContext(ctx, cdp.logger)
 	cdp.logger.Info("started consul-dataplane process")
+	cdp.logger.Info(fmt.Sprintf("consul-dataplane mode: %s", cdp.cfg.Mode))
 
 	// At startup we need to cache metrics until we have information from the bootstrap envoy config
 	// that the consumer wants metrics enabled. Until then we will set our own light weight metrics
@@ -178,6 +179,24 @@ func (cdp *ConsulDataplane) Run(ctx context.Context) error {
 	cdp.aclToken = state.Token
 	cdp.dpServiceClient = pbdataplane.NewDataplaneServiceClient(state.GRPCConn)
 
+	doneCh := make(chan error)
+	// start up DNS server
+	if err = cdp.startDNSProxy(ctx); err != nil {
+		cdp.logger.Error("failed to start the dns proxy", "error", err)
+		return err
+	}
+
+	// if running as DNS PRoxy, xDS Server and Envoy are disabled, so
+	// return before configuring them.
+	if cdp.cfg.Mode == ModeTypeDNSProxy {
+		go func() {
+			<-ctx.Done()
+			doneCh <- nil
+		}()
+		return <-doneCh
+	}
+
+	cdp.logger.Info("configuring xDS and Envoy")
 	err = cdp.setupXDSServer()
 	if err != nil {
 		return err
@@ -191,11 +210,7 @@ func (cdp *ConsulDataplane) Run(ctx context.Context) error {
 	}
 	cdp.logger.Debug("generated envoy bootstrap config", "config", string(cfg))
 
-	if err = cdp.startDNSProxy(ctx); err != nil {
-		cdp.logger.Error("failed to start the dns proxy", "error", err)
-		return err
-	}
-
+	cdp.logger.Info("configuring envoy and xDS")
 	proxy, err := envoy.NewProxy(cdp.envoyProxyConfig(cfg))
 	if err != nil {
 		cdp.logger.Error("failed to create new proxy", "error", err)
@@ -218,7 +233,6 @@ func (cdp *ConsulDataplane) Run(ctx context.Context) error {
 		return err
 	}
 
-	doneCh := make(chan error)
 	go func() {
 		select {
 		case <-ctx.Done():
@@ -266,7 +280,7 @@ func (cdp *ConsulDataplane) startDNSProxy(ctx context.Context) error {
 		Token:     cdp.aclToken,
 	})
 	if err == dns.ErrServerDisabled {
-		cdp.logger.Info("dns proxy disabled: configure the Consul DNS port to enable")
+		cdp.logger.Info("dns server disabled: configure the Consul DNS port to enable")
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("failed to create dns server: %w", err)