fix: test improvments

internal/config/loader.go

@@ -10,7 +10,6 @@ import (
 	"path"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/dgate-io/dgate/pkg/util"
 	"github.com/hashicorp/raft"
@@ -24,13 +23,13 @@ import (
 	"github.com/mitchellh/mapstructure"
 )
 
-func LoadConfig(dgateConfigPath string) (*DGateConfig, error) {
-	ctx, cancel := context.WithTimeout(
-		context.Background(),
-		time.Second*10,
-	)
-	defer cancel()
+var (
+	EnvVarRegex  = regexp.MustCompile(`\${(?P<var_name>[a-zA-Z0-9_]{1,})(:-(?P<default>.*?)?)?}`)
+	CommandRegex = regexp.MustCompile(`\$\((?P<cmd>.*?)\)`)
+)
 
+func LoadConfig(dgateConfigPath string) (*DGateConfig, error) {
+	ctx := context.Background()
 	var dgateConfigData string
 	if dgateConfigPath == "" {
 		dgateConfigPath = os.Getenv("DG_CONFIG_PATH")
@@ -91,33 +90,34 @@ func LoadConfig(dgateConfigPath string) (*DGateConfig, error) {
 	}
 
 	panicVars := []string{}
-	data := k.All()
 	if !util.EnvVarCheckBool("DG_DISABLE_SHELL_PARSER") {
-		commandRegex := regexp.MustCompile(`\$\((?P<cmd>.*?)\)`)
+		data := k.All()
 		shell := "/bin/sh"
-		if shellEnv, exists := os.LookupEnv("SHELL"); exists {
+		if shellEnv := os.Getenv("SHELL"); shellEnv != "" {
 			shell = shellEnv
-		}
-		resolveConfigStringPattern(data, commandRegex, func(value string, results map[string]string) (string, error) {
-			cmdResult, err := exec.CommandContext(ctx, shell, "-c", results["cmd"]).Output()
+		} 
+		resolveConfigStringPattern(data, CommandRegex, func(value string, results map[string]string) (string, error) {
+			cmdResult, err := exec.CommandContext(
+				ctx, shell, "-c", results["cmd"]).Output()
 			if err != nil {
+				panicVars = append(panicVars, results["cmd"])
 				return "", err
 			}
 			return strings.TrimSpace(string(cmdResult)), nil
 		}, func(results map[string]string, err error) {
 			panic("error on command - `" + results["cmd"] + "`: " + err.Error())
 		})
+		k.Load(confmap.Provider(data, "."), nil)
 	}
 
 	if !util.EnvVarCheckBool("DG_DISABLE_ENV_PARSER") {
-		envVarRe := regexp.MustCompile(`\${(?P<var_name>[a-zA-Z0-9_]{1,})(:-(?P<default>.*?)?)?}`)
-		resolveConfigStringPattern(data, envVarRe, func(value string, results map[string]string) (string, error) {
+		data := k.All()
+		resolveConfigStringPattern(data, EnvVarRegex, func(value string, results map[string]string) (string, error) {
 			if envVar := os.Getenv(results["var_name"]); envVar != "" {
 				return envVar, nil
 			} else if strings.Contains(value, results["var_name"]+":-") {
 				return results["default"], nil
 			}
-			panicVars = append(panicVars, results["var_name"])
 			return "", nil
 		}, func(results map[string]string, err error) {
 			panicVars = append(panicVars, results["var_name"])
@@ -126,10 +126,9 @@ func LoadConfig(dgateConfigPath string) (*DGateConfig, error) {
 		if len(panicVars) > 0 {
 			panic("required env vars not set: " + strings.Join(panicVars, ", "))
 		}
+		k.Load(confmap.Provider(data, "."), nil)
 	}
 
-	k.Load(confmap.Provider(data, "."), nil)
-
 	// validate configuration
 	var err error
 	kDefault(k, "log_level", "info")
@@ -168,11 +167,11 @@ func LoadConfig(dgateConfigPath string) (*DGateConfig, error) {
 		return nil, err
 	}
 
-	kDefault(k, "proxy.transport.max_idle_conns", 100)
-	kDefault(k, "proxy.transport.force_attempt_http2", true)
-	kDefault(k, "proxy.transport.idle_conn_timeout", "90s")
-	kDefault(k, "proxy.transport.tls_handshake_timeout", "10s")
-	kDefault(k, "proxy.transport.expect_continue_timeout", "1s")
+	// kDefault(k, "proxy.transport.max_idle_conns", 100)
+	// kDefault(k, "proxy.transport.force_attempt_http2", true)
+	// kDefault(k, "proxy.transport.idle_conn_timeout", "90s")
+	// kDefault(k, "proxy.transport.tls_handshake_timeout", "10s")
+	// kDefault(k, "proxy.transport.expect_continue_timeout", "1s")
 	if k.Exists("test_server") {
 		kDefault(k, "test_server.enable_h2c", true)
 		kDefault(k, "test_server.enable_http2", true)

internal/config/loader_test.go

@@ -0,0 +1,83 @@
+package config_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/dgate-io/dgate/internal/config"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfig_EnvVarRegex(t *testing.T) {
+	re := config.EnvVarRegex
+	inputs := []string{
+		"${var_name1}",
+		"${var_name2:-default}",
+		"${var_name3:-}",
+		"${var_name4:default}",
+		"${var_name5:}",
+	}
+	results := make(map[string]string)
+	for _, input := range inputs {
+		matches := re.FindAllStringSubmatch(input, -1)
+		for _, match := range matches {
+			results[input] = match[1] + "//" + match[3]
+		}
+	}
+	assert.Equal(t, results, map[string]string{
+		"${var_name1}":          "var_name1//",
+		"${var_name2:-default}": "var_name2//default",
+		"${var_name3:-}":        "var_name3//",
+	})
+}
+
+func TestConfig_CommandRegex(t *testing.T) {
+	re := config.CommandRegex
+	inputs := []string{
+		"$(cmd1)",
+		"$(cmd2 arg1 arg2)",
+		"$(cmd3 \"arg1\" 'arg2')",
+	}
+	results := make(map[string]string)
+	for _, input := range inputs {
+		matches := re.FindAllStringSubmatch(input, -1)
+		for _, match := range matches {
+			results[input] = match[1]
+		}
+	}
+	assert.Equal(t, results, map[string]string{
+		"$(cmd1)":                 "cmd1",
+		"$(cmd2 arg1 arg2)":       "cmd2 arg1 arg2",
+		"$(cmd3 \"arg1\" 'arg2')": "cmd3 \"arg1\" 'arg2'",
+	})
+}
+
+func TestConfig_LoaderVariables(t *testing.T) {
+	os.Setenv("ENV1", "test1")
+	os.Setenv("ENV2", "test2")
+	os.Setenv("ENV3", "test3")
+	os.Setenv("ENV4", "")
+	os.Setenv("ENV5", "test5")
+	os.Setenv("ADMIN_PORT", "8080")
+	conf, err := config.LoadConfig("testdata/env.config.yaml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, []string{
+		"test1",
+		"$ENV2",
+		"test3",
+		"test4",
+		"testing",
+		"test test5",
+	}, conf.Tags)
+	assert.Equal(t, "v1", conf.Version)
+	assert.Equal(t, true, conf.TestServerConfig.EnableH2C)
+	assert.Equal(t, true, conf.TestServerConfig.EnableHTTP2)
+	assert.Equal(t, false, conf.TestServerConfig.EnableEnvVars)
+	assert.Equal(t, 80, conf.ProxyConfig.Port)
+	assert.Equal(t, 8080, conf.AdminConfig.Port)
+	assert.Equal(t, 1, len(conf.Storage.Config))
+	assert.Equal(t, "test1-test2-testing",
+		conf.Storage.Config["testing"])
+}

internal/config/testdata/env.config.yaml

@@ -0,0 +1,21 @@
+version: v1
+debug: true
+tags:
+  - ${ENV1}
+  - $ENV2
+  - ${ENV3:-abc123}
+  - ${ENV4:-test4}
+  - $(echo "testing")
+  - $(echo "test $ENV5")
+test_server:
+  port: 8080
+storage:
+  type: debug
+  testing: ${ENV1}-${ENV2}-$(echo "testing")
+proxy:
+  port: ${PROXY_PORT:-80}
+  host: 0.0.0.0
+admin:
+  port: ${ADMIN_PORT:-9080}
+  host: 0.0.0.0
+

internal/config/utils.go

@@ -12,15 +12,21 @@ import (
 )
 
 type FoundFunc func(string, map[string]string) (string, error)
-
 type ErrorFunc func(map[string]string, error)
 
-func resolveConfigStringPattern(data map[string]any, re *regexp.Regexp, foundFunc FoundFunc, errorFunc ErrorFunc) {
+func resolveConfigStringPattern(
+	data map[string]any,
+	re *regexp.Regexp,
+	foundFunc FoundFunc,
+	errorFunc ErrorFunc,
+) {
 	for k, v := range data {
 		var values []string
 		switch vt := v.(type) {
 		case string:
 			values = []string{vt}
+		case []string:
+			values = vt
 		case []any:
 			values = sliceMap(vt, func(val any) string {
 				if s, ok := val.(string); ok {
@@ -31,6 +37,12 @@ func resolveConfigStringPattern(data map[string]any, re *regexp.Regexp, foundFun
 			if len(values) == 0 {
 				continue
 			}
+		case any:
+			if vv, ok := vt.(string); ok {
+				values = []string{vv}
+			} else {
+				continue
+			}
 		default:
 			continue
 		}

internal/proxy/dynamic_proxy.go

@@ -110,15 +110,15 @@ func (ps *ProxyState) setupRoutes() (err error) {
 			reqCtxProvider := NewRequestContextProvider(r, ps)
 			reqCtxProviders.Insert(r.Namespace.Name+"/"+r.Name, reqCtxProvider)
 			if len(r.Modules) > 0 {
-				modBuf, err := NewModuleBuffer(
+				modPool, err := NewModulePool(
 					256, 1024, reqCtxProvider,
 					ps.createModuleExtractorFunc(r),
 				)
 				if err != nil {
 					ps.logger.Err(err).Msg("Error creating module buffer")
 					return err
 				}
-				reqCtxProvider.SetModuleBuffer(modBuf)
+				reqCtxProvider.SetModulePool(modPool)
 			}
 			err = func() (err error) {
 				defer func() {
@@ -229,7 +229,7 @@ func (ps *ProxyState) startChangeLoop() {
 		func() {
 			ps.proxyLock.Lock()
 			defer ps.proxyLock.Unlock()
-			
+
 			err := ps.reconfigureState(false, log)
 			if log.PushError(err); err != nil {
 				ps.logger.Err(err).
@@ -352,7 +352,7 @@ func (ps *ProxyState) Stop() {
 
 func (ps *ProxyState) HandleRoute(requestCtxProvider *RequestContextProvider, pattern string) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		// ctx, cancel := context.WithCancel(requestCtxProvider.ctx)
+		// ctx, cancel := context.WithCancel(requestCtxPrdovider.ctx)
 		// defer cancel()
 		ps.ProxyHandlerFunc(ps, requestCtxProvider.
 			CreateRequestContext(requestCtxProvider.ctx, w, r, pattern))

internal/proxy/module_executor.go

@@ -5,14 +5,13 @@ import (
 	"errors"
 )
 
-type ModuleBuffer interface {
-	// Load(cb func())
+type ModulePool interface {
 	Borrow() ModuleExtractor
 	Return(me ModuleExtractor)
 	Close()
 }
 
-type moduleBuffer struct {
+type modulePool struct {
 	modExtBuffer chan ModuleExtractor
 	min, max     int
 
@@ -22,11 +21,11 @@ type moduleBuffer struct {
 	createModuleExtract func() ModuleExtractor
 }
 
-func NewModuleBuffer(
+func NewModulePool(
 	minBuffers, maxBuffers int,
 	reqCtxProvider *RequestContextProvider,
 	createModExts func(*RequestContextProvider) ModuleExtractor,
-) (ModuleBuffer, error) {
+) (ModulePool, error) {
 	if minBuffers < 1 {
 		panic("module concurrency must be greater than 0")
 	}
@@ -38,7 +37,7 @@ func NewModuleBuffer(
 	if me == nil {
 		return nil, errors.New("could not load moduleExtract")
 	}
-	mb := &moduleBuffer{
+	mb := &modulePool{
 		min:          minBuffers,
 		max:          maxBuffers,
 		modExtBuffer: make(chan ModuleExtractor, maxBuffers),
@@ -50,7 +49,7 @@ func NewModuleBuffer(
 	return mb, nil
 }
 
-func (mb *moduleBuffer) Borrow() ModuleExtractor {
+func (mb *modulePool) Borrow() ModuleExtractor {
 	if mb == nil || mb.ctx == nil || mb.ctx.Err() != nil {
 		return nil
 	}
@@ -65,7 +64,7 @@ func (mb *moduleBuffer) Borrow() ModuleExtractor {
 	return me
 }
 
-func (mb *moduleBuffer) Return(me ModuleExtractor) {
+func (mb *modulePool) Return(me ModuleExtractor) {
 	// if context is canceled, do not return module extract
 	if mb.ctx != nil && mb.ctx.Err() == nil {
 		select {
@@ -78,7 +77,7 @@ func (mb *moduleBuffer) Return(me ModuleExtractor) {
 	me.Stop(true)
 }
 
-func (mb *moduleBuffer) Close() {
+func (mb *modulePool) Close() {
 	if mb.ctxCancel != nil {
 		mb.ctxCancel()
 	}

internal/proxy/module_mock_test.go

@@ -11,34 +11,34 @@ import (
 	"github.com/stretchr/testify/mock"
 )
 
-type mockModuleBuffer struct {
+type mockModulePool struct {
 	mock.Mock
 }
 
-var _ proxy.ModuleBuffer = &mockModuleBuffer{}
+var _ proxy.ModulePool = &mockModulePool{}
 
-func NewMockModuleBuffer() *mockModuleBuffer {
-	return &mockModuleBuffer{}
+func NewMockModulePool() *mockModulePool {
+	return &mockModulePool{}
 }
 
-// Borrow implements proxy.ModuleBuffer.
-func (mb *mockModuleBuffer) Borrow() proxy.ModuleExtractor {
+// Borrow implements proxy.ModulePool.
+func (mb *mockModulePool) Borrow() proxy.ModuleExtractor {
 	args := mb.Called()
 	return args.Get(0).(proxy.ModuleExtractor)
 }
 
-// Close implements proxy.ModuleBuffer.
-func (mb *mockModuleBuffer) Close() {
+// Close implements proxy.ModulePool.
+func (mb *mockModulePool) Close() {
 	mb.Called()
 }
 
-// Load implements proxy.ModuleBuffer.
-func (mb *mockModuleBuffer) Load(cb func()) {
+// Load implements proxy.ModulePool.
+func (mb *mockModulePool) Load(cb func()) {
 	mb.Called(cb)
 }
 
-// Return implements proxy.ModuleBuffer.
-func (mb *mockModuleBuffer) Return(me proxy.ModuleExtractor) {
+// Return implements proxy.ModulePool.
+func (mb *mockModulePool) Return(me proxy.ModuleExtractor) {
 	mb.Called(me)
 }