HTTP client round trippers logging, metrics and request id

httpclient/config_test.go

@@ -0,0 +1,180 @@
+/*
+Copyright © 2024 Acronis International GmbH.
+
+Released under MIT license.
+*/
+
+package httpclient
+
+import (
+	"bytes"
+	"github.com/acronis/go-appkit/config"
+	"github.com/acronis/go-appkit/retry"
+	"github.com/stretchr/testify/require"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestConfigWithLoader(t *testing.T) {
+	yamlData := testYamlData(nil)
+	actualConfig := &Config{}
+	err := config.NewDefaultLoader("").LoadFromReader(bytes.NewReader(yamlData), config.DataTypeYAML, actualConfig)
+	require.NoError(t, err, "load configuration")
+
+	expectedConfig := &Config{
+		Retries: RetriesConfig{
+			Enabled:     true,
+			MaxAttempts: 30,
+			Policy: PolicyConfig{
+				Strategy:                          RetryPolicyExponential,
+				ExponentialBackoffInitialInterval: 3 * time.Second,
+				ExponentialBackoffMultiplier:      2,
+			},
+		},
+		RateLimits: RateLimitConfig{
+			Enabled:     true,
+			Limit:       300,
+			Burst:       3000,
+			WaitTimeout: 3 * time.Second,
+		},
+		Logger: LoggerConfig{
+			Enabled:              true,
+			SlowRequestThreshold: 5 * time.Second,
+			Mode:                 "all",
+		},
+		Metrics: MetricsConfig{Enabled: true},
+		Timeout: 30 * time.Second,
+	}
+
+	require.Equal(t, expectedConfig, actualConfig, "configuration does not match expected")
+}
+
+func TestConfigRateLimit(t *testing.T) {
+	yamlData := testYamlData([][]string{{"limit: 300", "limit: -300"}})
+	actualConfig := &Config{}
+	err := config.NewDefaultLoader("").LoadFromReader(bytes.NewReader(yamlData), config.DataTypeYAML, actualConfig)
+	require.Error(t, err)
+	require.Equal(t, "client rate limit must be positive", err.Error())
+
+	yamlData = testYamlData([][]string{{"burst: 3000", "burst: -3"}})
+	actualConfig = &Config{}
+	err = config.NewDefaultLoader("").LoadFromReader(bytes.NewReader(yamlData), config.DataTypeYAML, actualConfig)
+	require.Error(t, err)
+	require.Equal(t, "client burst must be positive", err.Error())
+
+	yamlData = testYamlData([][]string{{"waitTimeout: 3s", "waitTimeout: -3s"}})
+	actualConfig = &Config{}
+	err = config.NewDefaultLoader("").LoadFromReader(bytes.NewReader(yamlData), config.DataTypeYAML, actualConfig)
+	require.Error(t, err)
+	require.Equal(t, "client wait timeout must be positive", err.Error())
+}
+
+func TestConfigRetries(t *testing.T) {
+	yamlData := testYamlData([][]string{{"maxAttempts: 30", "maxAttempts: -30"}})
+	actualConfig := &Config{}
+	err := config.NewDefaultLoader("").LoadFromReader(bytes.NewReader(yamlData), config.DataTypeYAML, actualConfig)
+	require.Error(t, err)
+	require.Equal(t, "client max retry attempts must be positive", err.Error())
+}
+
+func TestConfigLogger(t *testing.T) {
+	yamlData := testYamlData([][]string{{"slowRequestThreshold: 5s", "slowRequestThreshold: -5s"}})
+	actualConfig := &Config{}
+	err := config.NewDefaultLoader("").LoadFromReader(bytes.NewReader(yamlData), config.DataTypeYAML, actualConfig)
+	require.Error(t, err)
+	require.Equal(t, "client logger slow request threshold can not be negative", err.Error())
+
+	yamlData = testYamlData([][]string{{"mode: all", "mode: invalid"}})
+	actualConfig = &Config{}
+	err = config.NewDefaultLoader("").LoadFromReader(bytes.NewReader(yamlData), config.DataTypeYAML, actualConfig)
+	require.Error(t, err)
+	require.Equal(t, "client logger invalid mode, choose one of: [none, all, failed]", err.Error())
+}
+
+func TestConfigRetriesPolicy(t *testing.T) {
+	yamlData := testYamlData([][]string{{"strategy: exponential", "strategy: invalid"}})
+	actualConfig := &Config{}
+	err := config.NewDefaultLoader("").LoadFromReader(bytes.NewReader(yamlData), config.DataTypeYAML, actualConfig)
+	require.Error(t, err)
+	require.Equal(t, "client retry policy must be one of: [exponential, constant]", err.Error())
+
+	yamlData = testYamlData([][]string{
+		{"exponentialBackoffInitialInterval: 3s", "exponentialBackoffInitialInterval: -1s"},
+	})
+	err = config.NewDefaultLoader("").LoadFromReader(bytes.NewReader(yamlData), config.DataTypeYAML, actualConfig)
+	require.Error(t, err)
+	require.Equal(t, "client exponential backoff initial interval must be positive", err.Error())
+
+	yamlData = testYamlData([][]string{{"exponentialBackoffMultiplier: 2", "exponentialBackoffMultiplier: 1"}})
+	err = config.NewDefaultLoader("").LoadFromReader(bytes.NewReader(yamlData), config.DataTypeYAML, actualConfig)
+	require.Error(t, err)
+	require.Equal(t, "client exponential backoff multiplier must be greater than 1", err.Error())
+
+	yamlData = testYamlData([][]string{
+		{"strategy: exponential", "strategy: constant"},
+		{"constantBackoffInterval: 2s", "constantBackoffInterval: -3s"},
+	})
+	err = config.NewDefaultLoader("").LoadFromReader(bytes.NewReader(yamlData), config.DataTypeYAML, actualConfig)
+	require.Error(t, err)
+	require.Equal(t, "client constant backoff interval must be positive", err.Error())
+
+	yamlData = testYamlData([][]string{
+		{"strategy: exponential", "strategy:"},
+	})
+	err = config.NewDefaultLoader("").LoadFromReader(bytes.NewReader(yamlData), config.DataTypeYAML, actualConfig)
+	require.NoError(t, err)
+	require.Nil(t, actualConfig.Retries.GetPolicy())
+
+	yamlData = testYamlData(nil)
+	err = config.NewDefaultLoader("").LoadFromReader(bytes.NewReader(yamlData), config.DataTypeYAML, actualConfig)
+	require.NoError(t, err)
+	require.Implements(t, (*retry.Policy)(nil), actualConfig.Retries.GetPolicy())
+}
+
+func TestConfigDisableWithLoader(t *testing.T) {
+	yamlData := []byte(`
+retries:
+  enabled: false
+rateLimits:
+  enabled: false
+logger:
+  enabled: false
+metrics:
+  enabled: false
+timeout: 30s
+`)
+	actualConfig := &Config{}
+	err := config.NewDefaultLoader("").LoadFromReader(bytes.NewReader(yamlData), config.DataTypeYAML, actualConfig)
+	require.NoError(t, err)
+}
+
+func testYamlData(replacements [][]string) []byte {
+	yamlData := `
+retries:
+  enabled: true
+  maxAttempts: 30
+  policy:
+    strategy: exponential
+    exponentialBackoffInitialInterval: 3s
+    exponentialBackoffMultiplier: 2
+    constantBackoffInterval: 2s
+rateLimits:
+  enabled: true
+  limit: 300
+  burst: 3000
+  waitTimeout: 3s
+logger:
+  enabled: true
+  slowRequestThreshold: 5s
+  mode: all
+metrics:
+  enabled: true
+timeout: 30s
+`
+	for i := range replacements {
+		yamlData = strings.Replace(yamlData, replacements[i][0], replacements[i][1], 1)
+	}
+
+	return []byte(yamlData)
+}

httpclient/httpclient.go

@@ -6,7 +6,12 @@ Released under MIT license.
 
 package httpclient
 
-import "net/http"
+import (
+	"context"
+	"fmt"
+	"github.com/acronis/go-appkit/log"
+	"net/http"
+)
 
 // CloneHTTPRequest creates a shallow copy of the request along with a deep copy of the Headers.
 func CloneHTTPRequest(req *http.Request) *http.Request {
@@ -26,3 +31,113 @@ func CloneHTTPHeader(in http.Header) http.Header {
 	}
 	return out
 }
+
+// ClientProviders for further customization of the client logging and request id.
+type ClientProviders struct {
+	// Logger is a function that provides a context-specific logger.
+	Logger func(ctx context.Context) log.FieldLogger
+
+	// RequestID is a function that provides a request ID.
+	RequestID func(ctx context.Context) string
+}
+
+// NewHTTPClient wraps delegate transports with logging, metrics, rate limiting, retryable, user agent, request id
+// and returns an error if any occurs.
+func NewHTTPClient(
+	cfg *Config,
+	userAgent string,
+	reqType string,
+	delegate http.RoundTripper,
+	providers ClientProviders,
+) (*http.Client, error) {
+	var err error
+
+	if delegate == nil {
+		delegate = http.DefaultTransport.(*http.Transport).Clone()
+	}
+
+	if cfg.Logger.Enabled {
+		opts := cfg.Logger.TransportOpts()
+		opts.LoggerProvider = providers.Logger
+		delegate = NewLoggingRoundTripperWithOpts(delegate, reqType, opts)
+	}
+
+	if cfg.Metrics.Enabled {
+		delegate = NewMetricsRoundTripper(delegate, reqType)
+	}
+
+	if cfg.RateLimits.Enabled {
+		delegate, err = NewRateLimitingRoundTripperWithOpts(
+			delegate, cfg.RateLimits.Limit, cfg.RateLimits.TransportOpts(),
+		)
+		if err != nil {
+			return nil, fmt.Errorf("create rate limiting round tripper: %w", err)
+		}
+	}
+
+	if cfg.Retries.Enabled {
+		opts := cfg.Retries.TransportOpts()
+		opts.LoggerProvider = providers.Logger
+		opts.BackoffPolicy = cfg.Retries.GetPolicy()
+		delegate, err = NewRetryableRoundTripperWithOpts(delegate, opts)
+		if err != nil {
+			return nil, fmt.Errorf("create retryable round tripper: %w", err)
+		}
+	}
+
+	delegate = NewUserAgentRoundTripper(delegate, userAgent)
+	delegate = NewRequestIDRoundTripperWithOpts(delegate, RequestIDRoundTripperOpts{
+		RequestIDProvider: providers.RequestID,
+	})
+
+	return &http.Client{Transport: delegate, Timeout: cfg.Timeout}, nil
+}
+
+// MustHTTPClient wraps delegate transports with logging, metrics, rate limiting, retryable, user agent, request id
+// and panics if any error occurs.
+func MustHTTPClient(
+	cfg *Config,
+	userAgent,
+	reqType string,
+	delegate http.RoundTripper,
+	providers ClientProviders,
+) *http.Client {
+	client, err := NewHTTPClient(cfg, userAgent, reqType, delegate, providers)
+	if err != nil {
+		panic(err)
+	}
+
+	return client
+}
+
+// ClientOpts provides options for NewHTTPClientWithOpts and MustHTTPClientWithOpts functions.
+type ClientOpts struct {
+	// Config is the configuration for the HTTP client.
+	Config Config
+
+	// UserAgent is a user agent string.
+	UserAgent string
+
+	// ReqType is a type of request.
+	ReqType string
+
+	// Delegate is the next RoundTripper in the chain.
+	Delegate http.RoundTripper
+
+	// Providers are the functions that provide a context-specific logger and request ID.
+	Providers ClientProviders
+}
+
+// NewHTTPClientWithOpts wraps delegate transports with options
+// logging, metrics, rate limiting, retryable, user agent, request id
+// and returns an error if any occurs.
+func NewHTTPClientWithOpts(opts ClientOpts) (*http.Client, error) {
+	return NewHTTPClient(&opts.Config, opts.UserAgent, opts.ReqType, opts.Delegate, opts.Providers)
+}
+
+// MustHTTPClientWithOpts wraps delegate transports with options
+// logging, metrics, rate limiting, retryable, user agent, request id
+// and panics if any error occurs.
+func MustHTTPClientWithOpts(opts ClientOpts) *http.Client {
+	return MustHTTPClient(&opts.Config, opts.UserAgent, opts.ReqType, opts.Delegate, opts.Providers)
+}