HTTP client round trippers logging, metrics and request id

httpclient/config.go

@@ -0,0 +1,161 @@
+/*
+Copyright © 2024 Acronis International GmbH.
+
+Released under MIT license.
+*/
+
+package httpclient
+
+import (
+	"github.com/acronis/go-appkit/config"
+	"time"
+)
+
+const (
+	// DefaultLimit is the default limit for rate limiting.
+	DefaultLimit = 10
+
+	// DefaultBurst is the default burst for rate limiting.
+	DefaultBurst = 100
+
+	// DefaultWaitTimeout is the default wait timeout for rate limiting.
+	DefaultWaitTimeout = 10 * time.Second
+
+	// configuration properties
+	cfgKeyRetriesMax            = "retries.max_retry_attempts"
+	cfgKeyRateLimitsLimit       = "rate_limits.limit"
+	cfgKeyRateLimitsBurst       = "rate_limits.burst"
+	cfgKeyRateLimitsWaitTimeout = "rate_limits.wait_timeout"
+	cfgKeyTimeout               = "timeout"
+)
+
+// RateLimitConfig represents configuration options for HTTP client rate limits.
+type RateLimitConfig struct {
+	Limit       int           `mapstructure:"limit"`
+	Burst       int           `mapstructure:"burst"`
+	WaitTimeout time.Duration `mapstructure:"wait_timeout"`
+}
+
+// Set is part of config interface implementation.
+func (c *RateLimitConfig) Set(dp config.DataProvider) (err error) {
+	limit, err := dp.GetInt(cfgKeyRateLimitsLimit)
+	if err != nil {
+		return err
+	}
+	c.Limit = limit
+
+	burst, err := dp.GetInt(cfgKeyRateLimitsBurst)
+	if err != nil {
+		return nil
+	}
+	c.Burst = burst
+
+	waitTimeout, err := dp.GetDuration(cfgKeyRateLimitsWaitTimeout)
+	if err != nil {
+		return err
+	}
+	c.WaitTimeout = waitTimeout
+
+	return nil
+}
+
+// SetProviderDefaults is part of config interface implementation.
+func (c *RateLimitConfig) SetProviderDefaults(dp config.DataProvider) {
+	dp.SetDefault(cfgKeyRateLimitsLimit, DefaultLimit)
+	dp.SetDefault(cfgKeyRateLimitsBurst, DefaultBurst)
+	dp.SetDefault(cfgKeyRateLimitsWaitTimeout, DefaultWaitTimeout)
+}
+
+// TransportOpts returns transport options.
+func (c *RateLimitConfig) TransportOpts() RateLimitingRoundTripperOpts {
+	return RateLimitingRoundTripperOpts{
+		Burst:       c.Burst,
+		WaitTimeout: c.WaitTimeout,
+	}
+}
+
+// RetriesConfig represents configuration options for HTTP client retries policy.
+type RetriesConfig struct {
+	MaxRetryAttempts int `mapstructure:"max_retry_attempts"`
+}
+
+// Set is part of config interface implementation.
+func (c *RetriesConfig) Set(dp config.DataProvider) error {
+	maxRetryAttempts, err := dp.GetInt(cfgKeyRetriesMax)
+	if err != nil {
+		return err
+	}
+	c.MaxRetryAttempts = maxRetryAttempts
+
+	return nil
+}
+
+// SetProviderDefaults is part of config interface implementation.
+func (c *RetriesConfig) SetProviderDefaults(dp config.DataProvider) {
+	dp.SetDefault(cfgKeyRetriesMax, DefaultMaxRetryAttempts)
+}
+
+// TransportOpts returns transport options.
+func (c *RetriesConfig) TransportOpts() RetryableRoundTripperOpts {
+	return RetryableRoundTripperOpts{MaxRetryAttempts: c.MaxRetryAttempts}
+}
+
+// Config represents options for HTTP client configuration.
+type Config struct {
+	Retries    *RetriesConfig   `mapstructure:"retries"`
+	RateLimits *RateLimitConfig `mapstructure:"rate_limits"`
+	Timeout    time.Duration    `mapstructure:"timeout"`
+}
+
+// Set is part of config interface implementation.
+func (c *Config) Set(dp config.DataProvider) error {
+	timeout, err := dp.GetDuration(cfgKeyTimeout)
+	if err != nil {
+		return err
+	}
+	c.Timeout = timeout
+
+	err = c.Retries.Set(config.NewKeyPrefixedDataProvider(dp, ""))
+	if err != nil {
+		return err
+	}
+
+	err = c.RateLimits.Set(config.NewKeyPrefixedDataProvider(dp, ""))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// SetProviderDefaults is part of config interface implementation.
+func (c *Config) SetProviderDefaults(dp config.DataProvider) {
+	if c.Retries == nil {
+		c.Retries = &RetriesConfig{}
+	}
+	c.Retries.SetProviderDefaults(dp)
+
+	if c.RateLimits == nil {
+		c.RateLimits = &RateLimitConfig{}
+	}
+	c.RateLimits.SetProviderDefaults(dp)
+
+	if c.Timeout == 0 {
+		c.Timeout = DefaultWaitTimeout
+	}
+}
+
+// NewHTTPClientConfig is unified configuration format for HTTP clients.
+func NewHTTPClientConfig() *Config {
+	return &Config{
+		Retries: &RetriesConfig{
+			MaxRetryAttempts: DefaultMaxRetryAttempts,
+		},
+		RateLimits: &RateLimitConfig{
+			Limit:       DefaultLimit,
+			Burst:       DefaultBurst,
+			WaitTimeout: DefaultWaitTimeout,
+		},
+		Timeout: DefaultWaitTimeout,
+	}
+}

httpclient/config_test.go

@@ -0,0 +1,45 @@
+/*
+Copyright © 2024 Acronis International GmbH.
+
+Released under MIT license.
+*/
+
+package httpclient
+
+import (
+	"bytes"
+	"github.com/acronis/go-appkit/config"
+	"github.com/stretchr/testify/require"
+	"testing"
+	"time"
+)
+
+func TestConfigWithLoader(t *testing.T) {
+	yamlData := []byte(`
+retries:
+  max_retry_attempts: 30
+rate_limits:
+  limit: 300
+  burst: 3000
+  wait_timeout: 3s
+timeout: 30s
+`)
+
+	actualConfig := &Config{}
+	err := config.NewDefaultLoader("").LoadFromReader(bytes.NewReader(yamlData), config.DataTypeYAML, actualConfig)
+	require.NoError(t, err, "load configuration")
+
+	expectedConfig := &Config{
+		Retries: &RetriesConfig{
+			MaxRetryAttempts: 30,
+		},
+		RateLimits: &RateLimitConfig{
+			Limit:       300,
+			Burst:       3000,
+			WaitTimeout: 3 * time.Second,
+		},
+		Timeout: 30 * time.Second,
+	}
+
+	require.Equal(t, expectedConfig, actualConfig, "configuration does not match expected")
+}

httpclient/httpclient.go

@@ -6,7 +6,10 @@ Released under MIT license.
 
 package httpclient
 
-import "net/http"
+import (
+	"fmt"
+	"net/http"
+)
 
 // CloneHTTPRequest creates a shallow copy of the request along with a deep copy of the Headers.
 func CloneHTTPRequest(req *http.Request) *http.Request {
@@ -26,3 +29,30 @@ func CloneHTTPHeader(in http.Header) http.Header {
 	}
 	return out
 }
+
+// MustHTTPClient wraps delegate transports with logging, metrics, rate limiting, retryable, user agent, request id
+// and panics if any error occurs.
+func MustHTTPClient(cfg *Config, userAgent, reqType string, delegate http.RoundTripper) *http.Client {
+	if delegate == nil {
+		delegate = http.DefaultTransport
+	}
+
+	delegate = NewLoggingRoundTripper(delegate, reqType)
+	delegate = NewMetricsRoundTripper(delegate, reqType)
+
+	delegate, err := NewRateLimitingRoundTripperWithOpts(
+		delegate, cfg.RateLimits.Limit, cfg.RateLimits.TransportOpts(),
+	)
+	if err != nil {
+		panic(fmt.Errorf("create rate limiting round tripper: %w", err))
+	}
+
+	delegate, err = NewRetryableRoundTripperWithOpts(delegate, cfg.Retries.TransportOpts())
+	if err != nil {
+		panic(fmt.Errorf("create retryable round tripper: %w", err))
+	}
+
+	delegate = NewUserAgentRoundTripper(delegate, userAgent)
+	delegate = NewRequestIDRoundTripper(delegate)
+	return &http.Client{Transport: delegate, Timeout: cfg.Timeout}
+}

httpclient/httpclient_test.go

@@ -0,0 +1,39 @@
+/*
+Copyright © 2024 Acronis International GmbH.
+
+Released under MIT license.
+*/
+
+package httpclient
+
+import (
+	"context"
+	"github.com/acronis/go-appkit/httpserver/middleware"
+	"github.com/acronis/go-appkit/log/logtest"
+	"github.com/stretchr/testify/require"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestMustHTTPClientLoggingRoundTripper(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		rw.WriteHeader(http.StatusTeapot)
+	}))
+	defer server.Close()
+
+	logger := logtest.NewRecorder()
+	cfg := NewHTTPClientConfig()
+	client := MustHTTPClient(cfg, "test-agent", "test-request", nil)
+	ctx := middleware.NewContextWithLogger(context.Background(), logger)
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, server.URL, nil)
+	require.NoError(t, err)
+
+	r, err := client.Do(req)
+	defer func() { _ = r.Body.Close() }()
+	require.NoError(t, err)
+	require.NotEmpty(t, logger.Entries())
+
+	loggerEntry := logger.Entries()[0]
+	require.Contains(t, loggerEntry.Text, "external request POST "+server.URL+" req type test-request status code 418")
+}

httpclient/logging_round_tripper.go

@@ -0,0 +1,58 @@
+/*
+Copyright © 2024 Acronis International GmbH.
+
+Released under MIT license.
+*/
+
+package httpclient
+
+import (
+	"fmt"
+	"github.com/acronis/go-appkit/httpserver/middleware"
+	"net/http"
+	"time"
+)
+
+// LoggingRoundTripper implements http.RoundTripper for logging requests.
+type LoggingRoundTripper struct {
+	Delegate http.RoundTripper
+	ReqType  string
+}
+
+// NewLoggingRoundTripper creates an HTTP transport that log requests.
+func NewLoggingRoundTripper(delegate http.RoundTripper, reqType string) http.RoundTripper {
+	return &LoggingRoundTripper{
+		Delegate: delegate,
+		ReqType:  reqType,
+	}
+}
+
+// RoundTrip adds logging capabilities to the HTTP transport.
+func (rt *LoggingRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
+	ctx := r.Context()
+	logger := middleware.GetLoggerFromContext(ctx)
+	start := time.Now()
+
+	resp, err := rt.Delegate.RoundTrip(r)
+	if logger != nil {
+		common := "external request %s %s req type %s "
+		elapsed := time.Since(start)
+
+		args := []interface{}{r.Method, r.URL.String(), rt.ReqType, elapsed.Seconds(), err}
+		message := common + "time taken %.3f, err %+v"
+
+		if resp != nil {
+			args = []interface{}{r.Method, r.URL.String(), rt.ReqType, resp.StatusCode, elapsed.Seconds(), err}
+			message = common + "status code %d, time taken %.3f, err %+v"
+		}
+
+		logger.Infof(message, args...)
+
+		loggingParams := middleware.GetLoggingParamsFromContext(ctx)
+		if loggingParams != nil {
+			loggingParams.AddTimeSlotDurationInMs(fmt.Sprintf("external_request_%s_ms", rt.ReqType), elapsed)
+		}
+	}
+
+	return resp, err
+}