From d12c3778f176bf84d464d0236a6e40e63c337bec Mon Sep 17 00:00:00 2001 From: Herman Slatman Date: Wed, 13 Nov 2024 00:41:54 +0100 Subject: [PATCH 01/11] Add foundation for `AppSec` integration --- appsec/appsec.go | 108 ++++++++++++++++++++++++++ crowdsec/crowdsec.go | 10 ++- http/http.go | 50 ++++-------- imports.go | 1 + internal/bouncer/appsec.go | 85 ++++++++++++++++++++ internal/bouncer/bouncer.go | 11 ++- internal/bouncer/bouncer_test.go | 2 +- internal/utils/http.go | 39 ++++++++++ {http => internal/utils}/http_test.go | 6 +- 9 files changed, 271 insertions(+), 41 deletions(-) create mode 100644 appsec/appsec.go create mode 100644 internal/bouncer/appsec.go create mode 100644 internal/utils/http.go rename {http => internal/utils}/http_test.go (95%) diff --git a/appsec/appsec.go b/appsec/appsec.go new file mode 100644 index 00000000..add26410 --- /dev/null +++ b/appsec/appsec.go @@ -0,0 +1,108 @@ +// Copyright 2024 Herman Slatman +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package appsec + +import ( + "errors" + "fmt" + "net/http" + + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" + "github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile" + "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "github.com/hslatman/caddy-crowdsec-bouncer/crowdsec" + "go.uber.org/zap" +) + +func init() { + caddy.RegisterModule(Handler{}) + httpcaddyfile.RegisterHandlerDirective("appsec", parseCaddyfileHandlerDirective) +} + +// Handler matches request IPs to CrowdSec decisions to (dis)allow access +type Handler struct { + logger *zap.Logger + crowdsec *crowdsec.CrowdSec +} + +// CaddyModule returns the Caddy module information. +func (Handler) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "http.handlers.appsec", + New: func() caddy.Module { return new(Handler) }, + } +} + +// Provision sets up the CrowdSec handler. +func (h *Handler) Provision(ctx caddy.Context) error { + crowdsecAppIface, err := ctx.App("crowdsec") + if err != nil { + return fmt.Errorf("getting crowdsec app: %v", err) + } + h.crowdsec = crowdsecAppIface.(*crowdsec.CrowdSec) + + h.logger = ctx.Logger(h) + defer h.logger.Sync() // nolint + + return nil +} + +// Validate ensures the app's configuration is valid. +func (h *Handler) Validate() error { + if h.crowdsec == nil { + return errors.New("crowdsec app not available") + } + + return nil +} + +// ServeHTTP is the Caddy handler for serving HTTP requests +func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { + err := h.crowdsec.CheckRequest(r.Context(), r) + if err != nil { + // TODO: do something with the error + // TODO: add (debug) logging + } + + // Continue down the handler stack + if err := next.ServeHTTP(w, r); err != nil { + return err + } + + return nil +} + +// UnmarshalCaddyfile implements caddyfile.Unmarshaler. +func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { + // TODO: parse additional handler directives (none exist now) + return nil +} + +// parseCaddyfileHandlerDirective parses the `crowdsec` Caddyfile directive +func parseCaddyfileHandlerDirective(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) { + var handler Handler + err := handler.UnmarshalCaddyfile(h.Dispenser) + return handler, err +} + +// Interface guards +var ( + _ caddy.Module = (*Handler)(nil) + _ caddy.Provisioner = (*Handler)(nil) + _ caddy.Validator = (*Handler)(nil) + _ caddyhttp.MiddlewareHandler = (*Handler)(nil) + _ caddyfile.Unmarshaler = (*Handler)(nil) +) diff --git a/crowdsec/crowdsec.go b/crowdsec/crowdsec.go index f01a2a89..e5f91249 100644 --- a/crowdsec/crowdsec.go +++ b/crowdsec/crowdsec.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "net/http" "net/netip" "reflect" "runtime/debug" @@ -69,6 +70,8 @@ type CrowdSec struct { // validations. Defaults to false. EnableHardFails *bool `json:"enable_hard_fails,omitempty"` + AppSecUrl string `json:"appsec_url,omitempty"` // TODO: documentation + ctx caddy.Context logger *zap.Logger bouncer *bouncer.Bouncer @@ -84,6 +87,7 @@ func (c *CrowdSec) Provision(ctx caddy.Context) error { c.APIUrl = repl.ReplaceKnown(c.APIUrl, "") c.APIKey = repl.ReplaceKnown(c.APIKey, "") c.TickerInterval = repl.ReplaceKnown(c.TickerInterval, "") + c.AppSecUrl = repl.ReplaceKnown(c.AppSecUrl, "") if c.APIUrl == "" { c.APIUrl = "http://127.0.0.1:8080/" @@ -92,7 +96,7 @@ func (c *CrowdSec) Provision(ctx caddy.Context) error { c.TickerInterval = "60s" } - bouncer, err := bouncer.New(c.APIKey, c.APIUrl, c.TickerInterval, c.logger) + bouncer, err := bouncer.New(c.APIKey, c.APIUrl, c.AppSecUrl, c.TickerInterval, c.logger) if err != nil { return err } @@ -253,6 +257,10 @@ func (c *CrowdSec) IsAllowed(ip netip.Addr) (bool, *models.Decision, error) { return c.bouncer.IsAllowed(ip) } +func (c *CrowdSec) CheckRequest(ctx context.Context, r *http.Request) error { + return c.bouncer.CheckRequest(ctx, r) +} + func (c *CrowdSec) isStreamingEnabled() bool { return c.EnableStreaming == nil || *c.EnableStreaming } diff --git a/http/http.go b/http/http.go index 6b784748..bc295ffc 100644 --- a/http/http.go +++ b/http/http.go @@ -18,15 +18,17 @@ import ( "errors" "fmt" "net/http" - "net/netip" "time" "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" "github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile" "github.com/caddyserver/caddy/v2/modules/caddyhttp" - "github.com/hslatman/caddy-crowdsec-bouncer/crowdsec" "go.uber.org/zap" + + _ "github.com/hslatman/caddy-crowdsec-bouncer/appsec" // include support for AppSec WAF component + "github.com/hslatman/caddy-crowdsec-bouncer/crowdsec" + "github.com/hslatman/caddy-crowdsec-bouncer/internal/utils" ) func init() { @@ -38,6 +40,8 @@ func init() { type Handler struct { logger *zap.Logger crowdsec *crowdsec.CrowdSec + + appsecEnabled bool } // CaddyModule returns the Caddy module information. @@ -56,6 +60,8 @@ func (h *Handler) Provision(ctx caddy.Context) error { } h.crowdsec = crowdsecAppIface.(*crowdsec.CrowdSec) + h.appsecEnabled = true // TODO: move to provisioning; unmarshaling of Caddyfile; etc + h.logger = ctx.Logger(h) defer h.logger.Sync() // nolint @@ -74,7 +80,7 @@ func (h *Handler) Validate() error { // ServeHTTP is the Caddy handler for serving HTTP requests func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { - ipToCheck, err := determineIPFromRequest(r) + ipToCheck, err := utils.DetermineIPFromRequest(r) if err != nil { return err // TODO: return error here? Or just log it and continue serving } @@ -107,9 +113,14 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhtt // TODO: if the IP is allowed, should we (temporarily) put it in an explicit allowlist for quicker check? + if h.appsecEnabled { + if err := h.crowdsec.CheckRequest(r.Context(), r); err != nil { + // TODO: do something with the error + } + } + // Continue down the handler stack - err = next.ServeHTTP(w, r) - if err != nil { + if err := next.ServeHTTP(w, r); err != nil { return err } @@ -144,35 +155,6 @@ func writeThrottleResponse(w http.ResponseWriter, duration string) error { return nil } -// determineIPFromRequest returns the IP of the client based on the value that -// Caddy extracts from the original request and stores in the request context. -// Support for setting the real client IP in case a proxy sits in front of -// Caddy was added, so the client IP reported here is the actual client IP. -func determineIPFromRequest(r *http.Request) (netip.Addr, error) { - zero := netip.Addr{} - clientIPVar := caddyhttp.GetVar(r.Context(), caddyhttp.ClientIPVarKey) - if clientIPVar == nil { - return zero, errors.New("failed getting client IP from context") - } - - var clientIP string - var ok bool - if clientIP, ok = clientIPVar.(string); !ok { - return zero, fmt.Errorf("client IP from request context is invalid type %T", clientIPVar) - } - - if clientIP == "" { - return zero, errors.New("client IP from request context is empty") - } - - ip, err := netip.ParseAddr(clientIP) - if err != nil { - return zero, fmt.Errorf("could not parse %q into netip.Addr", clientIP) - } - - return ip, nil -} - // UnmarshalCaddyfile implements caddyfile.Unmarshaler. func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { // TODO: parse additional handler directives (none exist now) diff --git a/imports.go b/imports.go index c9bc939f..ba536169 100644 --- a/imports.go +++ b/imports.go @@ -4,6 +4,7 @@ import ( // Import the default CrowdSec modules. Primary reason this // file exists is to satisfy the Caddy documentation and download // pages to list the modules correctly. + _ "github.com/hslatman/caddy-crowdsec-bouncer/appsec" _ "github.com/hslatman/caddy-crowdsec-bouncer/crowdsec" _ "github.com/hslatman/caddy-crowdsec-bouncer/http" _ "github.com/hslatman/caddy-crowdsec-bouncer/layer4" diff --git a/internal/bouncer/appsec.go b/internal/bouncer/appsec.go new file mode 100644 index 00000000..bab91aa5 --- /dev/null +++ b/internal/bouncer/appsec.go @@ -0,0 +1,85 @@ +package bouncer + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "time" + + "github.com/hslatman/caddy-crowdsec-bouncer/internal/utils" +) + +type appsec struct { + apiURL string + apiKey string + client *http.Client +} + +func newAppSec(apiURL, apiKey string) *appsec { + return &appsec{ + apiURL: apiURL, + apiKey: apiKey, + client: &http.Client{ + Timeout: 10 * time.Second, + }, + } +} + +func (a *appsec) checkRequest(ctx context.Context, r *http.Request) error { + if a.apiURL == "" { + return nil // AppSec component not enabled + } + + // TODO: add (debug) logging + // TODO: return a decision, and act on it in the handler (named/typed error) + + originalIP, err := utils.DetermineIPFromRequest(r) + if err != nil { + return err // TODO: return error here? Or just log it and continue serving + } + + originalBody, err := io.ReadAll(r.Body) + if err != nil { + return err + } + + method := http.MethodGet + var body io.ReadCloser = http.NoBody + if len(originalBody) > 0 { + method = http.MethodPost + body = io.NopCloser(bytes.NewBuffer(originalBody)) + } + r.Body = io.NopCloser(bytes.NewBuffer(originalBody)) + + req, err := http.NewRequestWithContext(ctx, method, a.apiURL, body) + if err != nil { + return err + } + + req.Header.Set("X-Crowdsec-Appsec-Ip", originalIP.String()) + req.Header.Set("X-Crowdsec-Appsec-Uri", r.URL.String()) + req.Header.Set("X-Crowdsec-Appsec-Host", r.Host) + req.Header.Set("X-Crowdsec-Appsec-Verb", r.Method) + req.Header.Set("X-Crowdsec-Appsec-Api-Key", a.apiKey) + + resp, err := a.client.Do(req) + if err != nil { + return err + } + + switch resp.StatusCode { + case 200: + return nil // TODO: read decision from body? + case 401: + return errors.New("not authenticated") + case 403: + return errors.New("not allowed") // TODO: read decision + case 500: + return errors.New("internal error") + default: + return fmt.Errorf("unsupported status code %d", resp.StatusCode) + } +} diff --git a/internal/bouncer/bouncer.go b/internal/bouncer/bouncer.go index 51c6b661..213d1bd6 100644 --- a/internal/bouncer/bouncer.go +++ b/internal/bouncer/bouncer.go @@ -19,6 +19,7 @@ import ( "encoding/hex" "fmt" "math/rand" + "net/http" "net/netip" "sync" "time" @@ -31,7 +32,7 @@ import ( const ( userAgentName = "caddy-cs-bouncer" - userAgentVersion = "v0.7.0" + userAgentVersion = "v0.8.0" maxNumberOfDecisionsToLog = 10 ) @@ -44,6 +45,7 @@ type Bouncer struct { streamingBouncer *csbouncer.StreamBouncer liveBouncer *csbouncer.LiveBouncer metricsProvider *csbouncer.MetricsProvider + appsec *appsec store *store logger *zap.Logger useStreamingBouncer bool @@ -62,7 +64,7 @@ type Bouncer struct { // New creates a new (streaming) Bouncer with a storage based on immutable radix tree // TODO: take a configuration struct instead, because more options will be added. -func New(apiKey, apiURL, tickerInterval string, logger *zap.Logger) (*Bouncer, error) { +func New(apiKey, apiURL, appSecURL, tickerInterval string, logger *zap.Logger) (*Bouncer, error) { userAgent := fmt.Sprintf("%s/%s", userAgentName, userAgentVersion) insecureSkipVerify := false instantiatedAt := time.Now() @@ -86,6 +88,7 @@ func New(apiKey, apiURL, tickerInterval string, logger *zap.Logger) (*Bouncer, e InsecureSkipVerify: &insecureSkipVerify, UserAgent: userAgent, }, + appsec: newAppSec(appSecURL, apiKey), store: newStore(), logger: logger, instantiatedAt: instantiatedAt, @@ -219,6 +222,10 @@ func (b *Bouncer) IsAllowed(ip netip.Addr) (bool, *models.Decision, error) { return isAllowed, nil, nil } +func (b *Bouncer) CheckRequest(ctx context.Context, r *http.Request) error { + return b.appsec.checkRequest(ctx, r) +} + func generateInstanceID(t time.Time) (string, error) { r := rand.New(rand.NewSource(t.Unix())) b := [4]byte{} diff --git a/internal/bouncer/bouncer_test.go b/internal/bouncer/bouncer_test.go index 8c16217b..196ef827 100644 --- a/internal/bouncer/bouncer_test.go +++ b/internal/bouncer/bouncer_test.go @@ -24,7 +24,7 @@ func newBouncer(t *testing.T) (*Bouncer, error) { tickerInterval := "10s" logger := zaptest.NewLogger(t) - bouncer, err := New(key, host, tickerInterval, logger) + bouncer, err := New(key, host, "", tickerInterval, logger) require.NoError(t, err) bouncer.EnableStreaming() diff --git a/internal/utils/http.go b/internal/utils/http.go new file mode 100644 index 00000000..b7be55a5 --- /dev/null +++ b/internal/utils/http.go @@ -0,0 +1,39 @@ +package utils + +import ( + "errors" + "fmt" + "net/http" + "net/netip" + + "github.com/caddyserver/caddy/v2/modules/caddyhttp" +) + +// DetermineIPFromRequest returns the IP of the client based on the value that +// Caddy extracts from the original request and stores in the request context. +// Support for setting the real client IP in case a proxy sits in front of +// Caddy was added, so the client IP reported here is the actual client IP. +func DetermineIPFromRequest(r *http.Request) (netip.Addr, error) { + zero := netip.Addr{} + clientIPVar := caddyhttp.GetVar(r.Context(), caddyhttp.ClientIPVarKey) + if clientIPVar == nil { + return zero, errors.New("failed getting client IP from context") + } + + var clientIP string + var ok bool + if clientIP, ok = clientIPVar.(string); !ok { + return zero, fmt.Errorf("client IP from request context is invalid type %T", clientIPVar) + } + + if clientIP == "" { + return zero, errors.New("client IP from request context is empty") + } + + ip, err := netip.ParseAddr(clientIP) + if err != nil { + return zero, fmt.Errorf("could not parse %q into netip.Addr", clientIP) + } + + return ip, nil +} diff --git a/http/http_test.go b/internal/utils/http_test.go similarity index 95% rename from http/http_test.go rename to internal/utils/http_test.go index 2776331c..6c84007d 100644 --- a/http/http_test.go +++ b/internal/utils/http_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 Herman Slatman +// Copyright 2024 Herman Slatman // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package http +package utils import ( "context" @@ -58,7 +58,7 @@ func Test_determineIPFromRequest(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := determineIPFromRequest(tt.args.r) + got, err := DetermineIPFromRequest(tt.args.r) if (err != nil) != tt.wantErr { t.Errorf("determineIPFromRequest() error = %v, wantErr %v", err, tt.wantErr) return From 52328ea2bfcfb1fa684ea24a1254a4495cf177b4 Mon Sep 17 00:00:00 2001 From: Herman Slatman Date: Fri, 15 Nov 2024 01:09:14 +0100 Subject: [PATCH 02/11] Enforce `AppSec` component rule evaluation results This commit ensures that responses from the CrowdSec `AppSec` remediation component are properly evaluated and acted upon. The `AppSec` support can be enabled either as a dedicated handler by configuring a route to have the `appsec` directive. That will use the `http.handlers.appsec` module. It also enables `AppSec` checks on the the HTTP handler configured through the `crowdsec` directive using the `http.handlers.crowdsec` module by default. In a future commit `AppSec` support will be enabled based on configuration on the `http.handlers.crowdsec` component. --- appsec/appsec.go | 30 +++++++++++--- crowdsec/caddyfile.go | 5 +++ http/http.go | 78 +++++++++++------------------------ internal/bouncer/appsec.go | 54 ++++++++++++++++++------ internal/bouncer/bouncer.go | 6 ++- internal/bouncer/decisions.go | 4 ++ internal/bouncer/errors.go | 12 ++++++ internal/bouncer/metrics.go | 17 ++++++++ internal/utils/http.go | 54 ++++++++++++++++++++++++ 9 files changed, 188 insertions(+), 72 deletions(-) create mode 100644 internal/bouncer/errors.go diff --git a/appsec/appsec.go b/appsec/appsec.go index add26410..b7bc83ca 100644 --- a/appsec/appsec.go +++ b/appsec/appsec.go @@ -24,6 +24,8 @@ import ( "github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile" "github.com/caddyserver/caddy/v2/modules/caddyhttp" "github.com/hslatman/caddy-crowdsec-bouncer/crowdsec" + "github.com/hslatman/caddy-crowdsec-bouncer/internal/bouncer" + "github.com/hslatman/caddy-crowdsec-bouncer/internal/utils" "go.uber.org/zap" ) @@ -32,7 +34,8 @@ func init() { httpcaddyfile.RegisterHandlerDirective("appsec", parseCaddyfileHandlerDirective) } -// Handler matches request IPs to CrowdSec decisions to (dis)allow access +// Handler checks the CrowdSec AppSec component decided whether +// an HTTP request is blocked or not. type Handler struct { logger *zap.Logger crowdsec *crowdsec.CrowdSec @@ -46,7 +49,7 @@ func (Handler) CaddyModule() caddy.ModuleInfo { } } -// Provision sets up the CrowdSec handler. +// Provision sets up the CrowdSec AppSec handler. func (h *Handler) Provision(ctx caddy.Context) error { crowdsecAppIface, err := ctx.App("crowdsec") if err != nil { @@ -71,10 +74,25 @@ func (h *Handler) Validate() error { // ServeHTTP is the Caddy handler for serving HTTP requests func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { - err := h.crowdsec.CheckRequest(r.Context(), r) - if err != nil { - // TODO: do something with the error - // TODO: add (debug) logging + if err := h.crowdsec.CheckRequest(r.Context(), r); err != nil { + a := &bouncer.AppSecError{} + if !errors.As(err, &a) { + return err + } + + ip, err := utils.DetermineIPFromRequest(r) + if err != nil { + return err // TODO: return error here? Or just log it and continue serving + } + + switch a.Action { + case "allow": + // nothing to do + case "log": + h.logger.Info("appsec rule triggered", zap.String("ip", ip.String()), zap.String("action", a.Action)) + default: + return utils.WriteResponse(w, h.logger, a.Action, ip.String(), a.Duration, a.StatusCode) + } } // Continue down the handler stack diff --git a/crowdsec/caddyfile.go b/crowdsec/caddyfile.go index 53d34aaf..a9e2e505 100644 --- a/crowdsec/caddyfile.go +++ b/crowdsec/caddyfile.go @@ -70,6 +70,11 @@ func parseCrowdSec(d *caddyfile.Dispenser, existingVal any) (any, error) { return nil, d.ArgErr() } cs.EnableHardFails = &tv + case "appsec_url": + if !d.NextArg() { + return nil, d.ArgErr() + } + cs.AppSecUrl = d.Val() default: return nil, d.Errf("invalid configuration token provided: %s", d.Val()) } diff --git a/http/http.go b/http/http.go index bc295ffc..a81a82bc 100644 --- a/http/http.go +++ b/http/http.go @@ -18,7 +18,6 @@ import ( "errors" "fmt" "net/http" - "time" "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" @@ -28,6 +27,7 @@ import ( _ "github.com/hslatman/caddy-crowdsec-bouncer/appsec" // include support for AppSec WAF component "github.com/hslatman/caddy-crowdsec-bouncer/crowdsec" + "github.com/hslatman/caddy-crowdsec-bouncer/internal/bouncer" "github.com/hslatman/caddy-crowdsec-bouncer/internal/utils" ) @@ -38,9 +38,8 @@ func init() { // Handler matches request IPs to CrowdSec decisions to (dis)allow access type Handler struct { - logger *zap.Logger - crowdsec *crowdsec.CrowdSec - + logger *zap.Logger + crowdsec *crowdsec.CrowdSec appsecEnabled bool } @@ -60,7 +59,7 @@ func (h *Handler) Provision(ctx caddy.Context) error { } h.crowdsec = crowdsecAppIface.(*crowdsec.CrowdSec) - h.appsecEnabled = true // TODO: move to provisioning; unmarshaling of Caddyfile; etc + h.appsecEnabled = true // TODO: make configurable h.logger = ctx.Logger(h) defer h.logger.Sync() // nolint @@ -80,42 +79,43 @@ func (h *Handler) Validate() error { // ServeHTTP is the Caddy handler for serving HTTP requests func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { - ipToCheck, err := utils.DetermineIPFromRequest(r) + ip, err := utils.DetermineIPFromRequest(r) if err != nil { return err // TODO: return error here? Or just log it and continue serving } - isAllowed, decision, err := h.crowdsec.IsAllowed(ipToCheck) + isAllowed, decision, err := h.crowdsec.IsAllowed(ip) if err != nil { return err // TODO: return error here? Or just log it and continue serving } + // TODO: if the IP is allowed, should we (temporarily) put it in an explicit allowlist for quicker check? + if !isAllowed { // TODO: maybe some configuration to override the type of action with a ban, some default, something like that? // TODO: can we provide the reason for the response to the Caddy logger, like the CrowdSec type, duration, etc. typ := *decision.Type - switch typ { - case "ban": - h.logger.Debug(fmt.Sprintf("serving ban response to %s", *decision.Value)) - return writeBanResponse(w) - case "captcha": - h.logger.Debug(fmt.Sprintf("serving captcha (ban) response to %s", *decision.Value)) - return writeCaptchaResponse(w) - case "throttle": - h.logger.Debug(fmt.Sprintf("serving throttle response to %s", *decision.Value)) - return writeThrottleResponse(w, *decision.Duration) - default: - h.logger.Warn(fmt.Sprintf("got crowdsec decision type: %s", typ)) - h.logger.Debug(fmt.Sprintf("serving ban response to %s", *decision.Value)) - return writeBanResponse(w) - } - } + value := *decision.Value + duration := *decision.Duration - // TODO: if the IP is allowed, should we (temporarily) put it in an explicit allowlist for quicker check? + return utils.WriteResponse(w, h.logger, typ, value, duration, 0) + } if h.appsecEnabled { if err := h.crowdsec.CheckRequest(r.Context(), r); err != nil { - // TODO: do something with the error + a := &bouncer.AppSecError{} + if !errors.As(err, &a) { + return err + } + + switch a.Action { + case "allow": + // nothing to do + case "log": + h.logger.Info("appsec rule triggered", zap.String("ip", ip.String()), zap.String("action", a.Action)) + default: + return utils.WriteResponse(w, h.logger, a.Action, ip.String(), a.Duration, a.StatusCode) + } } } @@ -127,34 +127,6 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhtt return nil } -// writeBanResponse writes a 403 status as response -func writeBanResponse(w http.ResponseWriter) error { - w.WriteHeader(http.StatusForbidden) - return nil -} - -// writeCaptchaResponse (currently) writes a 403 status as response -func writeCaptchaResponse(w http.ResponseWriter) error { - // TODO: implement showing a captcha in some way. How? hCaptcha? And how to handle afterwards? - return writeBanResponse(w) -} - -// writeThrottleResponse writes 429 status as response -func writeThrottleResponse(w http.ResponseWriter, duration string) error { - - d, err := time.ParseDuration(duration) - if err != nil { - return err - } - - // TODO: round this to the nearest multiple of the ticker interval? and/or include the time the decision was processed from stream vs. request time? - retryAfter := fmt.Sprintf("%.0f", d.Seconds()) - w.Header().Add("Retry-After", retryAfter) - w.WriteHeader(http.StatusTooManyRequests) - - return nil -} - // UnmarshalCaddyfile implements caddyfile.Unmarshaler. func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { // TODO: parse additional handler directives (none exist now) diff --git a/internal/bouncer/appsec.go b/internal/bouncer/appsec.go index bab91aa5..37d86e15 100644 --- a/internal/bouncer/appsec.go +++ b/internal/bouncer/appsec.go @@ -3,39 +3,44 @@ package bouncer import ( "bytes" "context" + "encoding/json" "errors" - "fmt" "io" "net/http" "time" "github.com/hslatman/caddy-crowdsec-bouncer/internal/utils" + "go.uber.org/zap" ) type appsec struct { apiURL string apiKey string + logger *zap.Logger client *http.Client } -func newAppSec(apiURL, apiKey string) *appsec { +func newAppSec(apiURL, apiKey string, logger *zap.Logger) *appsec { return &appsec{ apiURL: apiURL, apiKey: apiKey, + logger: logger, client: &http.Client{ Timeout: 10 * time.Second, }, } } +type appsecResponse struct { + Action string `json:"action"` + StatusCode int `json:"http_status"` +} + func (a *appsec) checkRequest(ctx context.Context, r *http.Request) error { if a.apiURL == "" { - return nil // AppSec component not enabled + return nil // AppSec component not enabled; skip check } - // TODO: add (debug) logging - // TODO: return a decision, and act on it in the handler (named/typed error) - originalIP, err := utils.DetermineIPFromRequest(r) if err != nil { return err // TODO: return error here? Or just log it and continue serving @@ -50,7 +55,7 @@ func (a *appsec) checkRequest(ctx context.Context, r *http.Request) error { var body io.ReadCloser = http.NoBody if len(originalBody) > 0 { method = http.MethodPost - body = io.NopCloser(bytes.NewBuffer(originalBody)) + body = io.NopCloser(bytes.NewBuffer(originalBody)) // TODO: reuse buffers? } r.Body = io.NopCloser(bytes.NewBuffer(originalBody)) @@ -65,21 +70,46 @@ func (a *appsec) checkRequest(ctx context.Context, r *http.Request) error { req.Header.Set("X-Crowdsec-Appsec-Verb", r.Method) req.Header.Set("X-Crowdsec-Appsec-Api-Key", a.apiKey) + totalAppSecCalls.Inc() resp, err := a.client.Do(req) + if err != nil { + totalAppSecErrors.Inc() + return err + } + defer resp.Body.Close() + + responseBody, err := io.ReadAll(resp.Body) if err != nil { return err } switch resp.StatusCode { case 200: - return nil // TODO: read decision from body? + return nil case 401: - return errors.New("not authenticated") + a.logger.Error("appsec component not authenticated", zap.String("appsec_url", a.apiURL)) + return nil // this fails open, currently; make it fail hard if configured to do so? case 403: - return errors.New("not allowed") // TODO: read decision + var r appsecResponse + if err := json.Unmarshal(responseBody, &r); err != nil { + return err + } + + return &AppSecError{Err: errors.New("appsec rule triggered"), Action: r.Action, Duration: "", StatusCode: r.StatusCode} case 500: - return errors.New("internal error") + a.logger.Error("appsec component internal error", zap.String("appsec_url", a.apiURL)) + return nil // this fails open, currently; make it fail hard if configured to do so? default: - return fmt.Errorf("unsupported status code %d", resp.StatusCode) + a.logger.Warn("appsec component returned unsupported status", zap.String("code", resp.Status)) + return nil + } +} + +func (b *Bouncer) logAppSecStatus() { + if b.appsec.apiURL == "" { + b.logger.Info("appsec disabled") + return } + + b.logger.Info("appsec enabled") } diff --git a/internal/bouncer/bouncer.go b/internal/bouncer/bouncer.go index 213d1bd6..acfa5b6a 100644 --- a/internal/bouncer/bouncer.go +++ b/internal/bouncer/bouncer.go @@ -88,7 +88,7 @@ func New(apiKey, apiURL, appSecURL, tickerInterval string, logger *zap.Logger) ( InsecureSkipVerify: &insecureSkipVerify, UserAgent: userAgent, }, - appsec: newAppSec(appSecURL, apiKey), + appsec: newAppSec(appSecURL, apiKey, logger.Named("appsec")), store: newStore(), logger: logger, instantiatedAt: instantiatedAt, @@ -129,6 +129,8 @@ func (b *Bouncer) Init() (err error) { return err } + b.logAppSecStatus() + return nil } @@ -142,6 +144,8 @@ func (b *Bouncer) Init() (err error) { return err } + b.logAppSecStatus() + return nil } diff --git a/internal/bouncer/decisions.go b/internal/bouncer/decisions.go index af2709d6..6865d449 100644 --- a/internal/bouncer/decisions.go +++ b/internal/bouncer/decisions.go @@ -100,18 +100,22 @@ func (b *Bouncer) retrieveDecision(ip netip.Addr) (*models.Decision, error) { return b.store.get(ip) } + totalLAPICalls.Inc() // increment; not built into liveBouncer decision, err := b.liveBouncer.Get(ip.String()) if err != nil { + totalLAPIErrors.Inc() // increment; not built into liveBouncer fields := []zapcore.Field{ b.zapField(), zap.String("address", b.liveBouncer.APIUrl), zap.Error(err), } + if b.shouldFailHard { b.logger.Fatal(err.Error(), fields...) } else { b.logger.Error(err.Error(), fields...) } + return nil, nil // when not failing hard, we return no error } diff --git a/internal/bouncer/errors.go b/internal/bouncer/errors.go new file mode 100644 index 00000000..4ec05cd5 --- /dev/null +++ b/internal/bouncer/errors.go @@ -0,0 +1,12 @@ +package bouncer + +type AppSecError struct { + Err error + Action string + Duration string + StatusCode int +} + +func (a AppSecError) Error() string { + return a.Err.Error() +} diff --git a/internal/bouncer/metrics.go b/internal/bouncer/metrics.go index 8ed72f7b..d73aa83c 100644 --- a/internal/bouncer/metrics.go +++ b/internal/bouncer/metrics.go @@ -9,9 +9,26 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/models" csbouncer "github.com/crowdsecurity/go-cs-bouncer" "github.com/crowdsecurity/go-cs-lib/ptr" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" ) +var ( + // metrics provided by the go-cs-bouncer package + totalLAPICalls = csbouncer.TotalLAPICalls + totalLAPIErrors = csbouncer.TotalLAPIError + + // appsec metrics + totalAppSecCalls = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "lapi_appsec_requests_total", + Help: "The total number of calls to CrowdSec LAPI AppSec component", + }) + totalAppSecErrors = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "lapi_appsec_requests_failures_total", + Help: "The total number of failed calls to CrowdSec LAPI AppSec component", + }) +) + func newMetricsProvider(client *apiclient.ApiClient, updater csbouncer.MetricsUpdater, interval time.Duration) (*csbouncer.MetricsProvider, error) { m, err := csbouncer.NewMetricsProvider( client, diff --git a/internal/utils/http.go b/internal/utils/http.go index b7be55a5..cc9584ed 100644 --- a/internal/utils/http.go +++ b/internal/utils/http.go @@ -5,8 +5,10 @@ import ( "fmt" "net/http" "net/netip" + "time" "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "go.uber.org/zap" ) // DetermineIPFromRequest returns the IP of the client based on the value that @@ -37,3 +39,55 @@ func DetermineIPFromRequest(r *http.Request) (netip.Addr, error) { return ip, nil } + +// WriteResponse writes a response to the [http.ResponseWriter] based on the typ, value, +// duration and status code provide. +func WriteResponse(w http.ResponseWriter, logger *zap.Logger, typ, value, duration string, statusCode int) error { + switch typ { + case "ban": + logger.Debug(fmt.Sprintf("serving ban response to %s", value)) + return writeBanResponse(w, statusCode) + case "captcha": + logger.Debug(fmt.Sprintf("serving captcha (ban) response to %s", value)) + return writeCaptchaResponse(w, statusCode) + case "throttle": + logger.Debug(fmt.Sprintf("serving throttle response to %s", value)) + return writeThrottleResponse(w, duration) + default: + logger.Warn(fmt.Sprintf("got crowdsec decision type: %s", typ)) + logger.Debug(fmt.Sprintf("serving ban response to %s", value)) + return writeBanResponse(w, statusCode) + } +} + +// writeBanResponse writes a 403 status as response +func writeBanResponse(w http.ResponseWriter, statusCode int) error { + code := statusCode + if code <= 0 { + code = http.StatusForbidden + } + + w.WriteHeader(code) + return nil +} + +// writeCaptchaResponse (currently) writes a 403 status as response +func writeCaptchaResponse(w http.ResponseWriter, statusCode int) error { + // TODO: implement showing a captcha in some way. How? hCaptcha? And how to handle afterwards? + return writeBanResponse(w, statusCode) +} + +// writeThrottleResponse writes 429 status as response +func writeThrottleResponse(w http.ResponseWriter, duration string) error { + d, err := time.ParseDuration(duration) + if err != nil { + return err + } + + // TODO: round this to the nearest multiple of the ticker interval? and/or include the time the decision was processed from stream vs. request time? + retryAfter := fmt.Sprintf("%.0f", d.Seconds()) + w.Header().Add("Retry-After", retryAfter) + w.WriteHeader(http.StatusTooManyRequests) + + return nil +} From 082b2264776041afbe913e7047ab356f7689f671 Mon Sep 17 00:00:00 2001 From: Herman Slatman Date: Mon, 18 Nov 2024 00:05:46 +0100 Subject: [PATCH 03/11] Reduce some duplicate logic supporting both HTTP and AppSec --- appsec/appsec.go | 38 ++++++---- crowdsec/caddyfile.go | 2 +- crowdsec/crowdsec.go | 32 +++++---- go.mod | 3 +- go.sum | 4 +- http/http.go | 61 ++++++---------- internal/bouncer/appsec.go | 32 +++++++-- internal/bouncer/bouncer.go | 6 +- internal/httputils/context.go | 58 ++++++++++++++++ internal/httputils/context_test.go | 81 ++++++++++++++++++++++ internal/{utils => httputils}/http.go | 20 +++++- internal/{utils => httputils}/http_test.go | 4 +- layer4/l4.go | 12 ++-- 13 files changed, 267 insertions(+), 86 deletions(-) create mode 100644 internal/httputils/context.go create mode 100644 internal/httputils/context_test.go rename internal/{utils => httputils}/http.go (79%) rename internal/{utils => httputils}/http_test.go (97%) diff --git a/appsec/appsec.go b/appsec/appsec.go index b7bc83ca..a4378e32 100644 --- a/appsec/appsec.go +++ b/appsec/appsec.go @@ -18,15 +18,17 @@ import ( "errors" "fmt" "net/http" + "net/netip" "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" "github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile" "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "go.uber.org/zap" + "github.com/hslatman/caddy-crowdsec-bouncer/crowdsec" "github.com/hslatman/caddy-crowdsec-bouncer/internal/bouncer" - "github.com/hslatman/caddy-crowdsec-bouncer/internal/utils" - "go.uber.org/zap" + "github.com/hslatman/caddy-crowdsec-bouncer/internal/httputils" ) func init() { @@ -58,7 +60,6 @@ func (h *Handler) Provision(ctx caddy.Context) error { h.crowdsec = crowdsecAppIface.(*crowdsec.CrowdSec) h.logger = ctx.Logger(h) - defer h.logger.Sync() // nolint return nil } @@ -72,31 +73,39 @@ func (h *Handler) Validate() error { return nil } -// ServeHTTP is the Caddy handler for serving HTTP requests -func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { - if err := h.crowdsec.CheckRequest(r.Context(), r); err != nil { +// Cleanup cleans up resources when the module is being stopped. +func (h *Handler) Cleanup() error { + h.logger.Sync() // nolint + + return nil +} + +// ServeHTTP is the Caddy handler for serving HTTP requests. +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { + var ( + ctx = r.Context() + ip netip.Addr + ) + + ctx, ip = httputils.EnsureIP(ctx, r) + if err := h.crowdsec.CheckRequest(ctx, r); err != nil { a := &bouncer.AppSecError{} if !errors.As(err, &a) { return err } - ip, err := utils.DetermineIPFromRequest(r) - if err != nil { - return err // TODO: return error here? Or just log it and continue serving - } - switch a.Action { case "allow": // nothing to do case "log": h.logger.Info("appsec rule triggered", zap.String("ip", ip.String()), zap.String("action", a.Action)) default: - return utils.WriteResponse(w, h.logger, a.Action, ip.String(), a.Duration, a.StatusCode) + return httputils.WriteResponse(w, h.logger, a.Action, ip.String(), a.Duration, a.StatusCode) } } // Continue down the handler stack - if err := next.ServeHTTP(w, r); err != nil { + if err := next.ServeHTTP(w, r.WithContext(ctx)); err != nil { return err } @@ -105,7 +114,6 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhtt // UnmarshalCaddyfile implements caddyfile.Unmarshaler. func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { - // TODO: parse additional handler directives (none exist now) return nil } @@ -113,7 +121,7 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { func parseCaddyfileHandlerDirective(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) { var handler Handler err := handler.UnmarshalCaddyfile(h.Dispenser) - return handler, err + return &handler, err } // Interface guards diff --git a/crowdsec/caddyfile.go b/crowdsec/caddyfile.go index a9e2e505..56973648 100644 --- a/crowdsec/caddyfile.go +++ b/crowdsec/caddyfile.go @@ -76,7 +76,7 @@ func parseCrowdSec(d *caddyfile.Dispenser, existingVal any) (any, error) { } cs.AppSecUrl = d.Val() default: - return nil, d.Errf("invalid configuration token provided: %s", d.Val()) + return nil, d.Errf("invalid configuration token %q provided", d.Val()) } } diff --git a/crowdsec/crowdsec.go b/crowdsec/crowdsec.go index e5f91249..be626f02 100644 --- a/crowdsec/crowdsec.go +++ b/crowdsec/crowdsec.go @@ -51,9 +51,9 @@ func (CrowdSec) CaddyModule() caddy.ModuleInfo { // which can be used by the HTTP handler and Layer4 matcher to decide if // a request or connection is allowed or not. type CrowdSec struct { - // APIUrl for the CrowdSec Local API. Defaults to http://127.0.0.1:8080/ + // APIUrl for the CrowdSec Local API. Defaults to http://127.0.0.1:8080/. APIUrl string `json:"api_url,omitempty"` - // APIKey for the CrowdSec Local API + // APIKey for the CrowdSec Local API. APIKey string `json:"api_key"` // TickerInterval is the interval the StreamBouncer uses for querying // the CrowdSec Local API. Defaults to "60s". @@ -69,8 +69,9 @@ type CrowdSec struct { // Caddy continuing operation (with a chance of not performing) // validations. Defaults to false. EnableHardFails *bool `json:"enable_hard_fails,omitempty"` - - AppSecUrl string `json:"appsec_url,omitempty"` // TODO: documentation + // AppSecUrl is the URL of the AppSec component served by your + // CrowdSec installation. Disabled by default. + AppSecUrl string `json:"appsec_url,omitempty"` ctx caddy.Context logger *zap.Logger @@ -130,11 +131,12 @@ func (c *CrowdSec) Validate() error { } const ( - handlerName = "http.handlers.crowdsec" - matcherName = "layer4.matchers.crowdsec" + appSecHandlerName = "http.handlers.appsec" + httpHandlerName = "http.handlers.crowdsec" + matcherName = "layer4.matchers.crowdsec" ) -var crowdSecModules = []string{handlerName, matcherName} +var crowdSecModules = []string{httpHandlerName, appSecHandlerName, matcherName} func (c *CrowdSec) checkModules() error { modules, err := matchModules(crowdSecModules...) @@ -150,13 +152,13 @@ func (c *CrowdSec) checkModules() error { hasLayer4 := len(layer4) > 0 switch { case hasLayer4 && len(modules) == 0: - c.logger.Warn(fmt.Sprintf("%s and %s modules are not available", handlerName, matcherName)) - case hasLayer4 && hasModule(modules, matcherName) && !hasModule(modules, handlerName): - c.logger.Warn(fmt.Sprintf("%s module is not available", handlerName)) - case hasLayer4 && hasModule(modules, handlerName) && !hasModule(modules, matcherName): + c.logger.Warn(fmt.Sprintf("%s and %s modules are not available", httpHandlerName, matcherName)) + case hasLayer4 && hasModule(modules, matcherName) && !hasModule(modules, httpHandlerName): + c.logger.Warn(fmt.Sprintf("%s module is not available", httpHandlerName)) + case hasLayer4 && hasModule(modules, httpHandlerName) && !hasModule(modules, matcherName): c.logger.Warn(fmt.Sprintf("%s module is not available", matcherName)) case len(modules) == 0: - c.logger.Warn(fmt.Sprintf("%s module is not available", handlerName)) + c.logger.Warn(fmt.Sprintf("%s module is not available", httpHandlerName)) } return nil @@ -231,6 +233,8 @@ func (c *CrowdSec) Cleanup() error { return fmt.Errorf("failed cleaning up: %w", err) } + c.logger.Sync() // nolint + return nil } @@ -251,12 +255,12 @@ func (c *CrowdSec) Stop() error { } // IsAllowed is used by the CrowdSec HTTP handler to check if -// an IP is allowed to perform a request +// an IP is allowed to perform a request. func (c *CrowdSec) IsAllowed(ip netip.Addr) (bool, *models.Decision, error) { - // TODO: check if running? fully loaded, etc? return c.bouncer.IsAllowed(ip) } +// CheckRequest checks the incoming request against AppSec. func (c *CrowdSec) CheckRequest(ctx context.Context, r *http.Request) error { return c.bouncer.CheckRequest(ctx, r) } diff --git a/go.mod b/go.mod index cc0ee66a..049d9e76 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,8 @@ require ( github.com/hslatman/ipstore v0.3.0 github.com/jarcoal/httpmock v1.3.1 github.com/mholt/caddy-l4 v0.0.0-20231016112149-a362a1fbf652 + github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c + github.com/prometheus/client_golang v1.20.4 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.9.0 go.uber.org/goleak v1.2.1 @@ -104,7 +106,6 @@ require ( github.com/onsi/ginkgo/v2 v2.13.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.20.4 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.59.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect diff --git a/go.sum b/go.sum index d802c372..f6c7c3e6 100644 --- a/go.sum +++ b/go.sum @@ -201,8 +201,6 @@ github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB github.com/gorilla/mux v1.4.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/groob/finalizer v0.0.0-20170707115354-4c2ed49aabda/go.mod h1:MyndkAZd5rUMdNogn35MWXBX1UiBigrU8eTj8DoAC2c= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hslatman/ipstore v0.2.1-0.20241003102639-77b98e171659 h1:kkKqw+NR37yM2LSz2n4KDrk7euiWKDxW7Uy2okVpv98= -github.com/hslatman/ipstore v0.2.1-0.20241003102639-77b98e171659/go.mod h1:fUg+lu09+OKKllPSRSvE6OdJ8AZB4sAjHxqW6QChpmU= github.com/hslatman/ipstore v0.3.0 h1:3lUtYZMDGRdDePFFL2wUIrpHqMsqzJEluDnQwt43cfs= github.com/hslatman/ipstore v0.3.0/go.mod h1:fUg+lu09+OKKllPSRSvE6OdJ8AZB4sAjHxqW6QChpmU= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= @@ -346,6 +344,8 @@ github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4 github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw= +github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv/v3 v3.0.1 h1:x06SQA46+PKIUftmEujdwSEpIx8kR+M9eLYsUxeYveU= github.com/peterbourgon/diskv/v3 v3.0.1/go.mod h1:kJ5Ny7vLdARGU3WUuy6uzO6T0nb/2gWcT1JiBvRmb5o= diff --git a/http/http.go b/http/http.go index a81a82bc..fe368a10 100644 --- a/http/http.go +++ b/http/http.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "net/http" + "net/netip" "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" @@ -27,8 +28,7 @@ import ( _ "github.com/hslatman/caddy-crowdsec-bouncer/appsec" // include support for AppSec WAF component "github.com/hslatman/caddy-crowdsec-bouncer/crowdsec" - "github.com/hslatman/caddy-crowdsec-bouncer/internal/bouncer" - "github.com/hslatman/caddy-crowdsec-bouncer/internal/utils" + "github.com/hslatman/caddy-crowdsec-bouncer/internal/httputils" ) func init() { @@ -36,11 +36,10 @@ func init() { httpcaddyfile.RegisterHandlerDirective("crowdsec", parseCaddyfileHandlerDirective) } -// Handler matches request IPs to CrowdSec decisions to (dis)allow access +// Handler matches request IPs to CrowdSec decisions to (dis)allow access. type Handler struct { - logger *zap.Logger - crowdsec *crowdsec.CrowdSec - appsecEnabled bool + logger *zap.Logger + crowdsec *crowdsec.CrowdSec } // CaddyModule returns the Caddy module information. @@ -59,17 +58,13 @@ func (h *Handler) Provision(ctx caddy.Context) error { } h.crowdsec = crowdsecAppIface.(*crowdsec.CrowdSec) - h.appsecEnabled = true // TODO: make configurable - h.logger = ctx.Logger(h) - defer h.logger.Sync() // nolint return nil } // Validate ensures the app's configuration is valid. func (h *Handler) Validate() error { - if h.crowdsec == nil { return errors.New("crowdsec app not available") } @@ -77,13 +72,21 @@ func (h *Handler) Validate() error { return nil } -// ServeHTTP is the Caddy handler for serving HTTP requests -func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { - ip, err := utils.DetermineIPFromRequest(r) - if err != nil { - return err // TODO: return error here? Or just log it and continue serving - } +// Cleanup cleans up resources when the module is being stopped. +func (h *Handler) Cleanup() error { + h.logger.Sync() // nolint + return nil +} + +// ServeHTTP is the Caddy handler for serving HTTP requests. +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { + var ( + ctx = r.Context() + ip netip.Addr + ) + + ctx, ip = httputils.EnsureIP(ctx, r) isAllowed, decision, err := h.crowdsec.IsAllowed(ip) if err != nil { return err // TODO: return error here? Or just log it and continue serving @@ -98,29 +101,11 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhtt value := *decision.Value duration := *decision.Duration - return utils.WriteResponse(w, h.logger, typ, value, duration, 0) - } - - if h.appsecEnabled { - if err := h.crowdsec.CheckRequest(r.Context(), r); err != nil { - a := &bouncer.AppSecError{} - if !errors.As(err, &a) { - return err - } - - switch a.Action { - case "allow": - // nothing to do - case "log": - h.logger.Info("appsec rule triggered", zap.String("ip", ip.String()), zap.String("action", a.Action)) - default: - return utils.WriteResponse(w, h.logger, a.Action, ip.String(), a.Duration, a.StatusCode) - } - } + return httputils.WriteResponse(w, h.logger, typ, value, duration, 0) } // Continue down the handler stack - if err := next.ServeHTTP(w, r); err != nil { + if err := next.ServeHTTP(w, r.WithContext(ctx)); err != nil { return err } @@ -129,7 +114,6 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhtt // UnmarshalCaddyfile implements caddyfile.Unmarshaler. func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { - // TODO: parse additional handler directives (none exist now) return nil } @@ -137,7 +121,7 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { func parseCaddyfileHandlerDirective(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) { var handler Handler err := handler.UnmarshalCaddyfile(h.Dispenser) - return handler, err + return &handler, err } // Interface guards @@ -147,4 +131,5 @@ var ( _ caddy.Validator = (*Handler)(nil) _ caddyhttp.MiddlewareHandler = (*Handler)(nil) _ caddyfile.Unmarshaler = (*Handler)(nil) + _ caddy.CleanerUpper = (*Handler)(nil) ) diff --git a/internal/bouncer/appsec.go b/internal/bouncer/appsec.go index 37d86e15..3acfc24e 100644 --- a/internal/bouncer/appsec.go +++ b/internal/bouncer/appsec.go @@ -6,11 +6,14 @@ import ( "encoding/json" "errors" "io" + "net" "net/http" "time" - "github.com/hslatman/caddy-crowdsec-bouncer/internal/utils" + "github.com/oxtoacart/bpool" "go.uber.org/zap" + + "github.com/hslatman/caddy-crowdsec-bouncer/internal/httputils" ) type appsec struct { @@ -18,6 +21,7 @@ type appsec struct { apiKey string logger *zap.Logger client *http.Client + pool *bpool.BufferPool } func newAppSec(apiURL, apiKey string, logger *zap.Logger) *appsec { @@ -27,7 +31,20 @@ func newAppSec(apiURL, apiKey string, logger *zap.Logger) *appsec { logger: logger, client: &http.Client{ Timeout: 10 * time.Second, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 60 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, }, + pool: bpool.NewBufferPool(64), } } @@ -41,9 +58,9 @@ func (a *appsec) checkRequest(ctx context.Context, r *http.Request) error { return nil // AppSec component not enabled; skip check } - originalIP, err := utils.DetermineIPFromRequest(r) - if err != nil { - return err // TODO: return error here? Or just log it and continue serving + originalIP, ok := httputils.FromContext(ctx) + if !ok { + return errors.New("could not retrieve netip.Addr from context") } originalBody, err := io.ReadAll(r.Body) @@ -55,7 +72,12 @@ func (a *appsec) checkRequest(ctx context.Context, r *http.Request) error { var body io.ReadCloser = http.NoBody if len(originalBody) > 0 { method = http.MethodPost - body = io.NopCloser(bytes.NewBuffer(originalBody)) // TODO: reuse buffers? + + buffer := a.pool.Get() + defer a.pool.Put(buffer) + + _, _ = buffer.Write(originalBody) + body = io.NopCloser(buffer) } r.Body = io.NopCloser(bytes.NewBuffer(originalBody)) diff --git a/internal/bouncer/bouncer.go b/internal/bouncer/bouncer.go index acfa5b6a..0fde7839 100644 --- a/internal/bouncer/bouncer.go +++ b/internal/bouncer/bouncer.go @@ -17,6 +17,7 @@ package bouncer import ( "context" "encoding/hex" + "errors" "fmt" "math/rand" "net/http" @@ -63,7 +64,6 @@ type Bouncer struct { } // New creates a new (streaming) Bouncer with a storage based on immutable radix tree -// TODO: take a configuration struct instead, because more options will be added. func New(apiKey, apiURL, appSecURL, tickerInterval string, logger *zap.Logger) (*Bouncer, error) { userAgent := fmt.Sprintf("%s/%s", userAgentName, userAgentVersion) insecureSkipVerify := false @@ -211,6 +211,10 @@ func (b *Bouncer) Shutdown() error { func (b *Bouncer) IsAllowed(ip netip.Addr) (bool, *models.Decision, error) { // TODO: perform lookup in explicit allowlist as a kind of quick lookup in front of the CrowdSec lookup list? isAllowed := false + if !ip.IsValid() { + return isAllowed, nil, errors.New("could not obtain netip.Addr from request") // fail closed + } + decision, err := b.retrieveDecision(ip) if err != nil { return isAllowed, nil, err // fail closed diff --git a/internal/httputils/context.go b/internal/httputils/context.go new file mode 100644 index 00000000..07b4b7b0 --- /dev/null +++ b/internal/httputils/context.go @@ -0,0 +1,58 @@ +// Copyright 2024 Herman Slatman +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httputils + +import ( + "context" + "net/http" + "net/netip" +) + +type contextKey struct{} + +func EnsureIP(ctx context.Context, r *http.Request) (context.Context, netip.Addr) { // TODO: pass in ctx only? + var ( + ip netip.Addr + err error + ) + + ip, ok := FromContext(ctx) + if !ok { + if ip, err = determineIPFromRequest(r); err != nil { // TODO: pass in ctx only? + ip = netip.Addr{} + } + + ctx = newContext(ctx, ip) + } + + return ctx, ip +} + +func newContext(ctx context.Context, ip netip.Addr) context.Context { + return context.WithValue(ctx, contextKey{}, ip) +} + +func FromContext(ctx context.Context) (netip.Addr, bool) { + v, ok := ctx.Value(contextKey{}).(netip.Addr) + if !ok { + return netip.Addr{}, false + } + + if !v.IsValid() { + return v, false + } + + return v, true +} diff --git a/internal/httputils/context_test.go b/internal/httputils/context_test.go new file mode 100644 index 00000000..8d403db7 --- /dev/null +++ b/internal/httputils/context_test.go @@ -0,0 +1,81 @@ +// Copyright 2024 Herman Slatman +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httputils + +import ( + "context" + "net/http" + "net/http/httptest" + "net/netip" + "testing" + + "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "github.com/stretchr/testify/assert" +) + +func TestEnsureIP(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "/caddy", http.NoBody) + ipFromRequest := newCaddyVarsContext() + caddyhttp.SetVar(ipFromRequest, caddyhttp.ClientIPVarKey, "127.0.0.1") + ipFromContext := newCaddyVarsContext() + caddyhttp.SetVar(ipFromContext, caddyhttp.ClientIPVarKey, "127.0.0.2") + ipFromContext = newContext(ipFromContext, netip.MustParseAddr("127.0.0.3")) + invalidIPCtx := newCaddyVarsContext() + caddyhttp.SetVar(invalidIPCtx, caddyhttp.ClientIPVarKey, "127.0.0.1.x") + + type args struct { + ctx context.Context + r *http.Request + } + tests := []struct { + name string + args args + wantCtx context.Context + wantIP netip.Addr + }{ + { + name: "ip-from-request", + args: args{ + r: r.WithContext(ipFromRequest), + ctx: ipFromRequest, + }, + wantIP: netip.MustParseAddr("127.0.0.1"), + }, + { + name: "ip-from-context", + args: args{ + r: r.WithContext(ipFromContext), + ctx: ipFromContext, + }, + wantIP: netip.MustParseAddr("127.0.0.3"), + }, + { + name: "invalid-ip", + args: args{ + r: r.WithContext(invalidIPCtx), + ctx: invalidIPCtx, + }, + wantIP: netip.Addr{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, ip := EnsureIP(tt.args.ctx, tt.args.r) + + assert.Equal(t, tt.wantIP, ip) + assert.Equal(t, tt.wantIP, ctx.Value(contextKey{}).(netip.Addr)) + }) + } +} diff --git a/internal/utils/http.go b/internal/httputils/http.go similarity index 79% rename from internal/utils/http.go rename to internal/httputils/http.go index cc9584ed..6873c6b5 100644 --- a/internal/utils/http.go +++ b/internal/httputils/http.go @@ -1,4 +1,18 @@ -package utils +// Copyright 2024 Herman Slatman +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httputils import ( "errors" @@ -11,11 +25,11 @@ import ( "go.uber.org/zap" ) -// DetermineIPFromRequest returns the IP of the client based on the value that +// determineIPFromRequest returns the IP of the client based on the value that // Caddy extracts from the original request and stores in the request context. // Support for setting the real client IP in case a proxy sits in front of // Caddy was added, so the client IP reported here is the actual client IP. -func DetermineIPFromRequest(r *http.Request) (netip.Addr, error) { +func determineIPFromRequest(r *http.Request) (netip.Addr, error) { zero := netip.Addr{} clientIPVar := caddyhttp.GetVar(r.Context(), caddyhttp.ClientIPVarKey) if clientIPVar == nil { diff --git a/internal/utils/http_test.go b/internal/httputils/http_test.go similarity index 97% rename from internal/utils/http_test.go rename to internal/httputils/http_test.go index 6c84007d..af6ffa0e 100644 --- a/internal/utils/http_test.go +++ b/internal/httputils/http_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package utils +package httputils import ( "context" @@ -58,7 +58,7 @@ func Test_determineIPFromRequest(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := DetermineIPFromRequest(tt.args.r) + got, err := determineIPFromRequest(tt.args.r) if (err != nil) != tt.wantErr { t.Errorf("determineIPFromRequest() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/layer4/l4.go b/layer4/l4.go index 702404bc..769ddbd8 100644 --- a/layer4/l4.go +++ b/layer4/l4.go @@ -21,9 +21,10 @@ import ( "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" - "github.com/hslatman/caddy-crowdsec-bouncer/crowdsec" l4 "github.com/mholt/caddy-l4/layer4" "go.uber.org/zap" + + "github.com/hslatman/caddy-crowdsec-bouncer/crowdsec" ) func init() { @@ -53,7 +54,6 @@ func (m *Matcher) Provision(ctx caddy.Context) error { m.crowdsec = crowdsecAppIface.(*crowdsec.CrowdSec) m.logger = ctx.Logger(m) - defer m.logger.Sync() // nolint return nil } @@ -67,9 +67,7 @@ func (m *Matcher) Validate() error { // not denied according to CrowdSec decisions stored in the // CrowdSec app module. func (m Matcher) Match(cx *l4.Connection) (bool, error) { - // TODO: needs to be tested with TCP as well as UDP. - clientIP, err := m.getClientIP(cx) if err != nil { return false, err @@ -88,6 +86,12 @@ func (m Matcher) Match(cx *l4.Connection) (bool, error) { return true, nil } +func (m *Matcher) Cleanup() error { + m.logger.Sync() // nolint + + return nil +} + // getClientIP determines the IP of the client connecting // Implementation taken from github.com/mholt/caddy-l4/layer4/matchers.go func (m Matcher) getClientIP(cx *l4.Connection) (netip.Addr, error) { From c27081dbc14fc6d6f128c08223d0b541da22019e Mon Sep 17 00:00:00 2001 From: Herman Slatman Date: Mon, 18 Nov 2024 00:59:07 +0100 Subject: [PATCH 04/11] Update README.md with AppSec and Caddyfile support --- README.md | 166 ++++++++++++++++------------------------------ docker/Dockerfile | 1 + 2 files changed, 57 insertions(+), 110 deletions(-) diff --git a/README.md b/README.md index 1f33c696..d8cc1939 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,6 @@ A [Caddy](https://caddyserver.com/) module that blocks malicious traffic based o ## Description -__This repository is currently a WIP. Things may change a bit.__ - CrowdSec is a free and open source security automation tool that uses local logs and a set of scenarios to infer malicious intent. In addition to operating locally, an optional community integration is also available, through which crowd-sourced IP reputation lists are distributed. @@ -14,27 +12,33 @@ At its core is the CrowdSec Agent, which keeps track of all data and related sys Bouncers are pieces of software that perform specific actions based on the decisions of the Agent. This repository contains a custom CrowdSec Bouncer that can be embedded as a Caddy module. -It consists of the follwing three main pieces: +It consists of the following four main pieces: * A Caddy App -* A Caddy HTTP Handler +* A Caddy Bouncer HTTP Handler * A Caddy [Layer 4](https://github.com/mholt/caddy-l4) Connection Matcher +* A Caddy AppSec HTTP Handler The App is responsible for communicating with a CrowdSec Agent via the CrowdSec *Local API* and keeping track of the decisions of the Agent. -The HTTP Handler checks client IPs of incoming requests against the decisions stored by the App. -This way, multiple independent HTTP Handlers or Connection Matchers can use the storage exposed by the App. +The Bouncer HTTP Handler checks client IPs of incoming requests against the decisions stored by the App. +This way, multiple independent HTTP Handlers and Connection Matchers can use the storage exposed by the App. The App can be configured to use either the StreamBouncer, which gets decisions via a HTTP polling mechanism, or the LiveBouncer, which sends a request on every incoming HTTP request or Layer 4 connection setup. +The Layer 4 Connection Matcher matches TCP and UDP IP addresses against the CrowdSec *Local API*. +Finally, the AppSec HTTP Handler communicates with an AppSec component configured on your CrowdSec deployment, and will check incoming HTTP requests against the rulesets configured. ## Usage Get the module ```bash -# get the http handler +# get the CrowdSec Bouncer HTTP handler go get github.com/hslatman/caddy-crowdsec-bouncer/http -# get the layer4 connection matcher (only required if you need support for TCP/UDP level blocking) +# get the CrowdSec layer4 connection matcher (only required if you need support for TCP/UDP level blocking) go get github.com/hslatman/caddy-crowdsec-bouncer/layer4 + +# get the AppSec HTTP handler (only required if you want CrowdSec AppSec support) +go get github.com/hslatman/caddy-crowdsec-bouncer/appsec ``` Create a (custom) Caddy server (or use *xcaddy*) @@ -45,10 +49,12 @@ package main import ( cmd "github.com/caddyserver/caddy/v2/cmd" _ "github.com/caddyserver/caddy/v2/modules/standard" - // import the http handler + // import the bouncer HTTP handler _ "github.com/hslatman/caddy-crowdsec-bouncer/http" // import the layer4 matcher (in case you want to block connections to layer4 servers using CrowdSec) _ "github.com/hslatman/caddy-crowdsec-bouncer/layer4" + // import the appsec HTTP handler (in case you want to block requests using the CrowdSec AppSec component) + _ "github.com/hslatman/caddy-crowdsec-bouncer/appsec" ) func main() { @@ -56,114 +62,57 @@ func main() { } ``` +Configuration using a Caddyfile is supported for HTTP handlers and Layer 4 matchers. +You'll also need to use a recent version of Caddy (i.e. 2.7.3 and newer) and Go 1.20 (or newer). + Example Caddyfile: ``` { - debug - crowdsec { - api_url http://localhost:8080 - api_key - ticker_interval 15s - #disable_streaming - #enable_hard_fails + debug + + crowdsec { + api_url http://localhost:8080 + api_key + ticker_interval 15s + appsec_url http://localhost:7422 + #disable_streaming + #enable_hard_fails + } + + layer4 { + localhost:4444 { + @crowdsec crowdsec + route @crowdsec { + proxy { + upstream localhost:6443 + } + } } + } } -localhost { - route { - crowdsec - respond "Allowed by CrowdSec!" - } +localhost:8443 { + route { + crowdsec + respond "Allowed by Bouncer!" + } } -``` -Configuration using a Caddyfile is only supported for HTTP handlers. -You'll also need to use a recent version of Caddy (i.e. 2.7.3 and newer) and Go 1.20 (or newer). -In case you want to use the CrowdSec bouncer on TCP or UDP level, you'll need to configure Caddy using the native JSON format. -An example configuration is shown below: - -```json -{ - "apps": { - "crowdsec": { - "api_key": "", - "api_url": "http://127.0.0.1:8080/", - "ticker_interval": "10s", - "enable_streaming": true, - "enable_hard_fails": false, - }, - "http": { - "http_port": 9080, - "https_port": 9443, - "servers": { - "example": { - "listen": [ - "127.0.0.1:9443" - ], - "routes": [ - { - "group": "example-group", - "match": [ - { - "path": [ - "/*" - ] - } - ], - "handle": [ - { - "handler": "crowdsec" - }, - { - "handler": "static_response", - "status_code": "200", - "body": "Hello World!" - }, - { - "handler": "headers", - "response": { - "set": { - "Server": ["caddy-cs-bouncer-example-server"] - } - } - } - ] - } - ], - "logs": {} - } - } - }, - "layer4": { - "servers": { - "https_proxy": { - "listen": ["localhost:8443"], - "routes": [ - { - "match": [ - { - "crowdsec": {}, - "tls": {} - } - ], - "handle": [ - { - "handler": "proxy", - "upstreams": [ - { - "dial": ["localhost:9443"] - } - ] - } - ] - } - ] - } - } - }, - } +localhost:7443 { + route { + appsec + respond "Allowed by AppSec!" } +} + +localhost:6443 { + route { + crowdsec + appsec + respond "Allowed by Bouncer and AppSec!" + } +} ``` Run the Caddy server @@ -171,9 +120,6 @@ Run the Caddy server ```bash # with a Caddyfile go run main.go run -config Caddyfile - -# with JSON configuration -go run main.go run -config config.json ``` ## Demo diff --git a/docker/Dockerfile b/docker/Dockerfile index 7179a191..7a2c41ff 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,6 +12,7 @@ RUN xcaddy build \ --with github.com/mholt/caddy-l4 \ --with github.com/caddyserver/transform-encoder \ --with github.com/hslatman/caddy-crowdsec-bouncer/http@main \ + --with github.com/hslatman/caddy-crowdsec-bouncer/appsec@main \ --with github.com/hslatman/caddy-crowdsec-bouncer/layer4@main FROM caddy:${CADDY_VERSION} AS caddy From 3ce8c4270a94f3432a00f487f9321796f3a39e10 Mon Sep 17 00:00:00 2001 From: Herman Slatman Date: Tue, 19 Nov 2024 02:01:20 +0100 Subject: [PATCH 05/11] Add HTTP Bouncer and AppSec integration tests --- README.md | 2 +- appsec/appsec.go | 2 +- crowdsec/crowdsec_test.go | 13 +++ go.mod | 57 +++++++-- go.sum | 179 +++++++++++++++++++++++------ http/http.go | 4 +- internal/bouncer/appsec.go | 9 +- internal/httputils/context.go | 5 +- internal/httputils/context_test.go | 30 +++-- internal/httputils/http.go | 5 +- internal/httputils/http_test.go | 30 +++-- internal/testutils/testutils.go | 163 ++++++++++++++++++++++++++ test/appsec_test.go | 62 ++++++++++ test/live_test.go | 66 +++++++++++ test/stream_test.go | 67 +++++++++++ 15 files changed, 616 insertions(+), 78 deletions(-) create mode 100644 internal/testutils/testutils.go create mode 100644 test/appsec_test.go create mode 100644 test/live_test.go create mode 100644 test/stream_test.go diff --git a/README.md b/README.md index d8cc1939..820d4abc 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ Example Caddyfile: } } } - } + } } localhost:8443 { diff --git a/appsec/appsec.go b/appsec/appsec.go index a4378e32..a96205a6 100644 --- a/appsec/appsec.go +++ b/appsec/appsec.go @@ -87,7 +87,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyht ip netip.Addr ) - ctx, ip = httputils.EnsureIP(ctx, r) + ctx, ip = httputils.EnsureIP(ctx) if err := h.crowdsec.CheckRequest(ctx, r); err != nil { a := &bouncer.AppSecError{} if !errors.As(err, &a) { diff --git a/crowdsec/crowdsec_test.go b/crowdsec/crowdsec_test.go index 35c6582e..10dd9cb7 100644 --- a/crowdsec/crowdsec_test.go +++ b/crowdsec/crowdsec_test.go @@ -31,6 +31,19 @@ import ( "go.uber.org/goleak" ) +type fakeModule struct{} + +func (m fakeModule) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "http.handlers.crowdsec", + New: func() caddy.Module { return new(fakeModule) }, + } +} + +func init() { + caddy.RegisterModule(fakeModule{}) // prevents module warning logs +} + func TestCrowdSec_Provision(t *testing.T) { tests := []struct { name string diff --git a/go.mod b/go.mod index 049d9e76..b9b75d30 100644 --- a/go.mod +++ b/go.mod @@ -15,19 +15,21 @@ require ( github.com/prometheus/client_golang v1.20.4 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.9.0 + github.com/testcontainers/testcontainers-go v0.34.0 go.uber.org/goleak v1.2.1 go.uber.org/zap v1.26.0 ) require ( - cloud.google.com/go/compute v1.23.1 // indirect - cloud.google.com/go/iam v1.1.3 // indirect + cloud.google.com/go/kms v1.15.7 // indirect + dario.cat/mergo v1.0.0 // indirect filippo.io/edwards25519 v1.0.0 // indirect github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Masterminds/sprig/v3 v3.2.3 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/OneOfOne/xxhash v1.2.8 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b // indirect @@ -36,23 +38,35 @@ require ( github.com/bits-and-blooms/bitset v1.14.3 // indirect github.com/blackfireio/osinfo v1.0.5 // indirect github.com/caddyserver/certmagic v0.19.2 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chzyer/readline v1.5.1 // indirect + github.com/containerd/containerd v1.7.18 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgraph-io/badger v1.6.2 // indirect github.com/dgraph-io/badger/v2 v2.2007.4 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v27.1.1+incompatible // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/expr-lang/expr v1.16.9 // indirect github.com/fatih/color v1.17.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gaissmai/bart v0.13.0 // indirect github.com/go-kit/kit v0.13.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/analysis v0.23.0 // indirect github.com/go-openapi/errors v0.22.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect @@ -65,14 +79,16 @@ require ( github.com/go-sql-driver/mysql v1.7.1 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/goccy/go-yaml v1.12.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang/glog v1.1.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/glog v1.2.0 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/cel-go v0.17.1 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a // indirect github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/gax-go/v2 v2.12.2 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -89,6 +105,8 @@ require ( github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/lib/pq v1.10.9 // indirect github.com/libdns/libdns v0.2.1 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/manifoldco/promptui v0.9.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -101,11 +119,20 @@ require ( github.com/mitchellh/go-ps v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/sys/user v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/onsi/ginkgo/v2 v2.13.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.59.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect @@ -113,6 +140,8 @@ require ( github.com/quic-go/qtls-go1-20 v0.4.1 // indirect github.com/quic-go/quic-go v0.40.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/shirou/gopsutil/v3 v3.23.12 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/slackhq/nebula v1.7.2 // indirect @@ -124,11 +153,19 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect github.com/tailscale/tscert v0.0.0-20230806124524-28a91b69a046 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/urfave/cli v1.22.14 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect github.com/zeebo/blake3 v0.2.3 // indirect go.etcd.io/bbolt v1.3.8 // indirect go.mongodb.org/mongo-driver v1.17.0 // indirect go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/sdk v1.22.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect go.step.sm/cli-utils v0.8.0 // indirect go.step.sm/crypto v0.36.1 // indirect go.step.sm/linkedca v0.20.1 // indirect @@ -144,9 +181,11 @@ require ( golang.org/x/text v0.18.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231030173426-d783a09b4405 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405 // indirect - google.golang.org/grpc v1.59.0 // indirect + google.golang.org/api v0.169.0 // indirect + google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/grpc v1.64.1 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect diff --git a/go.sum b/go.sum index f6c7c3e6..bdd37487 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,22 @@ -cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= -cloud.google.com/go/compute v1.23.1 h1:V97tBoDaZHb6leicZ1G6DLK2BAaZLJ/7+9BB/En3hR0= -cloud.google.com/go/compute v1.23.1/go.mod h1:CqB3xpmPKKt3OJpW2ndFIXnA9A4xAy/F3Xp1ixncW78= +cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= +cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= +cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/iam v1.1.3 h1:18tKG7DzydKWUnLjonWcJO6wjSCAtzh4GcRKlH/Hrzc= -cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE= -cloud.google.com/go/kms v1.15.3 h1:RYsbxTRmk91ydKCzekI2YjryO4c5Y2M80Zwcs9/D/cI= -cloud.google.com/go/kms v1.15.3/go.mod h1:AJdXqHxS2GlPyduM99s9iGqi2nwbviBbhV/hdmt4iOQ= +cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= +cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= +cloud.google.com/go/kms v1.15.7 h1:7caV9K3yIxvlQPAcaFffhlT7d1qpxjB1wHBtjWa13SM= +cloud.google.com/go/kms v1.15.7/go.mod h1:ub54lbsa6tDkUwnu4W7Yt1aAIFLnspgh0kPGToDukeI= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek= filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -22,8 +28,8 @@ github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYr github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= @@ -47,6 +53,8 @@ github.com/caddyserver/caddy/v2 v2.7.5 h1:HoysvZkLcN2xJExEepaFHK92Qgs7xAiCFydN5x github.com/caddyserver/caddy/v2 v2.7.5/go.mod h1:XswQdR/IFwTNsIx+GDze2jYy+7WbjrSe1GEI20/PZ84= github.com/caddyserver/certmagic v0.19.2 h1:HZd1AKLx4592MalEGQS39DKs2ZOAJCEM/xYPMQ2/ui0= github.com/caddyserver/certmagic v0.19.2/go.mod h1:fsL01NomQ6N+kE2j37ZCnig2MFosG+MIO4ztnmG/zz8= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -63,16 +71,26 @@ github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= +github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/crowdsecurity/crowdsec v1.6.3 h1:L/6iT2/Gfl9bc9DQkHJz2BbpKM3P+yW6ocCKRyF4j1g= github.com/crowdsecurity/crowdsec v1.6.3/go.mod h1:LrdAX9l4vgaExQbNUVnvZIu/DPwD9pSE9gBj14D4MTo= github.com/crowdsecurity/go-cs-bouncer v0.0.14 h1:0hxOaa59pMT274qDzJXNxps4QfMnhSNss+oUn36HTpw= @@ -93,6 +111,14 @@ github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkz github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= +github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -100,6 +126,8 @@ github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI github.com/expr-lang/expr v1.16.9/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -117,8 +145,13 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= @@ -153,17 +186,18 @@ github.com/goccy/go-yaml v1.12.0 h1:/1WHjnMsI1dlIBQutrvSMGZRQufVO3asrHfTwfACoPM= github.com/goccy/go-yaml v1.12.0/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= -github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -174,7 +208,8 @@ github.com/google/cel-go v0.17.1/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulN github.com/google/certificate-transparency-go v1.1.6 h1:SW5K3sr7ptST/pIvNkSVWMiJqemRmkjJPPT0jzXdOOY= github.com/google/certificate-transparency-go v1.1.6/go.mod h1:0OJjOsOk+wj6aYQgP7FU0ioQ0AJUmnWPFMqTjQeazPQ= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= @@ -193,13 +228,15 @@ github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ= -github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= -github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUhuHF+DA= +github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.4.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/groob/finalizer v0.0.0-20170707115354-4c2ed49aabda/go.mod h1:MyndkAZd5rUMdNogn35MWXBX1UiBigrU8eTj8DoAC2c= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hslatman/ipstore v0.3.0 h1:3lUtYZMDGRdDePFFL2wUIrpHqMsqzJEluDnQwt43cfs= github.com/hslatman/ipstore v0.3.0/go.mod h1:fUg+lu09+OKKllPSRSvE6OdJ8AZB4sAjHxqW6QChpmU= @@ -266,6 +303,7 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= @@ -297,7 +335,11 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis= github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA= @@ -336,6 +378,18 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= @@ -344,6 +398,10 @@ github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4 github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -355,6 +413,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI= github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= @@ -383,6 +443,12 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/schollz/jsonstore v1.1.0 h1:WZBDjgezFS34CHI+myb4s8GGpir3UMpy7vWoCeO0n6E= github.com/schollz/jsonstore v1.1.0/go.mod h1:15c6+9guw8vDRyozGjN3FoILt0wpruJk9Pi66vjaZfg= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= @@ -429,6 +495,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -443,11 +511,21 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tailscale/tscert v0.0.0-20230806124524-28a91b69a046 h1:8rUlviSVOEe7TMk7W0gIPrW8MqEzYfZHpsNWSf8s2vg= github.com/tailscale/tscert v0.0.0-20230806124524-28a91b69a046/go.mod h1:kNGUQ3VESx3VZwRwA9MSCUegIl6+saPL8Noq82ozCaU= +github.com/testcontainers/testcontainers-go v0.34.0 h1:5fbgF0vIN5u+nD3IWabQwRybuB4GY8G2HHgCkbMzMHo= +github.com/testcontainers/testcontainers-go v0.34.0/go.mod h1:6P/kMkQe8yqPHfPWNulFGdFHTD8HB2vLq/231xY2iPQ= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= @@ -464,6 +542,24 @@ go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdH go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= +go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.step.sm/cli-utils v0.8.0 h1:b/Tc1/m3YuQq+u3ghTFP7Dz5zUekZj6GUmd5pCvkEXQ= go.step.sm/cli-utils v0.8.0/go.mod h1:S77aISrC0pKuflqiDfxxJlUbiXcAanyJ4POOnzFSxD4= go.step.sm/crypto v0.36.1 h1:hrHIc0qVcOowJB/r1SgPGu10d59onUw3czYeMLJluBc= @@ -508,6 +604,8 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQz golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= @@ -516,6 +614,8 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= @@ -524,6 +624,8 @@ golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= @@ -538,11 +640,15 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -552,6 +658,9 @@ golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -569,6 +678,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -578,6 +689,8 @@ golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= @@ -589,18 +702,16 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/api v0.147.0 h1:Can3FaQo9LlVqxJCodNmeZW/ib3/qKAY3rFeXiHo5gc= -google.golang.org/api v0.147.0/go.mod h1:pQ/9j83DcmPd/5C9e2nFOdjjNkDZ1G+zkbK2uvdkJMs= -google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= -google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= -google.golang.org/genproto/googleapis/api v0.0.0-20231030173426-d783a09b4405 h1:HJMDndgxest5n2y77fnErkM62iUsptE/H8p0dC2Huo4= -google.golang.org/genproto/googleapis/api v0.0.0-20231030173426-d783a09b4405/go.mod h1:oT32Z4o8Zv2xPQTg0pbVaPr0MPOH6f14RgXt7zfIpwg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405 h1:AB/lmRny7e2pLhFEYIbl5qkDAUt2h0ZRO4wGPhZf+ik= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405/go.mod h1:67X1fPuzjcrkymZzZV1vvkFeTn2Rvc6lYF9MYFGCcwE= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/api v0.169.0 h1:QwWPy71FgMWqJN/l6jVlFHUa29a7dcUy02I8o799nPY= +google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -624,6 +735,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM= howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= diff --git a/http/http.go b/http/http.go index fe368a10..4ef9399e 100644 --- a/http/http.go +++ b/http/http.go @@ -26,7 +26,7 @@ import ( "github.com/caddyserver/caddy/v2/modules/caddyhttp" "go.uber.org/zap" - _ "github.com/hslatman/caddy-crowdsec-bouncer/appsec" // include support for AppSec WAF component + _ "github.com/hslatman/caddy-crowdsec-bouncer/appsec" // always include AppSec module when HTTP is added "github.com/hslatman/caddy-crowdsec-bouncer/crowdsec" "github.com/hslatman/caddy-crowdsec-bouncer/internal/httputils" ) @@ -86,7 +86,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyht ip netip.Addr ) - ctx, ip = httputils.EnsureIP(ctx, r) + ctx, ip = httputils.EnsureIP(ctx) isAllowed, decision, err := h.crowdsec.IsAllowed(ip) if err != nil { return err // TODO: return error here? Or just log it and continue serving diff --git a/internal/bouncer/appsec.go b/internal/bouncer/appsec.go index 3acfc24e..6536f7c0 100644 --- a/internal/bouncer/appsec.go +++ b/internal/bouncer/appsec.go @@ -109,7 +109,7 @@ func (a *appsec) checkRequest(ctx context.Context, r *http.Request) error { case 200: return nil case 401: - a.logger.Error("appsec component not authenticated", zap.String("appsec_url", a.apiURL)) + a.logger.Error("appsec component not authenticated", zap.String("code", resp.Status), zap.String("appsec_url", a.apiURL)) return nil // this fails open, currently; make it fail hard if configured to do so? case 403: var r appsecResponse @@ -118,11 +118,14 @@ func (a *appsec) checkRequest(ctx context.Context, r *http.Request) error { } return &AppSecError{Err: errors.New("appsec rule triggered"), Action: r.Action, Duration: "", StatusCode: r.StatusCode} + case 404: + a.logger.Error("appsec component endpoint not found", zap.String("code", resp.Status), zap.String("appsec_url", a.apiURL)) + return nil case 500: - a.logger.Error("appsec component internal error", zap.String("appsec_url", a.apiURL)) + a.logger.Error("appsec component internal error", zap.String("code", resp.Status), zap.String("appsec_url", a.apiURL)) return nil // this fails open, currently; make it fail hard if configured to do so? default: - a.logger.Warn("appsec component returned unsupported status", zap.String("code", resp.Status)) + a.logger.Error("appsec component returned unsupported status", zap.String("code", resp.Status), zap.String("appsec_url", a.apiURL)) return nil } } diff --git a/internal/httputils/context.go b/internal/httputils/context.go index 07b4b7b0..5c448fa2 100644 --- a/internal/httputils/context.go +++ b/internal/httputils/context.go @@ -16,13 +16,12 @@ package httputils import ( "context" - "net/http" "net/netip" ) type contextKey struct{} -func EnsureIP(ctx context.Context, r *http.Request) (context.Context, netip.Addr) { // TODO: pass in ctx only? +func EnsureIP(ctx context.Context) (context.Context, netip.Addr) { var ( ip netip.Addr err error @@ -30,7 +29,7 @@ func EnsureIP(ctx context.Context, r *http.Request) (context.Context, netip.Addr ip, ok := FromContext(ctx) if !ok { - if ip, err = determineIPFromRequest(r); err != nil { // TODO: pass in ctx only? + if ip, err = determineIPFromRequest(ctx); err != nil { ip = netip.Addr{} } diff --git a/internal/httputils/context_test.go b/internal/httputils/context_test.go index 8d403db7..657aa4a8 100644 --- a/internal/httputils/context_test.go +++ b/internal/httputils/context_test.go @@ -16,17 +16,15 @@ package httputils import ( "context" - "net/http" - "net/http/httptest" "net/netip" "testing" "github.com/caddyserver/caddy/v2/modules/caddyhttp" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestEnsureIP(t *testing.T) { - r := httptest.NewRequest(http.MethodGet, "/caddy", http.NoBody) ipFromRequest := newCaddyVarsContext() caddyhttp.SetVar(ipFromRequest, caddyhttp.ClientIPVarKey, "127.0.0.1") ipFromContext := newCaddyVarsContext() @@ -37,7 +35,6 @@ func TestEnsureIP(t *testing.T) { type args struct { ctx context.Context - r *http.Request } tests := []struct { name string @@ -48,7 +45,6 @@ func TestEnsureIP(t *testing.T) { { name: "ip-from-request", args: args{ - r: r.WithContext(ipFromRequest), ctx: ipFromRequest, }, wantIP: netip.MustParseAddr("127.0.0.1"), @@ -56,7 +52,6 @@ func TestEnsureIP(t *testing.T) { { name: "ip-from-context", args: args{ - r: r.WithContext(ipFromContext), ctx: ipFromContext, }, wantIP: netip.MustParseAddr("127.0.0.3"), @@ -64,7 +59,6 @@ func TestEnsureIP(t *testing.T) { { name: "invalid-ip", args: args{ - r: r.WithContext(invalidIPCtx), ctx: invalidIPCtx, }, wantIP: netip.Addr{}, @@ -72,10 +66,30 @@ func TestEnsureIP(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx, ip := EnsureIP(tt.args.ctx, tt.args.r) + ctx, ip := EnsureIP(tt.args.ctx) assert.Equal(t, tt.wantIP, ip) assert.Equal(t, tt.wantIP, ctx.Value(contextKey{}).(netip.Addr)) }) } } + +func TestFromContext(t *testing.T) { + ctx := context.WithValue(context.Background(), contextKey{}, nil) + got, ok := FromContext(ctx) + require.False(t, ok) + require.Equal(t, netip.Addr{}, got) + require.False(t, got.IsValid()) + + ctx = context.WithValue(context.Background(), contextKey{}, netip.Addr{}) + got, ok = FromContext(ctx) + require.False(t, ok) + require.Equal(t, netip.Addr{}, got) + require.False(t, got.IsValid()) + + ip := netip.MustParseAddr("127.0.0.1") + ctx = context.WithValue(context.Background(), contextKey{}, ip) + got, ok = FromContext(ctx) + require.True(t, ok) + require.Equal(t, ip, got) +} diff --git a/internal/httputils/http.go b/internal/httputils/http.go index 6873c6b5..2aef47ca 100644 --- a/internal/httputils/http.go +++ b/internal/httputils/http.go @@ -15,6 +15,7 @@ package httputils import ( + "context" "errors" "fmt" "net/http" @@ -29,9 +30,9 @@ import ( // Caddy extracts from the original request and stores in the request context. // Support for setting the real client IP in case a proxy sits in front of // Caddy was added, so the client IP reported here is the actual client IP. -func determineIPFromRequest(r *http.Request) (netip.Addr, error) { +func determineIPFromRequest(ctx context.Context) (netip.Addr, error) { zero := netip.Addr{} - clientIPVar := caddyhttp.GetVar(r.Context(), caddyhttp.ClientIPVarKey) + clientIPVar := caddyhttp.GetVar(ctx, caddyhttp.ClientIPVarKey) if clientIPVar == nil { return zero, errors.New("failed getting client IP from context") } diff --git a/internal/httputils/http_test.go b/internal/httputils/http_test.go index af6ffa0e..05009520 100644 --- a/internal/httputils/http_test.go +++ b/internal/httputils/http_test.go @@ -16,13 +16,11 @@ package httputils import ( "context" - "net/http" - "net/http/httptest" "net/netip" - "reflect" "testing" "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "github.com/stretchr/testify/require" ) func newCaddyVarsContext() (ctx context.Context) { @@ -31,7 +29,6 @@ func newCaddyVarsContext() (ctx context.Context) { } func Test_determineIPFromRequest(t *testing.T) { - r := httptest.NewRequest(http.MethodGet, "/caddy", nil) okCtx := newCaddyVarsContext() caddyhttp.SetVar(okCtx, caddyhttp.ClientIPVarKey, "127.0.0.1") noIPCtx := newCaddyVarsContext() @@ -42,7 +39,7 @@ func Test_determineIPFromRequest(t *testing.T) { invalidIPCtx := newCaddyVarsContext() caddyhttp.SetVar(invalidIPCtx, caddyhttp.ClientIPVarKey, "127.0.0.1.x") type args struct { - r *http.Request + ctx context.Context } tests := []struct { name string @@ -50,22 +47,23 @@ func Test_determineIPFromRequest(t *testing.T) { want netip.Addr wantErr bool }{ - {"ok", args{r.WithContext(okCtx)}, netip.MustParseAddr("127.0.0.1"), false}, - {"no-ip", args{r.WithContext(noIPCtx)}, netip.Addr{}, true}, - {"wrong-type", args{r.WithContext(wrongTypeCtx)}, netip.Addr{}, true}, - {"empty-ip", args{r.WithContext(emptyIPCtx)}, netip.Addr{}, true}, - {"invalid-ip", args{r.WithContext(invalidIPCtx)}, netip.Addr{}, true}, + {"ok", args{okCtx}, netip.MustParseAddr("127.0.0.1"), false}, + {"no-ip", args{noIPCtx}, netip.Addr{}, true}, + {"wrong-type", args{wrongTypeCtx}, netip.Addr{}, true}, + {"empty-ip", args{emptyIPCtx}, netip.Addr{}, true}, + {"invalid-ip", args{invalidIPCtx}, netip.Addr{}, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := determineIPFromRequest(tt.args.r) - if (err != nil) != tt.wantErr { - t.Errorf("determineIPFromRequest() error = %v, wantErr %v", err, tt.wantErr) + got, err := determineIPFromRequest(tt.args.ctx) + if tt.wantErr { + require.Error(t, err) + require.Equal(t, netip.Addr{}, got) return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("determineIPFromRequest() = %v, want %v", got, tt.want) - } + + require.NoError(t, err) + require.Equal(t, tt.want, got) }) } } diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go new file mode 100644 index 00000000..e8217132 --- /dev/null +++ b/internal/testutils/testutils.go @@ -0,0 +1,163 @@ +package testutils + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "strings" + "testing" + "time" + + "github.com/caddyserver/caddy/v2" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/exec" + "github.com/testcontainers/testcontainers-go/wait" + + "github.com/hslatman/caddy-crowdsec-bouncer/crowdsec" +) + +const testAPIKey = "testbouncer1key" + +type container struct { + c testcontainers.Container + endpoint string + appsec string +} + +func NewCrowdSecContainer(t *testing.T, ctx context.Context) *container { + t.Helper() + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: "crowdsecurity/crowdsec:latest", + ExposedPorts: []string{"8080/tcp"}, + WaitingFor: wait.ForLog("CrowdSec Local API listening on 0.0.0.0:8080"), + Env: map[string]string{ + "BOUNCER_KEY_testbouncer1": testAPIKey, + "DISABLE_ONLINE_API": "true", + }, + }, + Started: true, + Logger: testcontainers.TestLogger(t), + }) + require.NoError(t, err) + require.NotNil(t, c) + t.Cleanup(func() { c.Terminate(ctx) }) + + endpointPort, err := c.MappedPort(ctx, "8080/tcp") + require.NoError(t, err) + + return &container{ + c: c, + endpoint: fmt.Sprintf("http://localhost:%d", endpointPort.Int()), + } +} + +func (c *container) APIUrl() string { + return c.endpoint +} + +func (c *container) APIKey() string { + return testAPIKey +} + +func (c *container) AppSecUrl() string { + return c.appsec +} + +func (c *container) Exec(ctx context.Context, cmd []string) (int, io.Reader, error) { + return c.c.Exec(ctx, cmd, []exec.ProcessOption{}...) +} + +const appSecConfig = `listen_addr: 0.0.0.0:7422 +appsec_config: crowdsecurity/appsec-default +name: test +source: appsec +labels: + type: appsec +` + +func NewAppSecContainer(t *testing.T, ctx context.Context) *container { + t.Helper() + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: "crowdsecurity/crowdsec:latest", + ExposedPorts: []string{"8080/tcp", "7422/tcp"}, + WaitingFor: wait.ForLog("CrowdSec Local API listening on 0.0.0.0:8080"), + Env: map[string]string{ + "BOUNCER_KEY_testbouncer1": testAPIKey, + "DISABLE_ONLINE_API": "true", + }, + Files: []testcontainers.ContainerFile{ + { + Reader: bytes.NewBuffer([]byte(appSecConfig)), + ContainerFilePath: "/etc/crowdsec/acquis.d/appsec.yaml", + }, + }, + }, + Started: true, + Logger: testcontainers.TestLogger(t), + }) + require.NoError(t, err) + require.NotNil(t, c) + t.Cleanup(func() { c.Terminate(ctx) }) + + code, reader, err := c.Exec(ctx, []string{"cscli", "collections", "install", "crowdsecurity/appsec-virtual-patching"}) + require.NoError(t, err) + require.Equal(t, 0, code) + LogContainerOutput(t, reader) + + code, reader, err = c.Exec(ctx, []string{"cscli", "collections", "install", "crowdsecurity/appsec-generic-rules"}) + require.NoError(t, err) + require.Equal(t, 0, code) + LogContainerOutput(t, reader) + + time.Sleep(2 * time.Second) + + err = c.Stop(ctx, nil) + require.NoError(t, err) + err = c.Start(ctx) + require.NoError(t, err) + + endpointPort, err := c.MappedPort(ctx, "8080/tcp") + require.NoError(t, err) + + appsecPort, err := c.MappedPort(ctx, "7422/tcp") + require.NoError(t, err) + + return &container{ + c: c, + endpoint: fmt.Sprintf("http://localhost:%d", endpointPort.Int()), + appsec: fmt.Sprintf("http://localhost:%d", appsecPort.Int()), + } +} + +func NewCrowdSecModule(t *testing.T, ctx context.Context, config string) *crowdsec.CrowdSec { + t.Helper() + + var c crowdsec.CrowdSec + err := json.Unmarshal([]byte(config), &c) + require.NoError(t, err) + + caddyCtx, cancel := caddy.NewContext(caddy.Context{Context: ctx}) + t.Cleanup(cancel) + + err = c.Provision(caddyCtx) + require.NoError(t, err) + + err = c.Validate() + require.NoError(t, err) + + return &c +} + +func LogContainerOutput(t *testing.T, reader io.Reader) { + t.Helper() + + buf := new(strings.Builder) + _, err := io.Copy(buf, reader) + require.NoError(t, err) + t.Log(buf.String()) +} diff --git a/test/appsec_test.go b/test/appsec_test.go new file mode 100644 index 00000000..4d716540 --- /dev/null +++ b/test/appsec_test.go @@ -0,0 +1,62 @@ +package test + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "github.com/stretchr/testify/require" + + _ "github.com/hslatman/caddy-crowdsec-bouncer/http" // prevent module warning logs + "github.com/hslatman/caddy-crowdsec-bouncer/internal/httputils" + "github.com/hslatman/caddy-crowdsec-bouncer/internal/testutils" +) + +func newCaddyVarsContext() (ctx context.Context) { + ctx = context.WithValue(context.Background(), caddyhttp.VarsCtxKey, map[string]any{}) + return +} + +func TestAppSec(t *testing.T) { + ctx := newCaddyVarsContext() + + container := testutils.NewAppSecContainer(t, ctx) + + config := fmt.Sprintf(`{ + "api_url": %q, + "api_key": %q, + "enable_streaming": false, + "appsec_url": %q + }`, container.APIUrl(), container.APIKey(), container.AppSecUrl()) + + caddyhttp.SetVar(ctx, caddyhttp.ClientIPVarKey, "127.0.0.1") + ctx, _ = httputils.EnsureIP(ctx) + crowdsec := testutils.NewCrowdSecModule(t, ctx, config) + + err := crowdsec.Start() + require.NoError(t, err) + + // wait a little bit of time to let the go-cs-bouncer do _some_ work, + // before it properly returns; seems to hang otherwise on b.wg.Wait(). + time.Sleep(1 * time.Second) + + r := httptest.NewRequest(http.MethodGet, "http://www.example.com", http.NoBody) + r = r.WithContext(ctx) + err = crowdsec.CheckRequest(ctx, r) + require.NoError(t, err) + + r = httptest.NewRequest(http.MethodGet, "http://www.example.com/rpc2", http.NoBody) + r = r.WithContext(ctx) + err = crowdsec.CheckRequest(ctx, r) + require.Error(t, err) + + err = crowdsec.Stop() + require.NoError(t, err) + + err = crowdsec.Cleanup() + require.NoError(t, err) +} diff --git a/test/live_test.go b/test/live_test.go new file mode 100644 index 00000000..7f370216 --- /dev/null +++ b/test/live_test.go @@ -0,0 +1,66 @@ +package test + +import ( + "context" + "fmt" + "net/netip" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + _ "github.com/hslatman/caddy-crowdsec-bouncer/http" // prevent module warning logs + "github.com/hslatman/caddy-crowdsec-bouncer/internal/testutils" +) + +func TestLiveBouncer(t *testing.T) { + ctx := context.Background() + + // TODO: do tests with the handlers/matchers (instead)? + + container := testutils.NewCrowdSecContainer(t, ctx) + + config := fmt.Sprintf(`{ + "api_url": %q, + "api_key": %q, + "enable_streaming": false + }`, container.APIUrl(), container.APIKey()) + + crowdsec := testutils.NewCrowdSecModule(t, ctx, config) + + err := crowdsec.Start() + require.NoError(t, err) + + // wait a little bit of time to let the go-cs-bouncer do _some_ work, + // before it properly returns; seems to hang otherwise on b.wg.Wait(). + time.Sleep(100 * time.Millisecond) + + // simulate a lookup; no decisions available, so will be allowed + allowed, decision, err := crowdsec.IsAllowed(netip.MustParseAddr("127.0.0.1")) + assert.NoError(t, err) + assert.True(t, allowed) + assert.Nil(t, decision) + + // add a ban for 127.0.0.1 + code, reader, err := container.Exec(ctx, []string{"cscli", "decisions", "add", "--ip", "127.0.0.1", "--duration", "20s"}) + require.NoError(t, err) + require.Equal(t, 0, code) + testutils.LogContainerOutput(t, reader) + + // simulate a lookup; 127.0.0.1 should now be banned + allowed, decision, err = crowdsec.IsAllowed(netip.MustParseAddr("127.0.0.1")) + assert.NoError(t, err) + assert.False(t, allowed) + if assert.NotNil(t, decision) { + assert.Equal(t, "ban", *decision.Type) + assert.Equal(t, "127.0.0.1", *decision.Value) + assert.Equal(t, "Ip", *decision.Scope) + } + + err = crowdsec.Stop() + require.NoError(t, err) + + err = crowdsec.Cleanup() + require.NoError(t, err) +} diff --git a/test/stream_test.go b/test/stream_test.go new file mode 100644 index 00000000..2631963e --- /dev/null +++ b/test/stream_test.go @@ -0,0 +1,67 @@ +package test + +import ( + "context" + "fmt" + "net/netip" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + _ "github.com/hslatman/caddy-crowdsec-bouncer/http" // prevent module warning logs + "github.com/hslatman/caddy-crowdsec-bouncer/internal/testutils" +) + +func TestStreamingBouncer(t *testing.T) { + ctx := context.Background() + + container := testutils.NewCrowdSecContainer(t, ctx) + + config := fmt.Sprintf(`{ + "api_url": %q, + "api_key": %q, + "ticker_interval": "1s" + }`, container.APIUrl(), container.APIKey()) + + crowdsec := testutils.NewCrowdSecModule(t, ctx, config) + + err := crowdsec.Start() + require.NoError(t, err) + + // wait a little bit of time to let the go-cs-bouncer do _some_ work, + // before it properly returns; seems to hang otherwise on b.wg.Wait(). + time.Sleep(100 * time.Millisecond) + + // simulate a lookup; no decisions available, so will be allowed + allowed, decision, err := crowdsec.IsAllowed(netip.MustParseAddr("127.0.0.1")) + assert.NoError(t, err) + assert.True(t, allowed) + assert.Nil(t, decision) + + // add a ban for 127.0.0.1 + code, reader, err := container.Exec(ctx, []string{"cscli", "decisions", "add", "--ip", "127.0.0.1", "--duration", "20s"}) + require.NoError(t, err) + require.Equal(t, 0, code) + testutils.LogContainerOutput(t, reader) + + // wait 3 seconds to obtain new decision; streaming ticker interval is 1 second, so should be enough time + time.Sleep(3 * time.Second) + + // simulate a lookup; 127.0.0.1 should now be banned + allowed, decision, err = crowdsec.IsAllowed(netip.MustParseAddr("127.0.0.1")) + assert.NoError(t, err) + assert.False(t, allowed) + if assert.NotNil(t, decision) { + assert.Equal(t, "ban", *decision.Type) + assert.Equal(t, "127.0.0.1", *decision.Value) + assert.Equal(t, "Ip", *decision.Scope) + } + + err = crowdsec.Stop() + require.NoError(t, err) + + err = crowdsec.Cleanup() + require.NoError(t, err) +} From 5c65ea6bdb5d15c3376164ad660c38718b72f55d Mon Sep 17 00:00:00 2001 From: Herman Slatman Date: Tue, 19 Nov 2024 02:04:45 +0100 Subject: [PATCH 06/11] Fix linter issue --- internal/testutils/testutils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go index e8217132..48d0563f 100644 --- a/internal/testutils/testutils.go +++ b/internal/testutils/testutils.go @@ -44,7 +44,7 @@ func NewCrowdSecContainer(t *testing.T, ctx context.Context) *container { }) require.NoError(t, err) require.NotNil(t, c) - t.Cleanup(func() { c.Terminate(ctx) }) + t.Cleanup(func() { _ = c.Terminate(ctx) }) endpointPort, err := c.MappedPort(ctx, "8080/tcp") require.NoError(t, err) @@ -102,7 +102,7 @@ func NewAppSecContainer(t *testing.T, ctx context.Context) *container { }) require.NoError(t, err) require.NotNil(t, c) - t.Cleanup(func() { c.Terminate(ctx) }) + t.Cleanup(func() { _ = c.Terminate(ctx) }) code, reader, err := c.Exec(ctx, []string{"cscli", "collections", "install", "crowdsecurity/appsec-virtual-patching"}) require.NoError(t, err) From 2153e8a9749f66e634dfe709ac0481cac9b117b8 Mon Sep 17 00:00:00 2001 From: Herman Slatman Date: Tue, 19 Nov 2024 13:42:51 +0100 Subject: [PATCH 07/11] Fix integration test for AppSec --- internal/testutils/testutils.go | 89 +++++++++++++++++++++++++-------- test/appsec_test.go | 2 +- 2 files changed, 69 insertions(+), 22 deletions(-) diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go index 48d0563f..5f5dc778 100644 --- a/internal/testutils/testutils.go +++ b/internal/testutils/testutils.go @@ -11,6 +11,7 @@ import ( "time" "github.com/caddyserver/caddy/v2" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/exec" @@ -51,7 +52,7 @@ func NewCrowdSecContainer(t *testing.T, ctx context.Context) *container { return &container{ c: c, - endpoint: fmt.Sprintf("http://localhost:%d", endpointPort.Int()), + endpoint: fmt.Sprintf("http://127.0.0.1:%d", endpointPort.Int()), } } @@ -81,11 +82,70 @@ labels: func NewAppSecContainer(t *testing.T, ctx context.Context) *container { t.Helper() + + // shared data between initialization and actual AppSec container + mounts := testcontainers.ContainerMounts{ + { + Source: testcontainers.GenericVolumeMountSource{ + Name: "crowdsec-etc", + }, + Target: "/etc/crowdsec", + }, + { + Source: testcontainers.GenericVolumeMountSource{ + Name: "crowdsec-data", + }, + Target: "/var/lib/crowdsec/data", + }, + } + + // AppSec requires some WAF rules to be present, so we start by initializing + // a container, installing the required collections, and then stopping it again. + initContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: "crowdsecurity/crowdsec:latest", + Mounts: mounts, + ExposedPorts: []string{"8080/tcp"}, + WaitingFor: wait.ForLog("CrowdSec Local API listening on 0.0.0.0:8080"), + Env: map[string]string{ + "BOUNCER_KEY_testbouncer1": testAPIKey, + "DISABLE_ONLINE_API": "true", + }, + }, + Started: true, + Logger: testcontainers.TestLogger(t), + }) + require.NoError(t, err) + require.NotNil(t, initContainer) + + // install some AppSec rule collections + code, reader, err := initContainer.Exec(ctx, []string{"cscli", "collections", "install", "crowdsecurity/appsec-virtual-patching"}) + assert.NoError(t, err) + assert.Equal(t, 0, code) + LogContainerOutput(t, reader) + + code, reader, err = initContainer.Exec(ctx, []string{"cscli", "collections", "install", "crowdsecurity/appsec-generic-rules"}) + assert.NoError(t, err) + assert.Equal(t, 0, code) + LogContainerOutput(t, reader) + + // allow container some slack + time.Sleep(1 * time.Second) + + // cleanly stop the initialization container + duration := 3 * time.Second + err = initContainer.Stop(ctx, &duration) + require.NoError(t, err) + err = initContainer.Terminate(ctx) + require.NoError(t, err) + + // create the actual AppSec container c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: testcontainers.ContainerRequest{ Image: "crowdsecurity/crowdsec:latest", + Mounts: mounts, ExposedPorts: []string{"8080/tcp", "7422/tcp"}, - WaitingFor: wait.ForLog("CrowdSec Local API listening on 0.0.0.0:8080"), + WaitingFor: wait.ForLog("Appsec Runner ready to process event"), Env: map[string]string{ "BOUNCER_KEY_testbouncer1": testAPIKey, "DISABLE_ONLINE_API": "true", @@ -104,23 +164,6 @@ func NewAppSecContainer(t *testing.T, ctx context.Context) *container { require.NotNil(t, c) t.Cleanup(func() { _ = c.Terminate(ctx) }) - code, reader, err := c.Exec(ctx, []string{"cscli", "collections", "install", "crowdsecurity/appsec-virtual-patching"}) - require.NoError(t, err) - require.Equal(t, 0, code) - LogContainerOutput(t, reader) - - code, reader, err = c.Exec(ctx, []string{"cscli", "collections", "install", "crowdsecurity/appsec-generic-rules"}) - require.NoError(t, err) - require.Equal(t, 0, code) - LogContainerOutput(t, reader) - - time.Sleep(2 * time.Second) - - err = c.Stop(ctx, nil) - require.NoError(t, err) - err = c.Start(ctx) - require.NoError(t, err) - endpointPort, err := c.MappedPort(ctx, "8080/tcp") require.NoError(t, err) @@ -129,8 +172,8 @@ func NewAppSecContainer(t *testing.T, ctx context.Context) *container { return &container{ c: c, - endpoint: fmt.Sprintf("http://localhost:%d", endpointPort.Int()), - appsec: fmt.Sprintf("http://localhost:%d", appsecPort.Int()), + endpoint: fmt.Sprintf("http://127.0.0.1:%d", endpointPort.Int()), + appsec: fmt.Sprintf("http://127.0.0.1:%d", appsecPort.Int()), } } @@ -156,6 +199,10 @@ func NewCrowdSecModule(t *testing.T, ctx context.Context, config string) *crowds func LogContainerOutput(t *testing.T, reader io.Reader) { t.Helper() + if reader == nil { + return + } + buf := new(strings.Builder) _, err := io.Copy(buf, reader) require.NoError(t, err) diff --git a/test/appsec_test.go b/test/appsec_test.go index 4d716540..284e3339 100644 --- a/test/appsec_test.go +++ b/test/appsec_test.go @@ -42,7 +42,7 @@ func TestAppSec(t *testing.T) { // wait a little bit of time to let the go-cs-bouncer do _some_ work, // before it properly returns; seems to hang otherwise on b.wg.Wait(). - time.Sleep(1 * time.Second) + time.Sleep(100 * time.Millisecond) r := httptest.NewRequest(http.MethodGet, "http://www.example.com", http.NoBody) r = r.WithContext(ctx) From d42bf76819ea78eb229cd5afd5c019e85a8987fe Mon Sep 17 00:00:00 2001 From: Herman Slatman Date: Wed, 20 Nov 2024 01:42:30 +0100 Subject: [PATCH 08/11] Improve AppSec request header handling and tests --- internal/bouncer/appsec.go | 23 +++++++++++------- internal/bouncer/bouncer.go | 2 +- internal/testutils/testutils.go | 13 +++++++---- test/appsec_test.go | 41 +++++++++++++++++++++++++++++++-- 4 files changed, 64 insertions(+), 15 deletions(-) diff --git a/internal/bouncer/appsec.go b/internal/bouncer/appsec.go index 6536f7c0..488871e0 100644 --- a/internal/bouncer/appsec.go +++ b/internal/bouncer/appsec.go @@ -63,34 +63,41 @@ func (a *appsec) checkRequest(ctx context.Context, r *http.Request) error { return errors.New("could not retrieve netip.Addr from context") } - originalBody, err := io.ReadAll(r.Body) - if err != nil { - return err - } - method := http.MethodGet var body io.ReadCloser = http.NoBody - if len(originalBody) > 0 { - method = http.MethodPost + if r.Body != nil && r.ContentLength > 0 { + originalBody, err := io.ReadAll(r.Body) + if err != nil { + return err + } + method = http.MethodPost buffer := a.pool.Get() defer a.pool.Put(buffer) _, _ = buffer.Write(originalBody) body = io.NopCloser(buffer) + + r.Body = io.NopCloser(bytes.NewBuffer(originalBody)) } - r.Body = io.NopCloser(bytes.NewBuffer(originalBody)) req, err := http.NewRequestWithContext(ctx, method, a.apiURL, body) if err != nil { return err } + for key, headers := range r.Header { + for _, value := range headers { + req.Header.Add(key, value) + } + } req.Header.Set("X-Crowdsec-Appsec-Ip", originalIP.String()) req.Header.Set("X-Crowdsec-Appsec-Uri", r.URL.String()) req.Header.Set("X-Crowdsec-Appsec-Host", r.Host) req.Header.Set("X-Crowdsec-Appsec-Verb", r.Method) req.Header.Set("X-Crowdsec-Appsec-Api-Key", a.apiKey) + req.Header.Set("X-Crowdsec-Appsec-User-Agent", r.Header.Get("User-Agent")) + req.Header.Set("User-Agent", userAgent) totalAppSecCalls.Inc() resp, err := a.client.Do(req) diff --git a/internal/bouncer/bouncer.go b/internal/bouncer/bouncer.go index 0fde7839..0fa360e7 100644 --- a/internal/bouncer/bouncer.go +++ b/internal/bouncer/bouncer.go @@ -34,6 +34,7 @@ import ( const ( userAgentName = "caddy-cs-bouncer" userAgentVersion = "v0.8.0" + userAgent = userAgentName + "/" + userAgentVersion maxNumberOfDecisionsToLog = 10 ) @@ -65,7 +66,6 @@ type Bouncer struct { // New creates a new (streaming) Bouncer with a storage based on immutable radix tree func New(apiKey, apiURL, appSecURL, tickerInterval string, logger *zap.Logger) (*Bouncer, error) { - userAgent := fmt.Sprintf("%s/%s", userAgentName, userAgentVersion) insecureSkipVerify := false instantiatedAt := time.Now() instanceID, err := generateInstanceID(instantiatedAt) diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go index 5f5dc778..1d8888c4 100644 --- a/internal/testutils/testutils.go +++ b/internal/testutils/testutils.go @@ -32,12 +32,13 @@ func NewCrowdSecContainer(t *testing.T, ctx context.Context) *container { t.Helper() c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: testcontainers.ContainerRequest{ - Image: "crowdsecurity/crowdsec:latest", + Image: "crowdsecurity/crowdsec:slim", ExposedPorts: []string{"8080/tcp"}, WaitingFor: wait.ForLog("CrowdSec Local API listening on 0.0.0.0:8080"), Env: map[string]string{ "BOUNCER_KEY_testbouncer1": testAPIKey, "DISABLE_ONLINE_API": "true", + "NO_HUB_UPGRADE": "true", }, }, Started: true, @@ -74,7 +75,7 @@ func (c *container) Exec(ctx context.Context, cmd []string) (int, io.Reader, err const appSecConfig = `listen_addr: 0.0.0.0:7422 appsec_config: crowdsecurity/appsec-default -name: test +name: appsec-test source: appsec labels: type: appsec @@ -103,13 +104,14 @@ func NewAppSecContainer(t *testing.T, ctx context.Context) *container { // a container, installing the required collections, and then stopping it again. initContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: testcontainers.ContainerRequest{ - Image: "crowdsecurity/crowdsec:latest", + Image: "crowdsecurity/crowdsec:slim", Mounts: mounts, ExposedPorts: []string{"8080/tcp"}, WaitingFor: wait.ForLog("CrowdSec Local API listening on 0.0.0.0:8080"), Env: map[string]string{ "BOUNCER_KEY_testbouncer1": testAPIKey, "DISABLE_ONLINE_API": "true", + "NO_HUB_UPGRADE": "true", }, }, Started: true, @@ -142,13 +144,16 @@ func NewAppSecContainer(t *testing.T, ctx context.Context) *container { // create the actual AppSec container c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: testcontainers.ContainerRequest{ - Image: "crowdsecurity/crowdsec:latest", + Image: "crowdsecurity/crowdsec:slim", Mounts: mounts, ExposedPorts: []string{"8080/tcp", "7422/tcp"}, WaitingFor: wait.ForLog("Appsec Runner ready to process event"), Env: map[string]string{ "BOUNCER_KEY_testbouncer1": testAPIKey, "DISABLE_ONLINE_API": "true", + "NO_HUB_UPGRADE": "true", + "LEVEL_DEBUG": "true", + "DEBUG": "true", }, Files: []testcontainers.ContainerFile{ { diff --git a/test/appsec_test.go b/test/appsec_test.go index 284e3339..e8ca5aaf 100644 --- a/test/appsec_test.go +++ b/test/appsec_test.go @@ -1,6 +1,7 @@ package test import ( + "bytes" "context" "fmt" "net/http" @@ -9,6 +10,7 @@ import ( "time" "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" _ "github.com/hslatman/caddy-crowdsec-bouncer/http" // prevent module warning logs @@ -44,15 +46,50 @@ func TestAppSec(t *testing.T) { // before it properly returns; seems to hang otherwise on b.wg.Wait(). time.Sleep(100 * time.Millisecond) + // simulate a request that is allowed r := httptest.NewRequest(http.MethodGet, "http://www.example.com", http.NoBody) r = r.WithContext(ctx) + r.Header.Set("User-Agent", "test-appsec") err = crowdsec.CheckRequest(ctx, r) - require.NoError(t, err) + assert.NoError(t, err) + // simulate a request that'll be banned using the currently installed rules, similar + // to https://docs.crowdsec.net/docs/appsec/installation/#making-sure-everything-works. + // This will simulate exploitation of JetBrains Teamcity Auth Bypass; CVE-2023-42793. r = httptest.NewRequest(http.MethodGet, "http://www.example.com/rpc2", http.NoBody) r = r.WithContext(ctx) + r.Header.Set("User-Agent", "test-appsec") + err = crowdsec.CheckRequest(ctx, r) + assert.Error(t, err) + + // TODO: find out why the below two POST requests with a body don't trigger + // a ban from the AppSec component. + // // simulate a request exploiting Ivanti EPM - SQLi; CVE-2024-29824. + // body := bytes.NewBufferString("blabla 'xp_cmdshell' blabla") + // r = httptest.NewRequest(http.MethodPost, "http://www.example.com/WSStatusEvents/EventHandler.asmx", body) + // r = r.WithContext(ctx) + // r.Header.Set("User-Agent", "test-appsec") + // err = crowdsec.CheckRequest(ctx, r) + // assert.Error(t, err) + + // // simulate a request exploiting Dasan GPON RCE; CVE-2018-10562 + // body := bytes.NewBufferString(`{"dest_host": "ping"}`) + // r = httptest.NewRequest(http.MethodPost, "http://www.example.com/gponform/diag_form", body) + // r = r.WithContext(ctx) + // r.Header.Set("User-Agent", "test-appsec") + // r.Header.Set("Content-Type", "application/json") + // err = crowdsec.CheckRequest(ctx, r) + // assert.Error(t, err) + + // simulate a request exploiting WooCommerce auth bypass; CVE-2023-28121, ensuring + // that headers are passed correctly. + body := bytes.NewBufferString("some body") + r = httptest.NewRequest(http.MethodPost, "http://www.example.com", body) + r = r.WithContext(ctx) + r.Header.Set("User-Agent", "test-appsec") + r.Header.Set("x-wcpay-platform-checkout-user", "x-wcpay-platform-checkout-user") err = crowdsec.CheckRequest(ctx, r) - require.Error(t, err) + assert.Error(t, err) err = crowdsec.Stop() require.NoError(t, err) From 098ef3283d58dee7c001083a4f6afd763fcb2e38 Mon Sep 17 00:00:00 2001 From: Herman Slatman Date: Mon, 25 Nov 2024 01:32:36 +0100 Subject: [PATCH 09/11] Add two additional tests for AppSec POST requests This commit explicitly sets the content length of the request, so that CrowdSec actually reads the request body, instead of skipping it. Two new tests that involve an HTTP request body have been added as integration tests. --- internal/bouncer/appsec.go | 14 ++++++++++-- internal/testutils/testutils.go | 2 +- test/appsec_test.go | 39 +++++++++++++++++---------------- 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/internal/bouncer/appsec.go b/internal/bouncer/appsec.go index 488871e0..12df4fd5 100644 --- a/internal/bouncer/appsec.go +++ b/internal/bouncer/appsec.go @@ -63,6 +63,7 @@ func (a *appsec) checkRequest(ctx context.Context, r *http.Request) error { return errors.New("could not retrieve netip.Addr from context") } + var contentLength int method := http.MethodGet var body io.ReadCloser = http.NoBody if r.Body != nil && r.ContentLength > 0 { @@ -71,13 +72,16 @@ func (a *appsec) checkRequest(ctx context.Context, r *http.Request) error { return err } - method = http.MethodPost buffer := a.pool.Get() defer a.pool.Put(buffer) _, _ = buffer.Write(originalBody) + + method = http.MethodPost + contentLength = buffer.Len() body = io.NopCloser(buffer) + // "reset" the original request body r.Body = io.NopCloser(bytes.NewBuffer(originalBody)) } @@ -97,7 +101,13 @@ func (a *appsec) checkRequest(ctx context.Context, r *http.Request) error { req.Header.Set("X-Crowdsec-Appsec-Verb", r.Method) req.Header.Set("X-Crowdsec-Appsec-Api-Key", a.apiKey) req.Header.Set("X-Crowdsec-Appsec-User-Agent", r.Header.Get("User-Agent")) - req.Header.Set("User-Agent", userAgent) + req.Header.Set("User-Agent", userAgentName) + + // explicitly setting the content length results in CrowdSec properly accepting + // the request body. Without this the Content-Length header won't be set to the + // correct value, resulting in CrowdSec skipping its evaluation. We should test + // whether https://github.com/crowdsecurity/crowdsec/pull/3342 makes it work. + req.ContentLength = int64(contentLength) totalAppSecCalls.Inc() resp, err := a.client.Do(req) diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go index 1d8888c4..8f31bab3 100644 --- a/internal/testutils/testutils.go +++ b/internal/testutils/testutils.go @@ -157,7 +157,7 @@ func NewAppSecContainer(t *testing.T, ctx context.Context) *container { }, Files: []testcontainers.ContainerFile{ { - Reader: bytes.NewBuffer([]byte(appSecConfig)), + Reader: bytes.NewBufferString(appSecConfig), ContainerFilePath: "/etc/crowdsec/acquis.d/appsec.yaml", }, }, diff --git a/test/appsec_test.go b/test/appsec_test.go index e8ca5aaf..3f352d4c 100644 --- a/test/appsec_test.go +++ b/test/appsec_test.go @@ -6,6 +6,8 @@ import ( "fmt" "net/http" "net/http/httptest" + "net/url" + "strings" "testing" "time" @@ -62,28 +64,27 @@ func TestAppSec(t *testing.T) { err = crowdsec.CheckRequest(ctx, r) assert.Error(t, err) - // TODO: find out why the below two POST requests with a body don't trigger - // a ban from the AppSec component. - // // simulate a request exploiting Ivanti EPM - SQLi; CVE-2024-29824. - // body := bytes.NewBufferString("blabla 'xp_cmdshell' blabla") - // r = httptest.NewRequest(http.MethodPost, "http://www.example.com/WSStatusEvents/EventHandler.asmx", body) - // r = r.WithContext(ctx) - // r.Header.Set("User-Agent", "test-appsec") - // err = crowdsec.CheckRequest(ctx, r) - // assert.Error(t, err) - - // // simulate a request exploiting Dasan GPON RCE; CVE-2018-10562 - // body := bytes.NewBufferString(`{"dest_host": "ping"}`) - // r = httptest.NewRequest(http.MethodPost, "http://www.example.com/gponform/diag_form", body) - // r = r.WithContext(ctx) - // r.Header.Set("User-Agent", "test-appsec") - // r.Header.Set("Content-Type", "application/json") - // err = crowdsec.CheckRequest(ctx, r) - // assert.Error(t, err) + // simulate a request exploiting Ivanti EPM - SQLi; CVE-2024-29824. + body := bytes.NewBufferString("blabla 'xp_cmdshell' blabla") + r = httptest.NewRequest(http.MethodPost, "http://www.example.com/wsstatusevents/eventhandler.asmx", body) + r = r.WithContext(ctx) + r.Header.Set("User-Agent", "test-appsec") + err = crowdsec.CheckRequest(ctx, r) + assert.Error(t, err) + + // simulate a request exploiting Dasan GPON RCE; CVE-2018-10562 + data := url.Values{} + data.Set("dest_host", "\\`arg\\`;bla") + r = httptest.NewRequest(http.MethodPost, "http://www.example.com/gponform/diag_form", strings.NewReader(data.Encode())) + r = r.WithContext(ctx) + r.Header.Set("User-Agent", "test-appsec") + r.Header.Set("Content-Type", "application/x-www-form-urlencoded") + err = crowdsec.CheckRequest(ctx, r) + assert.Error(t, err) // simulate a request exploiting WooCommerce auth bypass; CVE-2023-28121, ensuring // that headers are passed correctly. - body := bytes.NewBufferString("some body") + body = bytes.NewBufferString("some body") r = httptest.NewRequest(http.MethodPost, "http://www.example.com", body) r = r.WithContext(ctx) r.Header.Set("User-Agent", "test-appsec") From 98598e10cc13b8054da27fd2635f0e0e342dbd90 Mon Sep 17 00:00:00 2001 From: Herman Slatman Date: Fri, 29 Nov 2024 12:22:15 +0100 Subject: [PATCH 10/11] Amend comment regarding AppSec POST request content length --- internal/bouncer/appsec.go | 10 ++++++---- internal/testutils/testutils.go | 12 ++++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/internal/bouncer/appsec.go b/internal/bouncer/appsec.go index 12df4fd5..d45a6a17 100644 --- a/internal/bouncer/appsec.go +++ b/internal/bouncer/appsec.go @@ -103,10 +103,12 @@ func (a *appsec) checkRequest(ctx context.Context, r *http.Request) error { req.Header.Set("X-Crowdsec-Appsec-User-Agent", r.Header.Get("User-Agent")) req.Header.Set("User-Agent", userAgentName) - // explicitly setting the content length results in CrowdSec properly accepting - // the request body. Without this the Content-Length header won't be set to the - // correct value, resulting in CrowdSec skipping its evaluation. We should test - // whether https://github.com/crowdsecurity/crowdsec/pull/3342 makes it work. + // explicitly setting the content length results in CrowdSec (1.6.4) properly + // accepting the request body. Without this the Content-Length header won't be + // set to the correct value, resulting in CrowdSec skipping its evaluation. The + // PR at https://github.com/crowdsecurity/crowdsec/pull/3342 makes it work, but + // that's not merged yet, and will thus require the release of CrowdSec that + // includes the patch. req.ContentLength = int64(contentLength) totalAppSecCalls.Inc() diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go index 8f31bab3..63512428 100644 --- a/internal/testutils/testutils.go +++ b/internal/testutils/testutils.go @@ -20,7 +20,11 @@ import ( "github.com/hslatman/caddy-crowdsec-bouncer/crowdsec" ) -const testAPIKey = "testbouncer1key" +const ( + containerImage = "crowdsecurity/crowdsec:slim" + //containerImage = "crowdsec-local" + testAPIKey = "testbouncer1key" +) type container struct { c testcontainers.Container @@ -32,7 +36,7 @@ func NewCrowdSecContainer(t *testing.T, ctx context.Context) *container { t.Helper() c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: testcontainers.ContainerRequest{ - Image: "crowdsecurity/crowdsec:slim", + Image: containerImage, ExposedPorts: []string{"8080/tcp"}, WaitingFor: wait.ForLog("CrowdSec Local API listening on 0.0.0.0:8080"), Env: map[string]string{ @@ -104,7 +108,7 @@ func NewAppSecContainer(t *testing.T, ctx context.Context) *container { // a container, installing the required collections, and then stopping it again. initContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: testcontainers.ContainerRequest{ - Image: "crowdsecurity/crowdsec:slim", + Image: containerImage, Mounts: mounts, ExposedPorts: []string{"8080/tcp"}, WaitingFor: wait.ForLog("CrowdSec Local API listening on 0.0.0.0:8080"), @@ -144,7 +148,7 @@ func NewAppSecContainer(t *testing.T, ctx context.Context) *container { // create the actual AppSec container c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: testcontainers.ContainerRequest{ - Image: "crowdsecurity/crowdsec:slim", + Image: containerImage, Mounts: mounts, ExposedPorts: []string{"8080/tcp", "7422/tcp"}, WaitingFor: wait.ForLog("Appsec Runner ready to process event"), From f7ebfa3244df1940f17f48ff5a2f855d6dd324b7 Mon Sep 17 00:00:00 2001 From: Herman Slatman Date: Fri, 29 Nov 2024 15:55:34 +0100 Subject: [PATCH 11/11] Support setting a maximum AppSec request body size --- crowdsec/caddyfile.go | 10 +++ crowdsec/crowdsec.go | 5 +- internal/bouncer/appsec.go | 30 +++++--- internal/bouncer/appsec_test.go | 128 +++++++++++++++++++++++++++++++ internal/bouncer/bouncer.go | 4 +- internal/bouncer/bouncer_test.go | 2 +- 6 files changed, 164 insertions(+), 15 deletions(-) create mode 100644 internal/bouncer/appsec_test.go diff --git a/crowdsec/caddyfile.go b/crowdsec/caddyfile.go index 56973648..35adbaef 100644 --- a/crowdsec/caddyfile.go +++ b/crowdsec/caddyfile.go @@ -3,6 +3,7 @@ package crowdsec import ( "fmt" "net/url" + "strconv" "strings" "time" @@ -75,6 +76,15 @@ func parseCrowdSec(d *caddyfile.Dispenser, existingVal any) (any, error) { return nil, d.ArgErr() } cs.AppSecUrl = d.Val() + case "appsec_max_body_bytes": + if !d.NextArg() { + return nil, d.ArgErr() + } + v, err := strconv.Atoi(d.Val()) + if err != nil { + return nil, d.Errf("invalid maximum number of bytes %q: %v", d.Val(), err) + } + cs.AppSecMaxBodySize = v default: return nil, d.Errf("invalid configuration token %q provided", d.Val()) } diff --git a/crowdsec/crowdsec.go b/crowdsec/crowdsec.go index be626f02..67fa9000 100644 --- a/crowdsec/crowdsec.go +++ b/crowdsec/crowdsec.go @@ -72,6 +72,9 @@ type CrowdSec struct { // AppSecUrl is the URL of the AppSec component served by your // CrowdSec installation. Disabled by default. AppSecUrl string `json:"appsec_url,omitempty"` + // AppSecMaxBodySize is the maximum number of request body bytes that + // will be sent to your AppSec component. + AppSecMaxBodySize int `json:"appsec_max_body_bytes,omitempty"` ctx caddy.Context logger *zap.Logger @@ -97,7 +100,7 @@ func (c *CrowdSec) Provision(ctx caddy.Context) error { c.TickerInterval = "60s" } - bouncer, err := bouncer.New(c.APIKey, c.APIUrl, c.AppSecUrl, c.TickerInterval, c.logger) + bouncer, err := bouncer.New(c.APIKey, c.APIUrl, c.AppSecUrl, c.AppSecMaxBodySize, c.TickerInterval, c.logger) if err != nil { return err } diff --git a/internal/bouncer/appsec.go b/internal/bouncer/appsec.go index d45a6a17..5d32c0fc 100644 --- a/internal/bouncer/appsec.go +++ b/internal/bouncer/appsec.go @@ -17,18 +17,20 @@ import ( ) type appsec struct { - apiURL string - apiKey string - logger *zap.Logger - client *http.Client - pool *bpool.BufferPool + apiURL string + apiKey string + maxBodySize int + logger *zap.Logger + client *http.Client + pool *bpool.BufferPool } -func newAppSec(apiURL, apiKey string, logger *zap.Logger) *appsec { +func newAppSec(apiURL, apiKey string, maxBodySize int, logger *zap.Logger) *appsec { return &appsec{ - apiURL: apiURL, - apiKey: apiKey, - logger: logger, + apiURL: apiURL, + apiKey: apiKey, + maxBodySize: maxBodySize, + logger: logger, client: &http.Client{ Timeout: 10 * time.Second, Transport: &http.Transport{ @@ -75,11 +77,17 @@ func (a *appsec) checkRequest(ctx context.Context, r *http.Request) error { buffer := a.pool.Get() defer a.pool.Put(buffer) - _, _ = buffer.Write(originalBody) + if a.maxBodySize > 0 { + len := min(len(originalBody), a.maxBodySize) + _, _ = buffer.Write(originalBody[:len]) + + } else { + _, _ = buffer.Write(originalBody) + } method = http.MethodPost - contentLength = buffer.Len() body = io.NopCloser(buffer) + contentLength = buffer.Len() // "reset" the original request body r.Body = io.NopCloser(bytes.NewBuffer(originalBody)) diff --git a/internal/bouncer/appsec_test.go b/internal/bouncer/appsec_test.go new file mode 100644 index 00000000..239ffe05 --- /dev/null +++ b/internal/bouncer/appsec_test.go @@ -0,0 +1,128 @@ +package bouncer + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" + + "github.com/hslatman/caddy-crowdsec-bouncer/internal/httputils" +) + +func newCaddyVarsContext() (ctx context.Context) { + ctx = context.WithValue(context.Background(), caddyhttp.VarsCtxKey, map[string]any{}) + return +} + +func Test_appsec_checkRequest(t *testing.T) { + logger := zaptest.NewLogger(t) + ctx := newCaddyVarsContext() + caddyhttp.SetVar(ctx, caddyhttp.ClientIPVarKey, "10.0.0.10") + ctx, _ = httputils.EnsureIP(ctx) + noIPCtx := newCaddyVarsContext() + + noIPRequest := httptest.NewRequest(http.MethodGet, "/path", http.NoBody) + noIPRequest.Header.Set("User-Agent", "test-appsec") + + okGetRequest := httptest.NewRequest(http.MethodGet, "/path", http.NoBody) + okGetRequest.Header.Set("User-Agent", "test-appsec") + + okPostRequest := httptest.NewRequest(http.MethodPost, "/path", bytes.NewBufferString("body")) + okPostRequest.Header.Set("User-Agent", "test-appsec") + + // TODO: add test for no connection; reading error? + // TODO: add assertions for responses and how they're handled + type fields struct { + maxBodySize int + } + type args struct { + ctx context.Context + r *http.Request + } + tests := []struct { + name string + fields fields + args args + expectedMethod string + expectedBody []byte + wantErr bool + }{ + { + name: "ok get", + args: args{ + ctx: ctx, + r: okGetRequest, + }, + expectedMethod: "GET", + }, + { + name: "ok post", + args: args{ + ctx: ctx, + r: okPostRequest, + }, + expectedMethod: "POST", + expectedBody: []byte("body"), + }, + { + name: "ok post limit", + fields: fields{ + maxBodySize: 1, + }, + args: args{ + ctx: ctx, + r: okPostRequest, + }, + expectedMethod: "POST", + expectedBody: []byte("b"), + }, + { + name: "fail ip", + args: args{ + ctx: noIPCtx, + r: noIPRequest, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := http.NewServeMux() + h.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "caddy-cs-bouncer", r.Header.Get("User-Agent")) + assert.Equal(t, "test-appsec", r.Header.Get("X-Crowdsec-Appsec-User-Agent")) + assert.Equal(t, "10.0.0.10", r.Header.Get("X-Crowdsec-Appsec-Ip")) + assert.Equal(t, "/path", r.Header.Get("X-Crowdsec-Appsec-Uri")) + assert.Equal(t, "example.com", r.Header.Get("X-Crowdsec-Appsec-Host")) + assert.Equal(t, tt.expectedMethod, r.Header.Get("X-Crowdsec-Appsec-Verb")) + assert.Equal(t, "test-apikey", r.Header.Get("X-Crowdsec-Appsec-Api-Key")) + + if r.Method == http.MethodPost { + b, err := io.ReadAll(r.Body) + require.NoError(t, err) + assert.Equal(t, tt.expectedBody, b) + assert.Equal(t, len(tt.expectedBody), int(r.ContentLength)) + } + }) + + s := httptest.NewServer(h) + t.Cleanup(s.Close) + + a := newAppSec(s.URL, "test-apikey", tt.fields.maxBodySize, logger) + err := a.checkRequest(tt.args.ctx, tt.args.r) + if tt.wantErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + }) + } +} diff --git a/internal/bouncer/bouncer.go b/internal/bouncer/bouncer.go index 0fa360e7..7c915f0e 100644 --- a/internal/bouncer/bouncer.go +++ b/internal/bouncer/bouncer.go @@ -65,7 +65,7 @@ type Bouncer struct { } // New creates a new (streaming) Bouncer with a storage based on immutable radix tree -func New(apiKey, apiURL, appSecURL, tickerInterval string, logger *zap.Logger) (*Bouncer, error) { +func New(apiKey, apiURL, appSecURL string, appSecMaxBodySize int, tickerInterval string, logger *zap.Logger) (*Bouncer, error) { insecureSkipVerify := false instantiatedAt := time.Now() instanceID, err := generateInstanceID(instantiatedAt) @@ -88,7 +88,7 @@ func New(apiKey, apiURL, appSecURL, tickerInterval string, logger *zap.Logger) ( InsecureSkipVerify: &insecureSkipVerify, UserAgent: userAgent, }, - appsec: newAppSec(appSecURL, apiKey, logger.Named("appsec")), + appsec: newAppSec(appSecURL, apiKey, appSecMaxBodySize, logger.Named("appsec")), store: newStore(), logger: logger, instantiatedAt: instantiatedAt, diff --git a/internal/bouncer/bouncer_test.go b/internal/bouncer/bouncer_test.go index 196ef827..751a655c 100644 --- a/internal/bouncer/bouncer_test.go +++ b/internal/bouncer/bouncer_test.go @@ -24,7 +24,7 @@ func newBouncer(t *testing.T) (*Bouncer, error) { tickerInterval := "10s" logger := zaptest.NewLogger(t) - bouncer, err := New(key, host, "", tickerInterval, logger) + bouncer, err := New(key, host, "", 0, tickerInterval, logger) require.NoError(t, err) bouncer.EnableStreaming()