From 815bf2b2fcbb8761d099a7c709a3143d8bc79ac4 Mon Sep 17 00:00:00 2001 From: "J. Emrys Landivar" Date: Fri, 9 Jul 2021 16:51:09 -0500 Subject: [PATCH] feat: flag to allow blacklisting CIDR ranges (#2589) * feat: flag to allow blacklisting CIDR ranges --- client/v1/client.go | 5 ++ cmd/kapacitord/run/command.go | 26 +++++++- http/transport.go | 112 +++++++++++++++++++++++++++++--- influxdb/client.go | 22 ++++++- services/alerta/service.go | 9 +-- services/bigpanda/service.go | 8 +-- services/discord/service.go | 12 +--- services/httppost/service.go | 4 +- services/influxdb/service.go | 2 +- services/k8s/client/client.go | 8 +-- services/load/service.go | 7 +- services/sideload/service.go | 6 +- services/slack/service.go | 12 +--- services/swarm/client/client.go | 8 +-- 14 files changed, 175 insertions(+), 66 deletions(-) diff --git a/client/v1/client.go b/client/v1/client.go index 3f85b066f..8f09592d9 100644 --- a/client/v1/client.go +++ b/client/v1/client.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "io/ioutil" + "net" "net/http" "net/http/httptest" "net/url" @@ -450,6 +451,10 @@ func New(conf Config) (*Client, error) { if rt == nil { tr = khttp.NewDefaultTransportWithTLS(&tls.Config{ InsecureSkipVerify: conf.InsecureSkipVerify, + }, &net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + Control: khttp.Control(khttp.DefaultValidator), }) if conf.TLSConfig != nil { tr.TLSClientConfig = conf.TLSConfig diff --git a/cmd/kapacitord/run/command.go b/cmd/kapacitord/run/command.go index 2df3d02ba..72de4ba55 100644 --- a/cmd/kapacitord/run/command.go +++ b/cmd/kapacitord/run/command.go @@ -12,7 +12,9 @@ import ( "strings" "github.com/BurntSushi/toml" + furl "github.com/influxdata/flux/dependencies/url" "github.com/influxdata/flux/fluxinit" + khttp "github.com/influxdata/kapacitor/http" "github.com/influxdata/kapacitor/server" "github.com/influxdata/kapacitor/services/diagnostic" ) @@ -104,6 +106,18 @@ func (cmd *Command) Run(args ...string) error { if options.LogLevel != "" { config.Logging.Level = options.LogLevel } + + switch options.BlackListCIDRS { + case "": + khttp.DefaultValidator = furl.PassValidator{} + case "private": + khttp.DefaultValidator = furl.PrivateIPValidator{} + default: + if khttp.DefaultValidator, err = khttp.ParseCIDRsString(options.BlackListCIDRS); err != nil { + return fmt.Errorf("flag error: improper CIDRs: %s", err) + } + } + // Initialize Logging Services cmd.diagService = diagnostic.NewService(config.Logging, cmd.Stdout, cmd.Stderr) if err := cmd.diagService.Open(); err != nil { @@ -197,6 +211,7 @@ func (cmd *Command) ParseFlags(args ...string) (Options, error) { fs.StringVar(&options.MemProfile, "memprofile", "", "") fs.StringVar(&options.LogFile, "log-file", "", "") fs.StringVar(&options.LogLevel, "log-level", "", "") + fs.StringVar(&options.BlackListCIDRS, "blacklist-cidrs", "", "") fs.StringVar(&options.DisabledAlertHandlers, "disable-handlers", "", "") fs.Usage = func() { fmt.Fprintln(cmd.Stderr, usage) } if err := fs.Parse(args); err != nil { @@ -248,10 +263,18 @@ func (cmd *Command) ParseConfig(path string) (*server.Config, error) { var usage = `usage: run [flags] run starts the Kapacitor server. + -blacklist-cidrs + Comma seperated list of CIDRs to blacklist for + most http get/post operations -config Set the path to the configuration file. + -disable-handlers + Disables certain alert handlers. This is useful for + security, reasons. For example: disabling exec on + a shared system. + -hostname Override the hostname, the 'hostname' configuration option will be overridden. @@ -265,8 +288,6 @@ run starts the Kapacitor server. -log-level Sets the log level. One of debug,info,error. - -disable-handlers - Disables certain alert handlers. This is useful for security, reasons. For example: disabling exec on a shared system. ` // Options represents the command line options that can be parsed. @@ -279,4 +300,5 @@ type Options struct { LogFile string LogLevel string DisabledAlertHandlers string + BlackListCIDRS string } diff --git a/http/transport.go b/http/transport.go index 4a4ce7465..35ed1f86b 100644 --- a/http/transport.go +++ b/http/transport.go @@ -2,21 +2,32 @@ package http import ( "crypto/tls" + "fmt" "net" "net/http" + "net/url" + "strings" + "syscall" "time" + + furl "github.com/influxdata/flux/dependencies/url" ) // NewDefaultTransport creates a new transport with sane defaults. -func NewDefaultTransport() *http.Transport { - // These defaults are copied from http.DefaultTransport. - return &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ +func NewDefaultTransport(dialer *net.Dialer) *http.Transport { + + if dialer == nil { + dialer = &net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, DualStack: true, - }).DialContext, + } + } + + // These defaults are copied from http.DefaultTransport. + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (dialer).DialContext, MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, @@ -27,8 +38,93 @@ func NewDefaultTransport() *http.Transport { } // NewDefaultTransportWithTLS creates a new transport with the specified TLS configuration. -func NewDefaultTransportWithTLS(tlsConfig *tls.Config) *http.Transport { - t := NewDefaultTransport() +func NewDefaultTransportWithTLS(tlsConfig *tls.Config, dialer *net.Dialer) *http.Transport { + t := NewDefaultTransport(dialer) t.TLSClientConfig = tlsConfig return t } + +// Control is called after DNS lookup, but before the network connection is +// initiated. +func Control(urlValidator furl.Validator) func(network, address string, c syscall.RawConn) error { + return func(network, address string, c syscall.RawConn) error { + host, _, err := net.SplitHostPort(address) + if err != nil { + return err + } + + ip := net.ParseIP(host) + return urlValidator.ValidateIP(ip) + } +} + +// NewDefaultClientWithTLS creates a tls client with sane defaults. +func NewDefaultClientWithTLS(tlsConfig *tls.Config, urlValidator furl.Validator) *http.Client { + + // These defaults are copied from http.DefaultTransport. + return &http.Client{ + Transport: NewDefaultTransportWithTLS(tlsConfig, &net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + Control: Control(urlValidator), + }), + Timeout: 30 * time.Second, + } +} + +// NewDefaultClient creates a client with sane defaults. +func NewDefaultClient(urlValidator furl.Validator) *http.Client { + + // These defaults are copied from http.DefaultTransport. + return &http.Client{ + Transport: NewDefaultTransport(&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + Control: Control(urlValidator), + // DualStack is deprecated + }), + } +} + +// DefaultValidator is the default validator, it can be replaced at start time with a different validator +var DefaultValidator furl.Validator = furl.PassValidator{} + +type cidrValidator struct { + cidrs []*net.IPNet +} + +func (v cidrValidator) Validate(u *url.URL) error { + ips, err := net.LookupIP(u.Hostname()) + if err != nil { + return err + } + for _, ip := range ips { + err = v.ValidateIP(ip) + if err != nil { + return err + } + } + return nil +} + +func (v cidrValidator) ValidateIP(ip net.IP) error { + for i := range v.cidrs { + if v.cidrs[i].Contains(ip) { + return fmt.Errorf("ip '%s' is blacklisted", ip) + } + } + return nil +} + +func ParseCIDRsString(s string) (furl.Validator, error) { + cidrStrings := strings.Split(s, ",") + cidrs := make([]*net.IPNet, 0, len(cidrStrings)) + for i := range cidrStrings { + _, cidr, err := net.ParseCIDR(cidrStrings[i]) + if err != nil { + return nil, err + } + cidrs = append(cidrs, cidr) + } + return cidrValidator{cidrs: cidrs}, nil +} diff --git a/influxdb/client.go b/influxdb/client.go index 3a5398d10..108de1d34 100644 --- a/influxdb/client.go +++ b/influxdb/client.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "io/ioutil" + "net" "net/http" "net/url" "strconv" @@ -157,8 +158,21 @@ func NewHTTPClient(conf Config) (*HTTPClient, error) { return nil, errors.Wrap(err, "invalid URLs") } if conf.Transport == nil { - conf.Transport = khttp.NewDefaultTransport() + conf.Transport = khttp.NewDefaultTransport(&net.Dialer{ + Timeout: 30 * time.Second, // I am not sure if this is the right value to set it to + KeepAlive: 30 * time.Second, // I am not sure if this is the right value to set it to + Control: khttp.Control(khttp.DefaultValidator), + // DualStack is deprecated + }) } + + conf.Transport.DialContext = (&net.Dialer{ + Timeout: 30 * time.Second, // I am not sure if this is the right value to set it to + KeepAlive: 30 * time.Second, // I am not sure if this is the right value to set it to + Control: khttp.Control(khttp.DefaultValidator), + // DualStack is deprecated + }).DialContext + c := &HTTPClient{ config: conf, urls: urls, @@ -243,6 +257,12 @@ func (c *HTTPClient) Update(new Config) error { if tr == nil { tr = old.Transport } + tr.DialContext = (&net.Dialer{ + Timeout: 30 * time.Second, // I am not sure if this is the right value to set it to + KeepAlive: 30 * time.Second, // I am not sure if this is the right value to set it to + Control: khttp.Control(khttp.DefaultValidator), + // DualStack is deprecated + }).DialContext c.client = &http.Client{ Timeout: new.Timeout, Transport: tr, diff --git a/services/alerta/service.go b/services/alerta/service.go index c4e1e635a..d82f5cd12 100644 --- a/services/alerta/service.go +++ b/services/alerta/service.go @@ -45,9 +45,8 @@ func NewService(c Config, d Diagnostic) *Service { diag: d, } s.configValue.Store(c) - s.clientValue.Store(&http.Client{ - Transport: khttp.NewDefaultTransportWithTLS(&tls.Config{InsecureSkipVerify: c.InsecureSkipVerify}), - }) + s.clientValue.Store(khttp.NewDefaultClientWithTLS(&tls.Config{InsecureSkipVerify: c.InsecureSkipVerify}, khttp.DefaultValidator)) + return s } @@ -128,9 +127,7 @@ func (s *Service) Update(newConfig []interface{}) error { return fmt.Errorf("expected config object to be of type %T, got %T", c, newConfig[0]) } else { s.configValue.Store(c) - s.clientValue.Store(&http.Client{ - Transport: khttp.NewDefaultTransportWithTLS(&tls.Config{InsecureSkipVerify: c.InsecureSkipVerify}), - }) + s.clientValue.Store(khttp.NewDefaultClientWithTLS(&tls.Config{InsecureSkipVerify: c.InsecureSkipVerify}, khttp.DefaultValidator)) } return nil diff --git a/services/bigpanda/service.go b/services/bigpanda/service.go index 6d5bbe0c9..94eaa0caa 100644 --- a/services/bigpanda/service.go +++ b/services/bigpanda/service.go @@ -40,9 +40,7 @@ func NewService(c Config, d Diagnostic) (*Service, error) { diag: d, } s.configValue.Store(c) - s.clientValue.Store(&http.Client{ - Transport: khttp.NewDefaultTransportWithTLS(&tls.Config{InsecureSkipVerify: c.InsecureSkipVerify}), - }) + s.clientValue.Store(khttp.NewDefaultClientWithTLS(&tls.Config{InsecureSkipVerify: c.InsecureSkipVerify}, khttp.DefaultValidator)) return s, nil } @@ -67,9 +65,7 @@ func (s *Service) Update(newConfig []interface{}) error { return fmt.Errorf("expected config object to be of type %T, got %T", c, newConfig[0]) } else { s.configValue.Store(c) - s.clientValue.Store(&http.Client{ - Transport: khttp.NewDefaultTransportWithTLS(&tls.Config{InsecureSkipVerify: c.InsecureSkipVerify}), - }) + s.clientValue.Store(khttp.NewDefaultClientWithTLS(&tls.Config{InsecureSkipVerify: c.InsecureSkipVerify}, khttp.DefaultValidator)) } return nil } diff --git a/services/discord/service.go b/services/discord/service.go index c189e27e9..83a0b4b05 100644 --- a/services/discord/service.go +++ b/services/discord/service.go @@ -39,13 +39,9 @@ func NewWorkspace(c Config) (*Workspace, error) { return nil, err } - cl := &http.Client{ - Transport: khttp.NewDefaultTransportWithTLS(tlsConfig), - } - return &Workspace{ config: c, - client: cl, + client: khttp.NewDefaultClientWithTLS(tlsConfig, khttp.DefaultValidator), }, nil } @@ -67,11 +63,7 @@ func (w *Workspace) Update(c Config) error { return err } - cl := &http.Client{ - Transport: khttp.NewDefaultTransportWithTLS(tlsConfig), - } - - w.client = cl + w.client = khttp.NewDefaultClientWithTLS(tlsConfig, khttp.DefaultValidator) w.config = c return nil diff --git a/services/httppost/service.go b/services/httppost/service.go index 63ea0d3eb..0ae1b6600 100644 --- a/services/httppost/service.go +++ b/services/httppost/service.go @@ -369,9 +369,7 @@ func (h *handler) Handle(event alert.Event) { } } - httpClient := &http.Client{ - Transport: khttp.NewDefaultTransportWithTLS(tlsConfig), - } + httpClient := khttp.NewDefaultClientWithTLS(tlsConfig, khttp.DefaultValidator) // Execute the request resp, err := httpClient.Do(req) diff --git a/services/influxdb/service.go b/services/influxdb/service.go index 65020a942..54746e959 100644 --- a/services/influxdb/service.go +++ b/services/influxdb/service.go @@ -504,7 +504,7 @@ func httpConfig(c Config) (influxdb.Config, error) { if err != nil { return influxdb.Config{}, errors.Wrap(err, "invalid TLS options") } - tr := khttp.NewDefaultTransportWithTLS(tlsConfig) + tr := khttp.NewDefaultTransportWithTLS(tlsConfig, nil) var credentials influxdb.Credentials if c.Token != "" { credentials = influxdb.Credentials{ diff --git a/services/k8s/client/client.go b/services/k8s/client/client.go index 89fe53dc8..e99ffb236 100644 --- a/services/k8s/client/client.go +++ b/services/k8s/client/client.go @@ -123,9 +123,7 @@ func New(c Config) (Client, error) { return &httpClient{ config: c, urls: urls, - client: &http.Client{ - Transport: khttp.NewDefaultTransportWithTLS(c.TLSConfig), - }, + client: khttp.NewDefaultClientWithTLS(c.TLSConfig, khttp.DefaultValidator), }, nil } @@ -165,9 +163,7 @@ func (c *httpClient) Update(new Config) error { c.urls = urls if old.TLSConfig != new.TLSConfig { - c.client = &http.Client{ - Transport: khttp.NewDefaultTransportWithTLS(new.TLSConfig), - } + c.client = khttp.NewDefaultClientWithTLS(new.TLSConfig, khttp.DefaultValidator) } return nil } diff --git a/services/load/service.go b/services/load/service.go index b3e69e1f9..39518841c 100644 --- a/services/load/service.go +++ b/services/load/service.go @@ -242,7 +242,6 @@ func (s *Service) load() error { if err != nil && !os.IsNotExist(err) { return err } - s.diag.Debug("loading tasks") err = s.loadTasks() if err != nil && !os.IsNotExist(err) { @@ -361,7 +360,6 @@ func (s *Service) loadTemplate(f string) error { if err != nil { return fmt.Errorf("failed to open file %v: %v", f, err) } - data, err := ioutil.ReadAll(file) if err != nil { return fmt.Errorf("failed to read file %v: %v", f, err) @@ -374,6 +372,7 @@ func (s *Service) loadTemplate(f string) error { l := s.cli.TemplateLink(id) task, _ := s.cli.Template(l, nil) if task.ID == "" { + o := client.CreateTemplateOptions{ ID: id, TICKscript: script, @@ -383,12 +382,13 @@ func (s *Service) loadTemplate(f string) error { return fmt.Errorf("failed to create template: %v", err) } } else { + o := client.UpdateTemplateOptions{ ID: id, TICKscript: script, } if _, err := s.cli.UpdateTemplate(l, o); err != nil { - return fmt.Errorf("failed to create template: %v", err) + return fmt.Errorf("failed to update template: %v", err) } } @@ -398,7 +398,6 @@ func (s *Service) loadTemplate(f string) error { return err } s.templates[id] = true - return nil } diff --git a/services/sideload/service.go b/services/sideload/service.go index 177666a30..5158e10a1 100644 --- a/services/sideload/service.go +++ b/services/sideload/service.go @@ -14,6 +14,7 @@ import ( "time" "github.com/ghodss/yaml" + khttp "github.com/influxdata/kapacitor/http" "github.com/influxdata/kapacitor/keyvalue" "github.com/influxdata/kapacitor/services/httpd" "github.com/influxdata/kapacitor/services/httppost" @@ -272,9 +273,8 @@ func (s *httpSource) UpdateCache() error { req.SetBasicAuth(s.e.Auth.Username, s.e.Auth.Password) } - client := &http.Client{ - Timeout: time.Second * 10, - } + client := khttp.NewDefaultClient(khttp.DefaultValidator) + client.Timeout = time.Second * 10 resp, err := client.Do(req) if err != nil { return err diff --git a/services/slack/service.go b/services/slack/service.go index a36c1104c..bc72c9d06 100644 --- a/services/slack/service.go +++ b/services/slack/service.go @@ -36,13 +36,9 @@ func NewWorkspace(c Config) (*Workspace, error) { return nil, err } - cl := &http.Client{ - Transport: khttp.NewDefaultTransportWithTLS(tlsConfig), - } - return &Workspace{ config: c, - client: cl, + client: khttp.NewDefaultClientWithTLS(tlsConfig, khttp.DefaultValidator), }, nil } @@ -64,11 +60,7 @@ func (w *Workspace) Update(c Config) error { return err } - cl := &http.Client{ - Transport: khttp.NewDefaultTransportWithTLS(tlsConfig), - } - - w.client = cl + w.client = khttp.NewDefaultClientWithTLS(tlsConfig, khttp.DefaultValidator) w.config = c return nil diff --git a/services/swarm/client/client.go b/services/swarm/client/client.go index d50f104e5..79b1ea94d 100644 --- a/services/swarm/client/client.go +++ b/services/swarm/client/client.go @@ -52,9 +52,7 @@ func New(c Config) (Client, error) { return &httpClient{ config: c, urls: urls, - client: &http.Client{ - Transport: khttp.NewDefaultTransportWithTLS(c.TLSConfig), - }, + client: khttp.NewDefaultClientWithTLS(c.TLSConfig, khttp.DefaultValidator), }, nil } @@ -90,9 +88,7 @@ func (c *httpClient) Update(new Config) error { c.urls = urls if old.TLSConfig != new.TLSConfig { - c.client = &http.Client{ - Transport: khttp.NewDefaultTransportWithTLS(new.TLSConfig), - } + c.client = khttp.NewDefaultClientWithTLS(new.TLSConfig, khttp.DefaultValidator) } return nil }