⭐️ Make following redirects configurable for http.get resource (#5011)

providers/network/config/config.go

@@ -6,6 +6,7 @@ package config
 import (
 	"go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory"
 	"go.mondoo.com/cnquery/v11/providers-sdk/v1/plugin"
+	"go.mondoo.com/cnquery/v11/providers/network/connection"
 	"go.mondoo.com/cnquery/v11/providers/network/provider"
 )
 
@@ -42,6 +43,12 @@ var Config = plugin.Provider{
 					Default: "",
 					Desc:    "Disable TLS/SSL verification",
 				},
+				{
+					Long:    connection.OPTION_FOLLOW_REDIRECTS,
+					Type:    plugin.FlagType_Bool,
+					Default: "",
+					Desc:    "Follow HTTP redirects",
+				},
 			},
 		},
 	},

providers/network/connection/connection.go

@@ -13,11 +13,16 @@ import (
 	"go.mondoo.com/cnquery/v11/providers-sdk/v1/plugin"
 )
 
+const (
+	OPTION_FOLLOW_REDIRECTS = "follow-redirects"
+)
+
 type HostConnection struct {
 	plugin.Connection
-	Conf       *inventory.Config
-	asset      *inventory.Asset
-	httpClient *http.Client
+	Conf            *inventory.Config
+	FollowRedirects bool
+	asset           *inventory.Asset
+	transport       *http.Transport
 }
 
 func NewHostConnection(id uint32, asset *inventory.Asset, conf *inventory.Config) *HostConnection {
@@ -40,11 +45,17 @@ func NewHostConnection(id uint32, asset *inventory.Asset, conf *inventory.Config
 		}
 	}
 
+	var followRedirects bool
+	if followRedirectsStr, ok := conf.Options[OPTION_FOLLOW_REDIRECTS]; ok {
+		followRedirects = followRedirectsStr == "true"
+	}
+
 	return &HostConnection{
-		Connection: plugin.NewConnection(id, asset),
-		Conf:       conf,
-		asset:      asset,
-		httpClient: &http.Client{Transport: transport},
+		Connection:      plugin.NewConnection(id, asset),
+		Conf:            conf,
+		asset:           asset,
+		transport:       transport,
+		FollowRedirects: followRedirects,
 	}
 }
 
@@ -63,6 +74,15 @@ func (p *HostConnection) FQDN() string {
 	return p.Conf.Host
 }
 
-func (p *HostConnection) Client() *http.Client {
-	return p.httpClient
+func (p *HostConnection) Client(followRedirects bool) *http.Client {
+	c := &http.Client{
+		Transport: p.transport,
+	}
+
+	if !followRedirects {
+		c.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		}
+	}
+	return c
 }

providers/network/provider/provider.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"errors"
 	"net/url"
+	"strconv"
 	"strings"
 
 	"go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory"
@@ -44,6 +45,11 @@ func (s *Service) ParseCLI(req *plugin.ParseCLIReq) (*plugin.ParseCLIRes, error)
 		insecure, _ = found.RawData().Value.(bool)
 	}
 
+	options := map[string]string{}
+	if found, ok := req.Flags[connection.OPTION_FOLLOW_REDIRECTS]; ok {
+		options[connection.OPTION_FOLLOW_REDIRECTS] = strconv.FormatBool(found.RawData().Value.(bool))
+	}
+
 	asset := inventory.Asset{
 		Connections: []*inventory.Config{{
 			Type:     "host",
@@ -52,6 +58,7 @@ func (s *Service) ParseCLI(req *plugin.ParseCLIReq) (*plugin.ParseCLIRes, error)
 			Path:     path,
 			Runtime:  scheme,
 			Insecure: insecure,
+			Options:  options,
 		}},
 	}
 

providers/network/resources/http.go

@@ -75,13 +75,18 @@ func initHttpGet(runtime *plugin.Runtime, args map[string]*llx.RawData) (map[str
 			return nil, nil, err
 		}
 		args["url"] = llx.ResourceData(url, "url")
+		args["followRedirects"] = llx.BoolData(conn.FollowRedirects)
+	}
+
+	if _, ok := args["followRedirects"]; !ok {
+		args["followRedirects"] = llx.BoolData(false)
 	}
 
 	return args, nil, nil
 }
 
 func (x *mqlHttpGet) id() (string, error) {
-	return x.Url.Data.__id, nil
+	return strings.Join([]string{x.Url.Data.__id, strconv.FormatBool(x.FollowRedirects.Data)}, ";"), nil
 }
 
 func (x *mqlHttpGet) do() error {
@@ -97,7 +102,7 @@ func (x *mqlHttpGet) do() error {
 	}
 
 	conn := x.MqlRuntime.Connection.(*connection.HostConnection)
-	resp, err := conn.Client().Get(x.Url.Data.String.Data)
+	resp, err := conn.Client(x.FollowRedirects.Data).Get(x.Url.Data.String.Data)
 	x.resp.State = plugin.StateIsSet
 	x.resp.Data = resp
 	x.resp.Error = err

providers/network/resources/network.lr

@@ -19,9 +19,11 @@ http {}
 
 // HTTP GET requests
 http.get @defaults("url statusCode") {
-  init(rawUrl string)
+  init(rawUrl string, followRedirects bool)
   // URL for this request
   url url
+  // Follow redirects
+  followRedirects bool
   // Header returned from this request
   header() http.header
   // Status returned from this request

providers/network/resources/network.lr.manifest.yaml

@@ -91,6 +91,7 @@ resources:
   http.get:
     fields:
       body: {}
+      followRedirects: {}
       header: {}
       statusCode: {}
       url: {}

providers/network/resources/url.go

@@ -82,7 +82,8 @@ func (x *mqlUrl) string() (string, error) {
 	}
 
 	host := x.Host.Data
-	if x.Port.Data != 0 {
+	isStandardPort := x.Port.Data == 80 && x.Scheme.Data == "http" || x.Port.Data == 443 && x.Scheme.Data == "https"
+	if x.Port.Data != 0 && !isStandardPort {
 		host += ":" + strconv.Itoa(int(x.Port.Data))
 	}