fix(httpclientx): add support for cloudfronting (#1577)

I originally thought about dropping support for cloudfronting given that
we are not using it anymore. I wonder why.

It's a few lines of code (including testing) and it seems a good idea to
keep supporting it.

Part of ooni/probe#2723

internal/httpclientx/config.go

@@ -12,6 +12,9 @@ type Config struct {
 	// Client is the MANDATORY [model.HTTPClient] to use.
 	Client model.HTTPClient
 
+	// Host is the OPTIONAL Host header to use to implement cloudfronting.
+	Host string
+
 	// Logger is the MANDATORY [model.Logger] to use.
 	Logger model.Logger
 

internal/httpclientx/getjson_test.go

@@ -107,13 +107,15 @@ func TestGetJSON(t *testing.T) {
 // This test ensures that GetJSON sets correct HTTP headers
 func TestGetJSONHeadersOkay(t *testing.T) {
 	var (
+		gothost    string
 		gotheaders http.Header
 		gotmu      sync.Mutex
 	)
 
 	server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// save the headers
 		gotmu.Lock()
+		gothost = r.Host
 		gotheaders = r.Header
 		gotmu.Unlock()
 
@@ -127,6 +129,7 @@ func TestGetJSONHeadersOkay(t *testing.T) {
 	apiresp, err := GetJSON[*apiResponse](context.Background(), server.URL, &Config{
 		Authorization: "scribai",
 		Client:        http.DefaultClient,
+		Host:          "www.cloudfront.com",
 		Logger:        model.DiscardLogger,
 		UserAgent:     model.HTTPHeaderUserAgent,
 	})
@@ -159,6 +162,11 @@ func TestGetJSONHeadersOkay(t *testing.T) {
 	if value := gotheaders.Get("Accept-Encoding"); value != "gzip" {
 		t.Fatal("unexpected Accept-Encoding value", value)
 	}
+
+	// now make sure we could use cloudfronting
+	if gothost != "www.cloudfront.com" {
+		t.Fatal("unexpected Host value", gothost)
+	}
 }
 
 // This test ensures GetJSON logs the response body at Debug level.

internal/httpclientx/getraw_test.go

@@ -69,13 +69,15 @@ func TestGetRaw(t *testing.T) {
 // This test ensures that GetRaw sets correct HTTP headers
 func TestGetRawHeadersOkay(t *testing.T) {
 	var (
+		gothost    string
 		gotheaders http.Header
 		gotmu      sync.Mutex
 	)
 
 	server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// save the headers
 		gotmu.Lock()
+		gothost = r.Host
 		gotheaders = r.Header
 		gotmu.Unlock()
 
@@ -89,6 +91,7 @@ func TestGetRawHeadersOkay(t *testing.T) {
 	rawresp, err := GetRaw(context.Background(), server.URL, &Config{
 		Authorization: "scribai",
 		Client:        http.DefaultClient,
+		Host:          "www.cloudfront.com",
 		Logger:        model.DiscardLogger,
 		UserAgent:     model.HTTPHeaderUserAgent,
 	})
@@ -121,6 +124,11 @@ func TestGetRawHeadersOkay(t *testing.T) {
 	if value := gotheaders.Get("Accept-Encoding"); value != "gzip" {
 		t.Fatal("unexpected Accept-Encoding value", value)
 	}
+
+	// now make sure we could use cloudfronting
+	if gothost != "www.cloudfront.com" {
+		t.Fatal("unexpected Host value", gothost)
+	}
 }
 
 // This test ensures GetRaw logs the response body at Debug level.

internal/httpclientx/getxml_test.go

@@ -103,13 +103,15 @@ func TestGetXML(t *testing.T) {
 // This test ensures that GetXML sets correct HTTP headers
 func TestGetXMLHeadersOkay(t *testing.T) {
 	var (
+		gothost    string
 		gotheaders http.Header
 		gotmu      sync.Mutex
 	)
 
 	server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// save the headers
 		gotmu.Lock()
+		gothost = r.Host
 		gotheaders = r.Header
 		gotmu.Unlock()
 
@@ -123,6 +125,7 @@ func TestGetXMLHeadersOkay(t *testing.T) {
 	apiresp, err := GetXML[*apiResponse](context.Background(), server.URL, &Config{
 		Authorization: "scribai",
 		Client:        http.DefaultClient,
+		Host:          "www.cloudfront.com",
 		Logger:        model.DiscardLogger,
 		UserAgent:     model.HTTPHeaderUserAgent,
 	})
@@ -155,6 +158,11 @@ func TestGetXMLHeadersOkay(t *testing.T) {
 	if value := gotheaders.Get("Accept-Encoding"); value != "gzip" {
 		t.Fatal("unexpected Accept-Encoding value", value)
 	}
+
+	// now make sure we could use cloudfronting
+	if gothost != "www.cloudfront.com" {
+		t.Fatal("unexpected Host value", gothost)
+	}
 }
 
 // This test ensures GetXML logs the response body at Debug level.

internal/httpclientx/httpclientx.go

@@ -53,6 +53,10 @@ func do(ctx context.Context, req *http.Request, config *Config) ([]byte, error)
 	// say that we're accepting gzip encoded bodies
 	req.Header.Set("Accept-Encoding", "gzip")
 
+	// OPTIONALLY allow for cloudfronting (the default in net/http is for
+	// the req.Host to be empty and to use req.URL.Host)
+	req.Host = config.Host
+
 	// get the response
 	resp, err := config.Client.Do(req)
 

internal/httpclientx/postjson_test.go

@@ -163,6 +163,7 @@ func TestPostJSON(t *testing.T) {
 // This test ensures that PostJSON sets correct HTTP headers and sends the right body.
 func TestPostJSONCommunicationOkay(t *testing.T) {
 	var (
+		gothost    string
 		gotheaders http.Header
 		gotrawbody []byte
 		gotmu      sync.Mutex
@@ -174,6 +175,7 @@ func TestPostJSONCommunicationOkay(t *testing.T) {
 
 		// save the raw response body and headers
 		gotmu.Lock()
+		gothost = r.Host
 		gotrawbody = rawbody
 		gotheaders = r.Header
 		gotmu.Unlock()
@@ -194,6 +196,7 @@ func TestPostJSONCommunicationOkay(t *testing.T) {
 	apiresp, err := PostJSON[*apiRequest, *apiResponse](context.Background(), server.URL, apireq, &Config{
 		Authorization: "scribai",
 		Client:        http.DefaultClient,
+		Host:          "www.cloudfront.com",
 		Logger:        model.DiscardLogger,
 		UserAgent:     model.HTTPHeaderUserAgent,
 	})
@@ -236,6 +239,11 @@ func TestPostJSONCommunicationOkay(t *testing.T) {
 	if value := gotheaders.Get("Accept-Encoding"); value != "gzip" {
 		t.Fatal("unexpected Accept-Encoding value", value)
 	}
+
+	// now make sure we could use cloudfronting
+	if gothost != "www.cloudfront.com" {
+		t.Fatal("unexpected Host value", gothost)
+	}
 }
 
 // This test ensures PostJSON logs the request and response body at Debug level.