fix(qatool): do not save large response bodies on disk (#1512)

It seems unnecessary in light of what we're currently testing. By doing this, we allow for much smaller commits. Tangentially related to ooni/probe#2677.
ooni · Feb 13, 2024 · a37f0d2 · a37f0d2
1 parent 595e167
commit a37f0d2
Show file tree

Hide file tree

Showing 37 changed files with 935 additions and 905 deletions.
diff --git a/internal/minipipeline/normalize.go b/internal/minipipeline/normalize.go
@@ -50,5 +50,14 @@ func NormalizeHTTPRequestResults(values []*model.ArchivalHTTPRequestResult) {
 	for _, entry := range values {
 		entry.T0 = 0
 		entry.T = 0
+
+		// Avoid storing large bodies because that wastes repository space.
+		//
+		// See https://github.com/ooni/probe/issues/2677.
+		const maxStoreBody = 1 << 14
+		if len(entry.Response.Body) > maxStoreBody {
+			entry.Response.Body = model.ArchivalScrubbedMaybeBinaryString("")
+			entry.Response.BodyIsTruncated = true
+		}
 	}
 }
diff --git a/internal/minipipeline/normalize_test.go b/internal/minipipeline/normalize_test.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/ooni/probe-cli/v3/internal/model"
+	"github.com/ooni/probe-cli/v3/internal/randx"
 )
 
 func TestNormalizeDNSLookupResults(t *testing.T) {
@@ -229,11 +230,35 @@ func TestNormalizeHTTPRequestResults(t *testing.T) {
 				T0: 0.11,
 				T:  0.4,
 			}, {
+				Response: model.ArchivalHTTPResponse{
+					Body: model.ArchivalScrubbedMaybeBinaryString("1234567"),
+				},
 				T0: 0.5,
 				T:  0.66,
+			}, {
+				Response: model.ArchivalHTTPResponse{
+					Body: model.ArchivalScrubbedMaybeBinaryString(randx.Letters(1 << 19)),
+				},
+				T0: 0.7,
+				T:  0.88,
 			}}
 		},
-		expect: []*model.ArchivalHTTPRequestResult{{}, {}},
+		expect: []*model.ArchivalHTTPRequestResult{
+			{
+				// empty
+			},
+			{
+				Response: model.ArchivalHTTPResponse{
+					Body: model.ArchivalScrubbedMaybeBinaryString("1234567"),
+				},
+			},
+			{
+				Response: model.ArchivalHTTPResponse{
+					Body:            model.ArchivalScrubbedMaybeBinaryString(""),
+					BodyIsTruncated: true,
+				},
+			},
+		},
 	}}
 
 	for _, tc := range cases {

diff --git a/...ata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/analysis.json b/...ata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/analysis.json
@@ -38,7 +38,7 @@
   "TCPConnectUnexplainedFailureDuringWebFetch": [],
   "TCPConnectUnexplainedFailureDuringConnectivityCheck": [],
   "TLSHandshakeExpectedFailure": [
-    40001
+    40002
   ],
   "TLSHandshakeUnexpectedFailure": [],
   "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
@@ -49,7 +49,7 @@
   "HTTPRoundTripUnexpectedFailure": [],
   "HTTPRoundTripUnexplainedFailure": [],
   "HTTPFinalResponseSuccessTLSWithoutControl": null,
-  "HTTPFinalResponseSuccessTLSWithControl": 40002,
+  "HTTPFinalResponseSuccessTLSWithControl": 40001,
   "HTTPFinalResponseSuccessTCPWithoutControl": null,
   "HTTPFinalResponseSuccessTCPWithControl": null,
   "HTTPFinalResponseDiffBodyProportionFactor": 1,
@@ -64,7 +64,7 @@
       "TagDepth": 0,
       "Type": 3,
       "Failure": "",
-      "TransactionID": 40002,
+      "TransactionID": 40001,
       "TagFetchBody": true,
       "DNSTransactionID": 20001,
       "DNSDomain": "www.example.com",
@@ -78,7 +78,7 @@
       "IPAddress": "93.184.216.34",
       "IPAddressASN": 15133,
       "IPAddressBogon": false,
-      "EndpointTransactionID": 40002,
+      "EndpointTransactionID": 40001,
       "EndpointProto": "tcp",
       "EndpointPort": "443",
       "EndpointAddress": "93.184.216.34:443",
@@ -121,7 +121,7 @@
       "TagDepth": 0,
       "Type": 2,
       "Failure": "ssl_unknown_authority",
-      "TransactionID": 40001,
+      "TransactionID": 40002,
       "TagFetchBody": true,
       "DNSTransactionID": 1,
       "DNSDomain": "www.example.com",
@@ -135,7 +135,7 @@
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
-      "EndpointTransactionID": 40001,
+      "EndpointTransactionID": 40002,
       "EndpointProto": "tcp",
       "EndpointPort": "443",
       "EndpointAddress": "104.154.89.105:443",

diff --git a/...onnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/analysis_classic.json b/...onnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/analysis_classic.json
@@ -30,7 +30,7 @@
   "TCPConnectUnexplainedFailureDuringWebFetch": [],
   "TCPConnectUnexplainedFailureDuringConnectivityCheck": [],
   "TLSHandshakeExpectedFailure": [
-    40001
+    40002
   ],
   "TLSHandshakeUnexpectedFailure": [],
   "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
@@ -53,7 +53,7 @@
       "TagDepth": 0,
       "Type": 2,
       "Failure": "ssl_unknown_authority",
-      "TransactionID": 40001,
+      "TransactionID": 40002,
       "TagFetchBody": true,
       "DNSTransactionID": 1,
       "DNSDomain": "www.example.com",
@@ -67,7 +67,7 @@
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
-      "EndpointTransactionID": 40001,
+      "EndpointTransactionID": 40002,
       "EndpointProto": "tcp",
       "EndpointPort": "443",
       "EndpointAddress": "104.154.89.105:443",

diff --git a/.../webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/measurement.json b/.../webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/measurement.json
@@ -218,7 +218,7 @@
           "depth=0",
           "fetch_body=true"
         ],
-        "transaction_id": 40002
+        "transaction_id": 40001
       }
     ],
     "tcp_connect": [
@@ -234,7 +234,7 @@
           "depth=0",
           "fetch_body=true"
         ],
-        "transaction_id": 40001
+        "transaction_id": 40002
       },
       {
         "ip": "93.184.216.34",
@@ -248,7 +248,7 @@
           "depth=0",
           "fetch_body=true"
         ],
-        "transaction_id": 40002
+        "transaction_id": 40001
       }
     ],
     "tls_handshakes": [
@@ -267,7 +267,7 @@
           "fetch_body=true"
         ],
         "tls_version": "",
-        "transaction_id": 40001
+        "transaction_id": 40002
       },
       {
         "network": "tcp",
@@ -284,7 +284,7 @@
           "fetch_body=true"
         ],
         "tls_version": "TLSv1.3",
-        "transaction_id": 40002
+        "transaction_id": 40001
       }
     ],
     "x_control_request": {
@@ -301,10 +301,10 @@
         ]
       },
       "tcp_connect": [
-        "104.154.89.105:443",
-        "104.154.89.105:80",
         "93.184.216.34:443",
-        "93.184.216.34:80"
+        "93.184.216.34:80",
+        "104.154.89.105:443",
+        "104.154.89.105:80"
       ],
       "x_quic_enabled": false
     },

diff --git a/...webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations.json b/...webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations.json
@@ -262,45 +262,50 @@
   "KnownTCPEndpoints": {
     "40001": {
       "TagDepth": 0,
-      "Type": 2,
-      "Failure": "ssl_unknown_authority",
+      "Type": 3,
+      "Failure": "",
       "TransactionID": 40001,
       "TagFetchBody": true,
-      "DNSTransactionID": 1,
+      "DNSTransactionID": 20001,
       "DNSDomain": "www.example.com",
       "DNSLookupFailure": "",
       "DNSQueryType": null,
       "DNSEngine": null,
       "DNSResolvedAddrs": [
-        "104.154.89.105"
+        "93.184.216.34"
       ],
       "IPAddressOrigin": "dns",
-      "IPAddress": "104.154.89.105",
-      "IPAddressASN": 396982,
+      "IPAddress": "93.184.216.34",
+      "IPAddressASN": 15133,
       "IPAddressBogon": false,
       "EndpointTransactionID": 40001,
       "EndpointProto": "tcp",
       "EndpointPort": "443",
-      "EndpointAddress": "104.154.89.105:443",
+      "EndpointAddress": "93.184.216.34:443",
       "TCPConnectFailure": "",
-      "TLSHandshakeFailure": "ssl_unknown_authority",
+      "TLSHandshakeFailure": "",
       "TLSServerName": "www.example.com",
-      "HTTPRequestURL": null,
-      "HTTPFailure": null,
-      "HTTPResponseStatusCode": null,
-      "HTTPResponseBodyLength": null,
-      "HTTPResponseBodyIsTruncated": null,
-      "HTTPResponseHeadersKeys": null,
+      "HTTPRequestURL": "https://www.example.com/",
+      "HTTPFailure": "",
+      "HTTPResponseStatusCode": 200,
+      "HTTPResponseBodyLength": 1533,
+      "HTTPResponseBodyIsTruncated": false,
+      "HTTPResponseHeadersKeys": {
+        "Alt-Svc": true,
+        "Content-Length": true,
+        "Content-Type": true,
+        "Date": true
+      },
       "HTTPResponseLocation": null,
-      "HTTPResponseTitle": null,
-      "HTTPResponseIsFinal": null,
+      "HTTPResponseTitle": "Default Web Page",
+      "HTTPResponseIsFinal": true,
       "ControlDNSDomain": "www.example.com",
       "ControlDNSLookupFailure": "",
       "ControlDNSResolvedAddrs": [
         "93.184.216.34"
       ],
       "ControlTCPConnectFailure": "",
-      "ControlTLSHandshakeFailure": "ssl_unknown_authority",
+      "ControlTLSHandshakeFailure": "",
       "ControlHTTPFailure": "",
       "ControlHTTPResponseStatusCode": 200,
       "ControlHTTPResponseBodyLength": 1533,
@@ -314,50 +319,45 @@
     },
     "40002": {
       "TagDepth": 0,
-      "Type": 3,
-      "Failure": "",
+      "Type": 2,
+      "Failure": "ssl_unknown_authority",
       "TransactionID": 40002,
       "TagFetchBody": true,
-      "DNSTransactionID": 20001,
+      "DNSTransactionID": 1,
       "DNSDomain": "www.example.com",
       "DNSLookupFailure": "",
       "DNSQueryType": null,
       "DNSEngine": null,
       "DNSResolvedAddrs": [
-        "93.184.216.34"
+        "104.154.89.105"
       ],
       "IPAddressOrigin": "dns",
-      "IPAddress": "93.184.216.34",
-      "IPAddressASN": 15133,
+      "IPAddress": "104.154.89.105",
+      "IPAddressASN": 396982,
       "IPAddressBogon": false,
       "EndpointTransactionID": 40002,
       "EndpointProto": "tcp",
       "EndpointPort": "443",
-      "EndpointAddress": "93.184.216.34:443",
+      "EndpointAddress": "104.154.89.105:443",
       "TCPConnectFailure": "",
-      "TLSHandshakeFailure": "",
+      "TLSHandshakeFailure": "ssl_unknown_authority",
       "TLSServerName": "www.example.com",
-      "HTTPRequestURL": "https://www.example.com/",
-      "HTTPFailure": "",
-      "HTTPResponseStatusCode": 200,
-      "HTTPResponseBodyLength": 1533,
-      "HTTPResponseBodyIsTruncated": false,
-      "HTTPResponseHeadersKeys": {
-        "Alt-Svc": true,
-        "Content-Length": true,
-        "Content-Type": true,
-        "Date": true
-      },
+      "HTTPRequestURL": null,
+      "HTTPFailure": null,
+      "HTTPResponseStatusCode": null,
+      "HTTPResponseBodyLength": null,
+      "HTTPResponseBodyIsTruncated": null,
+      "HTTPResponseHeadersKeys": null,
       "HTTPResponseLocation": null,
-      "HTTPResponseTitle": "Default Web Page",
-      "HTTPResponseIsFinal": true,
+      "HTTPResponseTitle": null,
+      "HTTPResponseIsFinal": null,
       "ControlDNSDomain": "www.example.com",
       "ControlDNSLookupFailure": "",
       "ControlDNSResolvedAddrs": [
         "93.184.216.34"
       ],
       "ControlTCPConnectFailure": "",
-      "ControlTLSHandshakeFailure": "",
+      "ControlTLSHandshakeFailure": "ssl_unknown_authority",
       "ControlHTTPFailure": "",
       "ControlHTTPResponseStatusCode": 200,
       "ControlHTTPResponseBodyLength": 1533,

diff --git a/...ctivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations_classic.json b/...ctivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations_classic.json
@@ -55,11 +55,11 @@
     }
   ],
   "KnownTCPEndpoints": {
-    "40001": {
+    "40002": {
       "TagDepth": 0,
       "Type": 2,
       "Failure": "ssl_unknown_authority",
-      "TransactionID": 40001,
+      "TransactionID": 40002,
       "TagFetchBody": true,
       "DNSTransactionID": 1,
       "DNSDomain": "www.example.com",
@@ -73,7 +73,7 @@
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
-      "EndpointTransactionID": 40001,
+      "EndpointTransactionID": 40002,
       "EndpointProto": "tcp",
       "EndpointPort": "443",
       "EndpointAddress": "104.154.89.105:443",