fix(qatool): remove remaining causes of churn (#1515)

This diff removes the remaining causes of churn in qatool (modulo flaky tests). Closes ooni/probe#2677.
ooni · Feb 13, 2024 · db5fd8c · db5fd8c
1 parent 7f8dda7
commit db5fd8c
Show file tree

Hide file tree

Showing 8 changed files with 111 additions and 72 deletions.
diff --git a/internal/cmd/qatool/main.go b/internal/cmd/qatool/main.go
@@ -6,6 +6,8 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"sort"
+	"time"
 
 	"github.com/ooni/probe-cli/v3/internal/experiment/webconnectivitylte"
 	"github.com/ooni/probe-cli/v3/internal/geoipx"
@@ -137,6 +139,23 @@ func runWebConnectivityLTE(tc *webconnectivityqa.TestCase) {
 	}
 }
 
+// override webconnectivitylte algorithm to make it less entropic
+func init() {
+	webconnectivitylte.MaybeSortAddresses = func(entries []webconnectivitylte.DNSEntry) {
+		sort.SliceStable(entries, func(i, j int) bool {
+			return entries[i].Addr < entries[j].Addr
+		})
+	}
+
+	webconnectivitylte.MaybeDelayCleartextFlows = func(index int) {
+		time.Sleep(10 * time.Millisecond * time.Duration(index))
+	}
+
+	webconnectivitylte.MaybeDelaySecureFlows = func(index int) {
+		time.Sleep(10 * time.Millisecond * time.Duration(index))
+	}
+}
+
 func main() {
 	// parse command line flags
 	flag.Parse()

diff --git a/internal/experiment/webconnectivitylte/dnsresolvers.go b/internal/experiment/webconnectivitylte/dnsresolvers.go
@@ -92,6 +92,11 @@ func (t *DNSResolvers) Start(ctx context.Context) {
 	}()
 }
 
+// MaybeSortAddresses is an OPTIONAL hook that possibly sorts the resolved addresses.
+var MaybeSortAddresses = func(entries []DNSEntry) {
+	// nothing!
+}
+
 // run performs a DNS lookup and returns the looked up addrs
 func (t *DNSResolvers) run(parentCtx context.Context) []DNSEntry {
 	// create output channels for the lookup
@@ -153,6 +158,9 @@ func (t *DNSResolvers) run(parentCtx context.Context) []DNSEntry {
 		entries = append(entries, *entry)
 	}
 
+	// allow specific users to sort addresses if needed
+	MaybeSortAddresses(entries)
+
 	return entries
 }
 
@@ -362,6 +370,11 @@ func (t *DNSResolvers) dohSplitQueries(
 	return
 }
 
+// MaybeDelayCleartextFlows is an OPTIONAL hook that possibly delays cleartext flows.
+var MaybeDelayCleartextFlows = func(index int) {
+	// nothing
+}
+
 // startCleartextFlows starts a TCP measurement flow for each IP addr.
 func (t *DNSResolvers) startCleartextFlows(
 	ctx context.Context,
@@ -377,7 +390,8 @@ func (t *DNSResolvers) startCleartextFlows(
 	if urlPort := t.URL.Port(); urlPort != "" {
 		port = urlPort
 	}
-	for _, addr := range addresses {
+	for index, addr := range addresses {
+		MaybeDelayCleartextFlows(index) // allow specific callers to space flows apart
 		task := &CleartextFlow{
 			Address:                 net.JoinHostPort(addr.Addr, port),
 			Depth:                   t.Depth,
@@ -402,6 +416,11 @@ func (t *DNSResolvers) startCleartextFlows(
 	}
 }
 
+// MaybeDelaySecureFlows is an OPTIONAL hook that possibly delays secure flows.
+var MaybeDelaySecureFlows = func(index int) {
+	// nothing
+}
+
 // startSecureFlows starts a TCP+TLS measurement flow for each IP addr.
 func (t *DNSResolvers) startSecureFlows(
 	ctx context.Context,
@@ -421,7 +440,8 @@ func (t *DNSResolvers) startSecureFlows(
 		}
 		port = urlPort
 	}
-	for _, addr := range addresses {
+	for index, addr := range addresses {
+		MaybeDelaySecureFlows(index) // allow specific callers to space flows apart
 		task := &SecureFlow{
 			Address:                 net.JoinHostPort(addr.Addr, port),
 			Depth:                   t.Depth,

diff --git a/internal/minipipeline/normalize.go b/internal/minipipeline/normalize.go
@@ -54,7 +54,7 @@ func NormalizeHTTPRequestResults(values []*model.ArchivalHTTPRequestResult) {
 		// Avoid storing large bodies because that wastes repository space.
 		//
 		// See https://github.com/ooni/probe/issues/2677.
-		const maxStoreBody = 1 << 14
+		const maxStoreBody = 12_000
 		if len(entry.Response.Body) > maxStoreBody {
 			entry.Response.Body = model.ArchivalScrubbedMaybeBinaryString("")
 			entry.Response.BodyIsTruncated = true

diff --git a/...minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/analysis.json b/...minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/analysis.json
@@ -49,7 +49,7 @@
   "HTTPFinalResponseSuccessTLSWithoutControl": null,
   "HTTPFinalResponseSuccessTLSWithControl": null,
   "HTTPFinalResponseSuccessTCPWithoutControl": null,
-  "HTTPFinalResponseSuccessTCPWithControl": 40002,
+  "HTTPFinalResponseSuccessTCPWithControl": 40001,
   "HTTPFinalResponseDiffBodyProportionFactor": 0.12263535551206783,
   "HTTPFinalResponseDiffStatusCodeMatch": true,
   "HTTPFinalResponseDiffTitleDifferentLongWords": {
@@ -63,7 +63,7 @@
       "TagDepth": 0,
       "Type": 3,
       "Failure": "",
-      "TransactionID": 40002,
+      "TransactionID": 40001,
       "TagFetchBody": true,
       "DNSTransactionID": 10001,
       "DNSDomain": "www.example.com",
@@ -77,7 +77,7 @@
       "IPAddress": "130.192.182.17",
       "IPAddressASN": 137,
       "IPAddressBogon": false,
-      "EndpointTransactionID": 40002,
+      "EndpointTransactionID": 40001,
       "EndpointProto": "tcp",
       "EndpointPort": "80",
       "EndpointAddress": "130.192.182.17:80",
@@ -117,22 +117,22 @@
       "Failure": "",
       "TransactionID": 50002,
       "TagFetchBody": false,
-      "DNSTransactionID": 10001,
+      "DNSTransactionID": 30001,
       "DNSDomain": "www.example.com",
       "DNSLookupFailure": "",
       "DNSQueryType": null,
       "DNSEngine": null,
       "DNSResolvedAddrs": [
-        "130.192.182.17"
+        "93.184.216.34"
       ],
       "IPAddressOrigin": "dns",
-      "IPAddress": "130.192.182.17",
-      "IPAddressASN": 137,
+      "IPAddress": "93.184.216.34",
+      "IPAddressASN": 15133,
       "IPAddressBogon": false,
       "EndpointTransactionID": 50002,
       "EndpointProto": "tcp",
       "EndpointPort": "443",
-      "EndpointAddress": "130.192.182.17:443",
+      "EndpointAddress": "93.184.216.34:443",
       "TCPConnectFailure": "",
       "TLSHandshakeFailure": "",
       "TLSServerName": "www.example.com",
@@ -169,22 +169,22 @@
       "Failure": "",
       "TransactionID": 50001,
       "TagFetchBody": false,
-      "DNSTransactionID": 30001,
+      "DNSTransactionID": 10001,
       "DNSDomain": "www.example.com",
       "DNSLookupFailure": "",
       "DNSQueryType": null,
       "DNSEngine": null,
       "DNSResolvedAddrs": [
-        "93.184.216.34"
+        "130.192.182.17"
       ],
       "IPAddressOrigin": "dns",
-      "IPAddress": "93.184.216.34",
-      "IPAddressASN": 15133,
+      "IPAddress": "130.192.182.17",
+      "IPAddressASN": 137,
       "IPAddressBogon": false,
       "EndpointTransactionID": 50001,
       "EndpointProto": "tcp",
       "EndpointPort": "443",
-      "EndpointAddress": "93.184.216.34:443",
+      "EndpointAddress": "130.192.182.17:443",
       "TCPConnectFailure": "",
       "TLSHandshakeFailure": "",
       "TLSServerName": "www.example.com",
@@ -219,7 +219,7 @@
       "TagDepth": 0,
       "Type": 1,
       "Failure": "",
-      "TransactionID": 40001,
+      "TransactionID": 40002,
       "TagFetchBody": true,
       "DNSTransactionID": 30001,
       "DNSDomain": "www.example.com",
@@ -233,7 +233,7 @@
       "IPAddress": "93.184.216.34",
       "IPAddressASN": 15133,
       "IPAddressBogon": false,
-      "EndpointTransactionID": 40001,
+      "EndpointTransactionID": 40002,
       "EndpointProto": "tcp",
       "EndpointPort": "80",
       "EndpointAddress": "93.184.216.34:80",

diff --git a/...line/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/analysis_classic.json b/...line/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/analysis_classic.json
@@ -41,7 +41,7 @@
   "HTTPFinalResponseSuccessTLSWithoutControl": null,
   "HTTPFinalResponseSuccessTLSWithControl": null,
   "HTTPFinalResponseSuccessTCPWithoutControl": null,
-  "HTTPFinalResponseSuccessTCPWithControl": 40002,
+  "HTTPFinalResponseSuccessTCPWithControl": 40001,
   "HTTPFinalResponseDiffBodyProportionFactor": 0.12263535551206783,
   "HTTPFinalResponseDiffStatusCodeMatch": true,
   "HTTPFinalResponseDiffTitleDifferentLongWords": {
@@ -55,7 +55,7 @@
       "TagDepth": 0,
       "Type": 3,
       "Failure": "",
-      "TransactionID": 40002,
+      "TransactionID": 40001,
       "TagFetchBody": true,
       "DNSTransactionID": 10001,
       "DNSDomain": "www.example.com",
@@ -69,7 +69,7 @@
       "IPAddress": "130.192.182.17",
       "IPAddressASN": 137,
       "IPAddressBogon": false,
-      "EndpointTransactionID": 40002,
+      "EndpointTransactionID": 40001,
       "EndpointProto": "tcp",
       "EndpointPort": "80",
       "EndpointAddress": "130.192.182.17:80",

diff --git a/...ipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/measurement.json b/...ipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/measurement.json
@@ -195,7 +195,7 @@
           "depth=0",
           "fetch_body=true"
         ],
-        "transaction_id": 40002
+        "transaction_id": 40001
       }
     ],
     "tcp_connect": [
@@ -211,7 +211,7 @@
           "depth=0",
           "fetch_body=true"
         ],
-        "transaction_id": 40002
+        "transaction_id": 40001
       },
       {
         "ip": "130.192.182.17",
@@ -225,7 +225,7 @@
           "depth=0",
           "fetch_body=false"
         ],
-        "transaction_id": 50002
+        "transaction_id": 50001
       },
       {
         "ip": "93.184.216.34",
@@ -239,7 +239,7 @@
           "depth=0",
           "fetch_body=true"
         ],
-        "transaction_id": 40001
+        "transaction_id": 40002
       },
       {
         "ip": "93.184.216.34",
@@ -253,7 +253,7 @@
           "depth=0",
           "fetch_body=false"
         ],
-        "transaction_id": 50001
+        "transaction_id": 50002
       }
     ],
     "tls_handshakes": [
@@ -272,7 +272,7 @@
           "fetch_body=false"
         ],
         "tls_version": "TLSv1.3",
-        "transaction_id": 50002
+        "transaction_id": 50001
       },
       {
         "network": "tcp",
@@ -289,7 +289,7 @@
           "fetch_body=false"
         ],
         "tls_version": "TLSv1.3",
-        "transaction_id": 50001
+        "transaction_id": 50002
       }
     ],
     "x_control_request": {
@@ -306,10 +306,10 @@
         ]
       },
       "tcp_connect": [
-        "93.184.216.34:443",
-        "93.184.216.34:80",
         "130.192.182.17:443",
-        "130.192.182.17:80"
+        "130.192.182.17:80",
+        "93.184.216.34:443",
+        "93.184.216.34:80"
       ],
       "x_quic_enabled": false
     },