fix(minipipeline): handle IP-addr URLs in classic linear analysis (#1472

) This diff fixes ooni/probe#1511 (comment). This is the rationale of the diff: we need to track the origin of IP addresses: * "dns" if discovered using DNS; * "th" if discovered using the test helper; * null otherwise. When filtering for classic analysis, we include "dns" entries if resolved using getaddrinfo, drop "th" entries because they're not relevant, and include null entries under the assumption that the probe discovered them either directly from the input URL or because a redirect redirected to an URL containing an IP address. We also update the minipipeline test suite and show that the only changes are related to the test added by #1471.
ooni · Jan 24, 2024 · 2cc9231 · 2cc9231
1 parent 4fb9f0f
commit 2cc9231
Show file tree

Hide file tree

Showing 186 changed files with 1,701 additions and 11 deletions.
diff --git a/internal/cmd/minipipeline/testdata/analysis.json b/internal/cmd/minipipeline/testdata/analysis.json
@@ -70,6 +70,7 @@
       "DNSResolvedAddrs": [
         "130.192.16.171"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,
@@ -146,6 +147,7 @@
       "DNSResolvedAddrs": [
         "130.192.16.171"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,
@@ -207,6 +209,7 @@
       "DNSResolvedAddrs": [
         "130.192.16.171"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,
@@ -268,6 +271,7 @@
       "DNSResolvedAddrs": [
         "130.192.16.171"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,
@@ -327,6 +331,7 @@
       "DNSQueryType": "AAAA",
       "DNSEngine": "doh",
       "DNSResolvedAddrs": null,
+      "IPAddressOrigin": null,
       "IPAddress": null,
       "IPAddressASN": null,
       "IPAddressBogon": null,
@@ -386,6 +391,7 @@
       "DNSQueryType": "AAAA",
       "DNSEngine": "udp",
       "DNSResolvedAddrs": null,
+      "IPAddressOrigin": null,
       "IPAddress": null,
       "IPAddressASN": null,
       "IPAddressBogon": null,

diff --git a/internal/cmd/minipipeline/testdata/analysis_classic.json b/internal/cmd/minipipeline/testdata/analysis_classic.json
@@ -65,6 +65,7 @@
       "DNSResolvedAddrs": [
         "130.192.16.171"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,
@@ -141,6 +142,7 @@
       "DNSResolvedAddrs": [
         "130.192.16.171"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,

diff --git a/internal/cmd/minipipeline/testdata/observations.json b/internal/cmd/minipipeline/testdata/observations.json
@@ -12,6 +12,7 @@
       "DNSQueryType": "AAAA",
       "DNSEngine": "udp",
       "DNSResolvedAddrs": null,
+      "IPAddressOrigin": null,
       "IPAddress": null,
       "IPAddressASN": null,
       "IPAddressBogon": null,
@@ -71,6 +72,7 @@
       "DNSQueryType": "AAAA",
       "DNSEngine": "doh",
       "DNSResolvedAddrs": null,
+      "IPAddressOrigin": null,
       "IPAddress": null,
       "IPAddressASN": null,
       "IPAddressBogon": null,
@@ -134,6 +136,7 @@
       "DNSResolvedAddrs": [
         "130.192.16.171"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,
@@ -195,6 +198,7 @@
       "DNSResolvedAddrs": [
         "130.192.16.171"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,
@@ -256,6 +260,7 @@
       "DNSResolvedAddrs": [
         "130.192.16.171"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,
@@ -319,6 +324,7 @@
       "DNSResolvedAddrs": [
         "130.192.16.171"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,

diff --git a/internal/cmd/minipipeline/testdata/observations_classic.json b/internal/cmd/minipipeline/testdata/observations_classic.json
@@ -15,6 +15,7 @@
       "DNSResolvedAddrs": [
         "130.192.16.171"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,
@@ -78,6 +79,7 @@
       "DNSResolvedAddrs": [
         "130.192.16.171"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,

diff --git a/internal/minipipeline/classic.go b/internal/minipipeline/classic.go
@@ -43,12 +43,33 @@ func ClassicFilter(input *WebObservationsContainer) (output *WebObservationsCont
 	for _, entry := range input.KnownTCPEndpoints {
 		ipAddr := entry.IPAddress.Unwrap() // it MUST be there
 		txid := entry.EndpointTransactionID.Unwrap()
-		if output.knownIPAddresses[ipAddr] == nil {
+
+		// Determine whether to keep entry depending on the IP addr origin
+		switch entry.IPAddressOrigin.UnwrapOr("") {
+
+		// If the address origin is the TH, then it does not belong to classic analysis
+		case IPAddressOriginTH:
 			continue
+
+		// If the address origin is the DNS, then it depends on whether it was
+		// resolved via getaddrinfo or via another resolver
+		case IPAddressOriginDNS:
+			if output.knownIPAddresses[ipAddr] == nil {
+				continue
+			}
+
+		// If the address origin is unknown, then we assume the probe
+		// already knows it, e.g., via the URL or via a subsequent redirect
+		// and thus we keep this specific entry
+		default:
+			// nothing
 		}
+
+		// Discard all the entries where we're not fetching body
 		if !entry.TagFetchBody.UnwrapOr(false) {
 			continue
 		}
+
 		output.KnownTCPEndpoints[txid] = entry
 	}
 

diff --git a/internal/minipipeline/observation.go b/internal/minipipeline/observation.go
@@ -65,6 +65,12 @@ const (
 	WebObservationTypeHTTPRoundTrip
 )
 
+// These are the possible origins for IP addresses.
+const (
+	IPAddressOriginDNS = "dns"
+	IPAddressOriginTH  = "th"
+)
+
 // WebObservation is an observation of the flow that starts with a DNS lookup that
 // either fails or discovers an IP address and proceeds by documenting binding such an
 // address to a part to obtain and use a TCP or UDP endpoint.
@@ -145,6 +151,9 @@ type WebObservation struct {
 	//
 	// 3. when the input URL contains an IP address.
 
+	// IPAddressOrigin is the optional origin of the IP address.
+	IPAddressOrigin optional.Value[string]
+
 	// IPAddress is the optional IP address that this observation is about. We typically derive
 	// this value from a DNS lookup, but sometimes we know it from other means (e.g., from
 	// the Web Connectivity test helper response). When DNSLookupFailure contains an nonempty
@@ -357,6 +366,7 @@ func (c *WebObservationsContainer) ingestDNSLookupSuccesses(
 				DNSQueryType:     optional.Some(ev.QueryType),
 				DNSEngine:        optional.Some(ev.Engine),
 				DNSResolvedAddrs: optional.Some(addrs),
+				IPAddressOrigin:  optional.Some(IPAddressOriginDNS),
 				IPAddress:        optional.Some(ipAddr),
 				IPAddressASN:     utilsGeoipxLookupASN(lookupper, ipAddr),
 				IPAddressBogon:   optional.Some(netxlite.IsBogon(ipAddr)),
@@ -383,9 +393,10 @@ func (c *WebObservationsContainer) IngestTCPConnectEvents(
 		obs, found := c.knownIPAddresses[ev.IP]
 		if !found {
 			obs = &WebObservation{
-				IPAddress:      optional.Some(ev.IP),
-				IPAddressASN:   utilsGeoipxLookupASN(lookupper, ev.IP),
-				IPAddressBogon: optional.Some(netxlite.IsBogon(ev.IP)),
+				IPAddressOrigin: optional.None[string](), // we don't know!
+				IPAddress:       optional.Some(ev.IP),
+				IPAddressASN:    utilsGeoipxLookupASN(lookupper, ev.IP),
+				IPAddressBogon:  optional.Some(netxlite.IsBogon(ev.IP)),
 			}
 		}
 
@@ -403,6 +414,7 @@ func (c *WebObservationsContainer) IngestTCPConnectEvents(
 			DNSDomain:             obs.DNSDomain,
 			DNSLookupFailure:      obs.DNSLookupFailure,
 			DNSResolvedAddrs:      obs.DNSResolvedAddrs,
+			IPAddressOrigin:       obs.IPAddressOrigin,
 			IPAddress:             obs.IPAddress,
 			IPAddressASN:          obs.IPAddressASN,
 			IPAddressBogon:        obs.IPAddressBogon,
@@ -531,6 +543,7 @@ func (c *WebObservationsContainer) controlMatchDNSLookupResults(inputDomain stri
 		// handle the case in which the IP address has been provided by the control, which
 		// is a case where the domain is empty and the IP address is in thAddrMap
 		if domain == "" && thAddrMap[addr] {
+			obs.IPAddressOrigin = optional.Some(IPAddressOriginTH)
 			obs.ControlDNSDomain = optional.Some(inputDomain)
 			obs.ControlDNSLookupFailure = optional.Some(utilsStringPointerToString(resp.DNS.Failure))
 			obs.ControlDNSResolvedAddrs = optional.Some(NewSet(resp.DNS.Addrs...))

diff --git a/...inipipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/analysis.json b/...inipipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/analysis.json
@@ -66,6 +66,7 @@
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
@@ -112,6 +113,7 @@
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
@@ -158,6 +160,7 @@
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
@@ -202,6 +205,7 @@
       "DNSQueryType": "AAAA",
       "DNSEngine": "udp",
       "DNSResolvedAddrs": null,
+      "IPAddressOrigin": null,
       "IPAddress": null,
       "IPAddressASN": null,
       "IPAddressBogon": null,

diff --git a/...ine/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/analysis_classic.json b/...ine/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/analysis_classic.json
@@ -63,6 +63,7 @@
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
@@ -109,6 +110,7 @@
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,

diff --git a/...ipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations.json b/...ipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations.json
@@ -12,6 +12,7 @@
       "DNSQueryType": "AAAA",
       "DNSEngine": "udp",
       "DNSResolvedAddrs": null,
+      "IPAddressOrigin": null,
       "IPAddress": null,
       "IPAddressASN": null,
       "IPAddressBogon": null,
@@ -60,6 +61,7 @@
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
@@ -106,6 +108,7 @@
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
@@ -154,6 +157,7 @@
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,

diff --git a/...testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations_classic.json b/...testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations_classic.json
@@ -15,6 +15,7 @@
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
@@ -63,6 +64,7 @@
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,

diff --git a/...tdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/analysis.json b/...tdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/analysis.json
@@ -69,6 +69,7 @@
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
@@ -115,6 +116,7 @@
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
@@ -161,6 +163,7 @@
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
@@ -207,6 +210,7 @@
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
@@ -251,6 +255,7 @@
       "DNSQueryType": "AAAA",
       "DNSEngine": "udp",
       "DNSResolvedAddrs": null,
+      "IPAddressOrigin": null,
       "IPAddress": null,
       "IPAddressASN": null,
       "IPAddressBogon": null,
@@ -295,6 +300,7 @@
       "DNSQueryType": "AAAA",
       "DNSEngine": "doh",
       "DNSResolvedAddrs": null,
+      "IPAddressOrigin": null,
       "IPAddress": null,
       "IPAddressASN": null,
       "IPAddressBogon": null,

diff --git a/...bconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/analysis_classic.json b/...bconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/analysis_classic.json
@@ -63,6 +63,7 @@
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
@@ -109,6 +110,7 @@
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
+      "IPAddressOrigin": "dns",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,