feat(minipipeline): distinguish failure by operation (#1411)

We want to know if the failure occurs while fetching a body or while just performing a connectivity check. Part of ooni/probe#2634
ooni · Nov 30, 2023 · 70d2e64 · 70d2e64
1 parent 0c683d7
commit 70d2e64
Show file tree

Hide file tree

Showing 69 changed files with 327 additions and 15 deletions.
diff --git a/internal/cmd/minipipeline/testdata/analysis.json b/internal/cmd/minipipeline/testdata/analysis.json
@@ -3,7 +3,11 @@
   "DNSLookupSuccessWithInvalidAddressesClassic": [],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": {},
   "HTTPDiffBodyProportionFactor": 1,

diff --git a/internal/cmd/minipipeline/testdata/analysis_classic.json b/internal/cmd/minipipeline/testdata/analysis_classic.json
@@ -3,7 +3,11 @@
   "DNSLookupSuccessWithInvalidAddressesClassic": [],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": null,
   "HTTPDiffBodyProportionFactor": 1,

diff --git a/internal/minipipeline/analysis.go b/internal/minipipeline/analysis.go
@@ -10,12 +10,12 @@ import (
 func AnalyzeWebObservations(container *WebObservationsContainer) *WebAnalysis {
 	analysis := &WebAnalysis{}
 
-	analysis.ComputeDNSLookupSuccessWithInvalidAddresses(container)
-	analysis.ComputeDNSLookupSuccessWithInvalidAddressesClassic(container)
-	analysis.ComputeDNSLookupUnexpectedFailure(container)
+	analysis.dnsComputeSuccessMetrics(container)
+	analysis.dnsComputeSuccessMetricsClassic(container)
+	analysis.dnsComputeFailureMetrics(container)
 
-	analysis.ComputeTCPConnectUnexpectedFailure(container)
-	analysis.ComputeTLSHandshakeUnexpectedFailure(container)
+	analysis.tcpComputeMetrics(container)
+	analysis.tlsComputeMetrics(container)
 
 	analysis.ComputeDNSExperimentFailure(container)
 	analysis.ComputeDNSPossiblyNonexistingDomains(container)
@@ -51,9 +51,25 @@ type WebAnalysis struct {
 	// TCPConnectUnexpectedFailure contains TCP endpoint transactions with unexpected failures.
 	TCPConnectUnexpectedFailure Set[int64]
 
+	// TCPConnectUnexpectedFailureDuringWebFetch contains TCP endpoint transactions with unexpected failures
+	// while performing a web fetch, as opposed to checking for connectivity.
+	TCPConnectUnexpectedFailureDuringWebFetch Set[int64]
+
+	// TCPConnectUnexpectedFailureDuringConnectivityCheck contains TCP endpoint transactions with unexpected failures
+	// while checking for connectivity, as opposed to fetching a webpage.
+	TCPConnectUnexpectedFailureDuringConnectivityCheck Set[int64]
+
 	// TLSHandshakeUnexpectedFailure contains TLS endpoint transactions with unexpected failures.
 	TLSHandshakeUnexpectedFailure Set[int64]
 
+	// TLSHandshakeUnexpectedFailureDuringWebFetch contains TLS endpoint transactions with unexpected failures.
+	// while performing a web fetch, as opposed to checking for connectivity.
+	TLSHandshakeUnexpectedFailureDuringWebFetch Set[int64]
+
+	// TLSHandshakeUnexpectedFailureDuringConnectivityCheck contains TLS endpoint transactions with unexpected failures.
+	// while checking for connectivity, as opposed to fetching a webpage.
+	TLSHandshakeUnexpectedFailureDuringConnectivityCheck Set[int64]
+
 	// DNSExperimentFailure is the first failure experienced by a getaddrinfo-like resolver.
 	DNSExperimentFailure optional.Value[string]
 
@@ -103,8 +119,7 @@ type WebAnalysis struct {
 	TCPTransactionsWithUnexplainedUnexpectedFailures optional.Value[map[int64]bool]
 }
 
-// ComputeDNSLookupSuccessWithInvalidAddresses computes the ComputeDNSLookupSuccessWithInvalidAddresses field.
-func (wa *WebAnalysis) ComputeDNSLookupSuccessWithInvalidAddresses(c *WebObservationsContainer) {
+func (wa *WebAnalysis) dnsComputeSuccessMetrics(c *WebObservationsContainer) {
 	// fill the invalid set
 	var already Set[int64]
 	for _, obs := range c.DNSLookupSuccesses {
@@ -166,8 +181,7 @@ func (wa *WebAnalysis) ComputeDNSLookupSuccessWithInvalidAddresses(c *WebObserva
 	}
 }
 
-// ComputeDNSLookupSuccessWithInvalidAddressesClassic computes the DNSLookupSuccessWithInvalidAddressesClassic field.
-func (wa *WebAnalysis) ComputeDNSLookupSuccessWithInvalidAddressesClassic(c *WebObservationsContainer) {
+func (wa *WebAnalysis) dnsComputeSuccessMetricsClassic(c *WebObservationsContainer) {
 	var already Set[int64]
 
 	for _, obs := range c.DNSLookupSuccesses {
@@ -206,8 +220,7 @@ func (wa *WebAnalysis) ComputeDNSLookupSuccessWithInvalidAddressesClassic(c *Web
 	}
 }
 
-// ComputeDNSLookupUnexpectedFailure computes the DNSLookupUnexpectedFailure field.
-func (wa *WebAnalysis) ComputeDNSLookupUnexpectedFailure(c *WebObservationsContainer) {
+func (wa *WebAnalysis) dnsComputeFailureMetrics(c *WebObservationsContainer) {
 	var already Set[int64]
 
 	for _, obs := range c.DNSLookupFailures {
@@ -261,8 +274,7 @@ func (wa *WebAnalysis) ComputeDNSLookupUnexpectedFailure(c *WebObservationsConta
 	}
 }
 
-// ComputeTCPConnectUnexpectedFailure computes the TCPConnectUnexpectedFailure field.
-func (wa *WebAnalysis) ComputeTCPConnectUnexpectedFailure(c *WebObservationsContainer) {
+func (wa *WebAnalysis) tcpComputeMetrics(c *WebObservationsContainer) {
 	for _, obs := range c.KnownTCPEndpoints {
 		// dials once we started following redirects should not be considered
 		if obs.TagDepth.IsNone() || obs.TagDepth.Unwrap() != 0 {
@@ -294,14 +306,19 @@ func (wa *WebAnalysis) ComputeTCPConnectUnexpectedFailure(c *WebObservationsCont
 			if utilsTCPConnectFailureSeemsMisconfiguredIPv6(obs) {
 				continue
 			}
+			switch {
+			case !obs.TagFetchBody.IsNone() && obs.TagFetchBody.Unwrap():
+				wa.TCPConnectUnexpectedFailureDuringWebFetch.Add(obs.EndpointTransactionID.Unwrap())
+			case !obs.TagFetchBody.IsNone() && !obs.TagFetchBody.Unwrap():
+				wa.TCPConnectUnexpectedFailureDuringConnectivityCheck.Add(obs.EndpointTransactionID.Unwrap())
+			}
 			wa.TCPConnectUnexpectedFailure.Add(obs.EndpointTransactionID.Unwrap())
 			continue
 		}
 	}
 }
 
-// ComputeTLSHandshakeUnexpectedFailure computes the TLSHandshakeUnexpectedFailure field.
-func (wa *WebAnalysis) ComputeTLSHandshakeUnexpectedFailure(c *WebObservationsContainer) {
+func (wa *WebAnalysis) tlsComputeMetrics(c *WebObservationsContainer) {
 	for _, obs := range c.KnownTCPEndpoints {
 		// handshakes once we started following redirects should not be considered
 		if obs.TagDepth.IsNone() || obs.TagDepth.Unwrap() != 0 {
@@ -330,6 +347,12 @@ func (wa *WebAnalysis) ComputeTLSHandshakeUnexpectedFailure(c *WebObservationsCo
 
 		// handle the case where only the probe fails
 		if obs.TLSHandshakeFailure.Unwrap() != "" {
+			switch {
+			case !obs.TagFetchBody.IsNone() && obs.TagFetchBody.Unwrap():
+				wa.TLSHandshakeUnexpectedFailureDuringWebFetch.Add(obs.EndpointTransactionID.Unwrap())
+			case !obs.TagFetchBody.IsNone() && !obs.TagFetchBody.Unwrap():
+				wa.TLSHandshakeUnexpectedFailureDuringConnectivityCheck.Add(obs.EndpointTransactionID.Unwrap())
+			}
 			wa.TLSHandshakeUnexpectedFailure.Add(obs.EndpointTransactionID.Unwrap())
 			continue
 		}

diff --git a/...inipipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/analysis.json b/...inipipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/analysis.json
@@ -3,7 +3,11 @@
   "DNSLookupSuccessWithInvalidAddressesClassic": [],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": {},
   "HTTPDiffBodyProportionFactor": null,

diff --git a/...ine/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/analysis_classic.json b/...ine/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/analysis_classic.json
@@ -3,7 +3,11 @@
   "DNSLookupSuccessWithInvalidAddressesClassic": [],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": null,
   "HTTPDiffBodyProportionFactor": null,

diff --git a/...tdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/analysis.json b/...tdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/analysis.json
@@ -3,7 +3,11 @@
   "DNSLookupSuccessWithInvalidAddressesClassic": [],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": {},
   "HTTPDiffBodyProportionFactor": null,

diff --git a/...bconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/analysis_classic.json b/...bconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/analysis_classic.json
@@ -3,7 +3,11 @@
   "DNSLookupSuccessWithInvalidAddressesClassic": [],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": null,
   "HTTPDiffBodyProportionFactor": null,

diff --git a/...ata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/analysis.json b/...ata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/analysis.json
@@ -9,7 +9,11 @@
   ],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": {},
   "HTTPDiffBodyProportionFactor": 1,

diff --git a/...onnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/analysis_classic.json b/...onnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/analysis_classic.json
@@ -7,7 +7,11 @@
   ],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": null,
   "HTTPDiffBodyProportionFactor": null,

diff --git a/...l/minipipeline/testdata/webconnectivity/generated/badSSLWithWrongServerName/analysis.json b/...l/minipipeline/testdata/webconnectivity/generated/badSSLWithWrongServerName/analysis.json
@@ -3,7 +3,11 @@
   "DNSLookupSuccessWithInvalidAddressesClassic": [],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": {},
   "HTTPDiffBodyProportionFactor": null,

diff --git a/...peline/testdata/webconnectivity/generated/badSSLWithWrongServerName/analysis_classic.json b/...peline/testdata/webconnectivity/generated/badSSLWithWrongServerName/analysis_classic.json
@@ -3,7 +3,11 @@
   "DNSLookupSuccessWithInvalidAddressesClassic": [],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": null,
   "HTTPDiffBodyProportionFactor": null,

diff --git a/...testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPSWebsite/analysis.json b/...testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPSWebsite/analysis.json
@@ -3,7 +3,11 @@
   "DNSLookupSuccessWithInvalidAddressesClassic": [],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": null,
   "HTTPDiffBodyProportionFactor": null,

diff --git a/.../webconnectivity/generated/controlFailureWithSuccessfulHTTPSWebsite/analysis_classic.json b/.../webconnectivity/generated/controlFailureWithSuccessfulHTTPSWebsite/analysis_classic.json
@@ -3,7 +3,11 @@
   "DNSLookupSuccessWithInvalidAddressesClassic": [],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": null,
   "HTTPDiffBodyProportionFactor": null,

diff --git a/.../testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPWebsite/analysis.json b/.../testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPWebsite/analysis.json
@@ -3,7 +3,11 @@
   "DNSLookupSuccessWithInvalidAddressesClassic": [],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": null,
   "HTTPDiffBodyProportionFactor": null,

diff --git a/...a/webconnectivity/generated/controlFailureWithSuccessfulHTTPWebsite/analysis_classic.json b/...a/webconnectivity/generated/controlFailureWithSuccessfulHTTPWebsite/analysis_classic.json
@@ -3,7 +3,11 @@
   "DNSLookupSuccessWithInvalidAddressesClassic": [],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": null,
   "HTTPDiffBodyProportionFactor": null,

diff --git a/...ipeline/testdata/webconnectivity/generated/dnsBlockingAndroidDNSCacheNoData/analysis.json b/...ipeline/testdata/webconnectivity/generated/dnsBlockingAndroidDNSCacheNoData/analysis.json
@@ -5,7 +5,11 @@
     1
   ],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": "android_dns_cache_no_data",
   "DNSPossiblyNonexistingDomains": {},
   "HTTPDiffBodyProportionFactor": 1,

diff --git a/...testdata/webconnectivity/generated/dnsBlockingAndroidDNSCacheNoData/analysis_classic.json b/...testdata/webconnectivity/generated/dnsBlockingAndroidDNSCacheNoData/analysis_classic.json
@@ -5,7 +5,11 @@
     1
   ],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": "android_dns_cache_no_data",
   "DNSPossiblyNonexistingDomains": {},
   "HTTPDiffBodyProportionFactor": null,

diff --git a/internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/analysis.json b/internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/analysis.json
@@ -7,7 +7,11 @@
   ],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": {},
   "HTTPDiffBodyProportionFactor": 1,

diff --git a/...al/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/analysis_classic.json b/...al/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/analysis_classic.json
@@ -7,7 +7,11 @@
   ],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": null,
   "HTTPDiffBodyProportionFactor": null,

diff --git a/internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingNXDOMAIN/analysis.json b/internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingNXDOMAIN/analysis.json
@@ -5,7 +5,11 @@
     2
   ],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": "dns_nxdomain_error",
   "DNSPossiblyNonexistingDomains": {},
   "HTTPDiffBodyProportionFactor": 1,

diff --git a/...minipipeline/testdata/webconnectivity/generated/dnsBlockingNXDOMAIN/analysis_classic.json b/...minipipeline/testdata/webconnectivity/generated/dnsBlockingNXDOMAIN/analysis_classic.json
@@ -5,7 +5,11 @@
     2
   ],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": "dns_nxdomain_error",
   "DNSPossiblyNonexistingDomains": {},
   "HTTPDiffBodyProportionFactor": null,

diff --git a/...pipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPSURL/analysis.json b/...pipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPSURL/analysis.json
@@ -8,7 +8,11 @@
   ],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": {},
   "HTTPDiffBodyProportionFactor": 1,

diff --git a/.../testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPSURL/analysis_classic.json b/.../testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPSURL/analysis_classic.json
@@ -5,7 +5,11 @@
   ],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": null,
   "HTTPDiffBodyProportionFactor": 1,

diff --git a/...ipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPURL/analysis.json b/...ipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPURL/analysis.json
@@ -8,7 +8,11 @@
   ],
   "DNSLookupUnexpectedFailure": [],
   "TCPConnectUnexpectedFailure": [],
+  "TCPConnectUnexpectedFailureDuringWebFetch": [],
+  "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
   "TLSHandshakeUnexpectedFailure": [],
+  "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+  "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
   "DNSExperimentFailure": null,
   "DNSPossiblyNonexistingDomains": {},
   "HTTPDiffBodyProportionFactor": 1,