Skip to content

Commit

Permalink
feat(minipipeline): add DNSLookupSuccessWithBogonAddresses (#1449)
Browse files Browse the repository at this point in the history
This diff adds the DNSLookupSuccessWithBogonAddresses analysis field to
minipipeline.

We'll need this field to complete
ooni/probe#2640.

While there, implement `io.Stringer` for the `minipeline.Set` type,
which we will also need to have.
  • Loading branch information
bassosimone committed Jan 16, 2024
1 parent 75780fd commit dfaeed9
Show file tree
Hide file tree
Showing 72 changed files with 94 additions and 2 deletions.
1 change: 1 addition & 0 deletions internal/cmd/minipipeline/testdata/analysis.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
2,
3
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1,
Expand Down
1 change: 1 addition & 0 deletions internal/cmd/minipipeline/testdata/analysis_classic.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
1,
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
2
Expand Down
7 changes: 6 additions & 1 deletion internal/minipipeline/analysis.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,9 @@ type WebAnalysis struct {
// DNSLookupSuccessWithValidAddress contains DNS transactions with valid IP addresses.
DNSLookupSuccessWithValidAddress Set[int64]

// DNSLookupSuccessWithBogonAddresses contains DNS transactions with bogon IP addresses.
DNSLookupSuccessWithBogonAddresses Set[int64]

// DNSLookupSuccessWithInvalidAddressesClassic is like DNSLookupInvalid but the algorithm is more relaxed
// to be compatible with Web Connectivity v0.4's behavior.
DNSLookupSuccessWithInvalidAddressesClassic Set[int64]
Expand Down Expand Up @@ -224,7 +227,8 @@ type WebAnalysis struct {
// HTTPFinalResponseDiffUncommonHeadersIntersection contains the uncommon headers intersection.
HTTPFinalResponseDiffUncommonHeadersIntersection optional.Value[map[string]bool]

// Linear contains the linear analysis.
// Linear contains the linear analysis. We only fill this field when using
// the [AnalyzeWebObservationsWithLinearAnalysis] constructor.
Linear []*WebObservation
}

Expand All @@ -246,6 +250,7 @@ func (wa *WebAnalysis) dnsComputeSuccessMetrics(c *WebObservationsContainer) {
// if there's a bogon, mark as invalid
if !obs.IPAddressBogon.IsNone() && obs.IPAddressBogon.Unwrap() {
wa.DNSLookupSuccessWithInvalidAddresses.Add(obs.DNSTransactionID.Unwrap())
wa.DNSLookupSuccessWithBogonAddresses.Add(obs.DNSTransactionID.Unwrap())
continue
}

Expand Down
2 changes: 1 addition & 1 deletion internal/minipipeline/qa_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ func testMustRunAllWebTestCases(t *testing.T, topdir string) {
//
// Note: if tests fail, you likely need to regenerate the static test
// cases using ./script/updateminipipeline.bash and you should also eyeball
// the diff for these fails to see if it makes sense.
// the diff for these changes to see if it makes sense.
//

t.Run("linear consistency checks", func(t *testing.T) {
Expand Down
6 changes: 6 additions & 0 deletions internal/minipipeline/set.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package minipipeline

import (
"encoding/json"
"fmt"
"sort"
)

Expand Down Expand Up @@ -77,3 +78,8 @@ func (sx *Set[T]) Contains(key T) bool {
_, found := sx.state[key]
return found
}

// String implements fmt.Stringer
func (sx *Set[T]) String() string {
return fmt.Sprintf("%v", sx.Keys())
}
9 changes: 9 additions & 0 deletions internal/minipipeline/set_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,4 +115,13 @@ func TestSet(t *testing.T) {
t.Fatal("expected false")
}
})

t.Run("String", func(t *testing.T) {
var set Set[int64]
set.Add(10, 11, 12, 13)
expect := "[10 11 12 13]"
if diff := cmp.Diff(expect, set.String()); diff != "" {
t.Fatal(diff)
}
})
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
1,
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"DNSLookupSuccessWithValidAddress": [
1
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
2,
3
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"DNSLookupSuccessWithValidAddress": [
1
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
2
],
"DNSLookupSuccessWithValidAddress": [],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [
1,
2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
2
],
"DNSLookupSuccessWithValidAddress": [],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [
2
],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
1,
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"DNSLookupSuccessWithValidAddress": [
1
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"DNSLookupSuccessWithValidAddress": [
1
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [],
"DNSLookupUnexpectedFailure": [],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"DNSLookupSuccessWithValidAddress": [
1
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [],
"DNSLookupUnexpectedFailure": [],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"DNSLookupSuccessWithValidAddress": [
1
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [],
"DNSLookupUnexpectedFailure": [],
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
{
"DNSLookupSuccessWithInvalidAddresses": [],
"DNSLookupSuccessWithValidAddress": [],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [],
"DNSLookupUnexpectedFailure": [],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"DNSLookupSuccessWithValidAddress": [
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
2
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
{
"DNSLookupSuccessWithInvalidAddresses": [],
"DNSLookupSuccessWithValidAddress": [],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [],
"DNSLookupUnexpectedFailure": [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
"DNSLookupSuccessWithValidAddress": [
2
],
"DNSLookupSuccessWithBogonAddresses": [
1
],
"DNSLookupSuccessWithInvalidAddressesClassic": [
1
],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
1
],
"DNSLookupSuccessWithValidAddress": [],
"DNSLookupSuccessWithBogonAddresses": [
1
],
"DNSLookupSuccessWithInvalidAddressesClassic": [
1
],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"DNSLookupSuccessWithValidAddress": [
1
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
{
"DNSLookupSuccessWithInvalidAddresses": [],
"DNSLookupSuccessWithValidAddress": [],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [],
"DNSLookupUnexpectedFailure": [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"DNSLookupSuccessWithValidAddress": [
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [
1,
2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"DNSLookupSuccessWithValidAddress": [
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [
2
],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"DNSLookupSuccessWithValidAddress": [
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [
1,
2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
2
],
"DNSLookupSuccessWithValidAddress": [],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [
2
],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
1,
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"DNSLookupSuccessWithValidAddress": [
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
1,
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"DNSLookupSuccessWithValidAddress": [
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"DNSLookupSuccessWithValidAddress": [
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [
1,
2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
2
],
"DNSLookupSuccessWithValidAddress": [],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [
2
],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
1,
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"DNSLookupSuccessWithValidAddress": [
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
1,
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
1,
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
1,
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
1,
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
1,
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"DNSLookupSuccessWithValidAddress": [
1
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
1,
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"DNSLookupSuccessWithValidAddress": [
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
1,
2
],
"DNSLookupSuccessWithBogonAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupSuccessWithValidAddressClassic": [
1,
Expand Down
Loading

0 comments on commit dfaeed9

Please sign in to comment.