diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 0cc8e89f1..30bfb3ad5 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -11,5 +11,5 @@ jobs:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
- go-version: "1.20.14"
+ go-version: "1.21.10"
- run: go build ./...
diff --git a/UPSTREAM b/UPSTREAM
index 329941841..981feb4ab 100644
--- a/UPSTREAM
+++ b/UPSTREAM
@@ -1 +1 @@
-v3.21.0-alpha
+v3.22.0-alpha
diff --git a/go.mod b/go.mod
index c82d7534a..5660405e9 100644
--- a/go.mod
+++ b/go.mod
@@ -1,42 +1,45 @@
module github.com/ooni/probe-engine
-go 1.20
+go 1.21
+
+toolchain go1.21.10
require (
filippo.io/age v1.1.1
- github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20240312131609-91e2a902d867
+ github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20240424194431-3612a5a6fb4c
github.com/apex/log v1.9.0
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
- github.com/cloudflare/circl v1.3.7
+ github.com/cloudflare/circl v1.3.8
github.com/cretz/bine v0.2.0
github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d
- github.com/dop251/goja_nodejs v0.0.0-20231122114759-e84d9a924c5c
+ github.com/dop251/goja_nodejs v0.0.0-20240418154818-2aae10d4cbcf
github.com/google/go-cmp v0.6.0
github.com/google/gopacket v1.1.19
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+ github.com/google/uuid v1.6.0
github.com/gorilla/websocket v1.5.1
github.com/hexops/gotextdiff v1.0.3
- github.com/miekg/dns v1.1.58
+ github.com/miekg/dns v1.1.59
github.com/montanaflynn/stats v0.7.1
github.com/ooni/netem v0.0.0-20240208095707-608dcbcd82b8
- github.com/ooni/oocrypto v0.5.8
- github.com/ooni/oohttp v0.6.8
- github.com/ooni/probe-assets v0.22.0
+ github.com/ooni/oocrypto v0.6.1
+ github.com/ooni/oohttp v0.7.2
+ github.com/ooni/probe-assets v0.23.0
github.com/pborman/getopt/v2 v2.1.0
github.com/pion/stun v0.6.1
github.com/pkg/errors v0.9.1
- github.com/quic-go/quic-go v0.40.1
+ github.com/quic-go/quic-go v0.43.1
github.com/rogpeppe/go-internal v1.12.0
- github.com/rubenv/sql-migrate v1.5.2
+ github.com/rubenv/sql-migrate v1.6.1
github.com/schollz/progressbar/v3 v3.14.2
github.com/upper/db/v4 v4.7.0
gitlab.com/yawning/obfs4.git v0.0.0-20231012084234-c3e2d44b1033
gitlab.com/yawning/utls.git v0.0.12-1
gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib v1.5.0
gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2 v2.6.1
- golang.org/x/crypto v0.21.0
- golang.org/x/net v0.22.0
- golang.org/x/sys v0.18.0
+ golang.org/x/crypto v0.23.0
+ golang.org/x/net v0.25.0
+ golang.org/x/sys v0.20.0
)
require (
@@ -44,44 +47,41 @@ require (
filippo.io/keygen v0.0.0-20230306160926-5201437acf8e // indirect
github.com/Psiphon-Labs/bolt v0.0.0-20200624191537-23cedaef7ad7 // indirect
github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464 // indirect
- github.com/Psiphon-Labs/qtls-go1-19 v0.0.0-20230608213623-d58aa73e519a // indirect
- github.com/Psiphon-Labs/qtls-go1-20 v0.0.0-20230608214729-dd57d6787acf // indirect
- github.com/Psiphon-Labs/quic-go v0.0.0-20230626192210-73f29effc9da // indirect
- github.com/Psiphon-Labs/tls-tris v0.0.0-20230824155421-58bf6d336a9a // indirect
+ github.com/Psiphon-Labs/psiphon-tls v0.0.0-20240424193802-52b2602ec60c // indirect
+ github.com/Psiphon-Labs/quic-go v0.0.0-20240424181006-45545f5e1536 // indirect
github.com/andybalholm/brotli v1.0.6 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dlclark/regexp2 v1.10.0 // indirect
- github.com/fatih/color v1.16.0 // indirect
+ github.com/fatih/color v1.17.0 // indirect
github.com/flynn/noise v1.0.1 // indirect
github.com/gaukas/godicttls v0.0.4 // indirect
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
+ github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/google/btree v1.1.2 // indirect
- github.com/google/pprof v0.0.0-20231212022811-ec68065c825e // indirect
- github.com/google/uuid v1.6.0 // indirect
+ github.com/google/pprof v0.0.0-20240509144519-723abb6459b7 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
- github.com/klauspost/compress v1.17.4 // indirect
+ github.com/klauspost/compress v1.17.8 // indirect
github.com/libp2p/go-reuseport v0.4.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
- github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/mroth/weightedrand v1.0.0 // indirect
- github.com/onsi/ginkgo/v2 v2.13.2 // indirect
+ github.com/onsi/ginkgo/v2 v2.17.3 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
- github.com/pion/transport/v2 v2.2.4 // indirect
+ github.com/pion/transport/v2 v2.2.5 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/quic-go/qpack v0.4.0 // indirect
- github.com/quic-go/qtls-go1-20 v0.4.1 // indirect
- github.com/refraction-networking/conjure v0.7.10 // indirect
+ github.com/refraction-networking/conjure v0.7.11-0.20240130155008-c8df96195ab2 // indirect
github.com/refraction-networking/ed25519 v0.1.2 // indirect
github.com/refraction-networking/obfs4 v0.1.2 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/segmentio/fasthash v1.0.3 // indirect
github.com/spf13/pflag v1.0.5 // indirect
- github.com/stretchr/testify v1.8.4 // indirect
+ github.com/stretchr/testify v1.9.0 // indirect
gitlab.com/yawning/edwards25519-extra v0.0.0-20231005122941-2149dcafc266 // indirect
- go.uber.org/mock v0.3.0 // indirect
- golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb // indirect
+ go.uber.org/mock v0.4.0 // indirect
+ golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
golang.org/x/exp/typeparams v0.0.0-20230522175609-2e198f4a06a1 // indirect
+ golang.org/x/sync v0.7.0 // indirect
golang.org/x/time v0.5.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
gvisor.dev/gvisor v0.0.0-20230922204349-b3f36d574a7f // indirect
@@ -93,7 +93,7 @@ require (
github.com/armon/go-proxyproto v0.1.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bifurcation/mint v0.0.0-20180306135233-198357931e61 // indirect
- github.com/cespare/xxhash/v2 v2.2.0 // indirect
+ github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cheekybits/genny v1.0.0 // indirect
github.com/cognusion/go-cache-lru v0.0.0-20170419142635-f73e2280ecea // indirect
github.com/dchest/siphash v1.2.3 // indirect
@@ -102,8 +102,7 @@ require (
github.com/dsnet/compress v0.0.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-gorp/gorp/v3 v3.1.0 // indirect
- github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
- github.com/golang/glog v1.2.0 // indirect
+ github.com/golang/glog v1.2.1 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/grafov/m3u8 v0.12.0 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
@@ -113,24 +112,24 @@ require (
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-sqlite3 v1.14.22
github.com/oschwald/maxminddb-golang v1.12.0
- github.com/pion/datachannel v1.5.5 // indirect
- github.com/pion/dtls/v2 v2.2.8 // indirect
- github.com/pion/ice/v2 v2.3.11 // indirect
- github.com/pion/interceptor v0.1.25 // indirect
+ github.com/pion/datachannel v1.5.6 // indirect
+ github.com/pion/dtls/v2 v2.2.11 // indirect
+ github.com/pion/ice/v2 v2.3.24 // indirect
+ github.com/pion/interceptor v0.1.29 // indirect
github.com/pion/logging v0.2.2 // indirect
- github.com/pion/mdns v0.0.9 // indirect
+ github.com/pion/mdns v0.0.12 // indirect
github.com/pion/randutil v0.1.0 // indirect
- github.com/pion/rtcp v1.2.13 // indirect
- github.com/pion/rtp v1.8.3 // indirect
- github.com/pion/sctp v1.8.9 // indirect
- github.com/pion/sdp/v3 v3.0.6 // indirect
+ github.com/pion/rtcp v1.2.14 // indirect
+ github.com/pion/rtp v1.8.6 // indirect
+ github.com/pion/sctp v1.8.16 // indirect
+ github.com/pion/sdp/v3 v3.0.9 // indirect
github.com/pion/srtp/v2 v2.0.18 // indirect
- github.com/pion/turn/v2 v2.1.4 // indirect
- github.com/pion/webrtc/v3 v3.2.23 // indirect
- github.com/prometheus/client_golang v1.17.0
- github.com/prometheus/client_model v0.5.0 // indirect
- github.com/prometheus/common v0.45.0 // indirect
- github.com/prometheus/procfs v0.12.0 // indirect
+ github.com/pion/turn/v2 v2.1.6 // indirect
+ github.com/pion/webrtc/v3 v3.2.40 // indirect
+ github.com/prometheus/client_golang v1.19.1
+ github.com/prometheus/client_model v0.6.1 // indirect
+ github.com/prometheus/common v0.53.0 // indirect
+ github.com/prometheus/procfs v0.14.0 // indirect
github.com/refraction-networking/gotapdance v1.7.10 // indirect
github.com/refraction-networking/utls v1.3.3 // indirect
github.com/sergeyfrolov/bsbuffer v0.0.0-20180903213811-94e85abb8507 // indirect
@@ -145,9 +144,9 @@ require (
github.com/xtaci/kcp-go/v5 v5.6.2 // indirect
github.com/xtaci/smux v1.5.24 // indirect
gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec // indirect
- golang.org/x/mod v0.14.0 // indirect
- golang.org/x/term v0.18.0 // indirect
- golang.org/x/text v0.14.0 // indirect
- golang.org/x/tools v0.17.0 // indirect
- google.golang.org/protobuf v1.33.0 // indirect
+ golang.org/x/mod v0.17.0 // indirect
+ golang.org/x/term v0.20.0 // indirect
+ golang.org/x/text v0.15.0 // indirect
+ golang.org/x/tools v0.21.0 // indirect
+ google.golang.org/protobuf v1.34.1 // indirect
)
diff --git a/go.sum b/go.sum
index dbe703833..c13619004 100644
--- a/go.sum
+++ b/go.sum
@@ -15,23 +15,21 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJc
github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
+github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/Psiphon-Inc/rotate-safe-writer v0.0.0-20210303140923-464a7a37606e h1:NPfqIbzmijrl0VclX2t8eO5EPBhqe47LLGKpRrcVjXk=
+github.com/Psiphon-Inc/rotate-safe-writer v0.0.0-20210303140923-464a7a37606e/go.mod h1:ZdY5pBfat/WVzw3eXbIf7N1nZN0XD5H5+X8ZMDWbCs4=
github.com/Psiphon-Labs/bolt v0.0.0-20200624191537-23cedaef7ad7 h1:Hx/NCZTnvoKZuIBwSmxE58KKoNLXIGG6hBJYN7pj9Ag=
github.com/Psiphon-Labs/bolt v0.0.0-20200624191537-23cedaef7ad7/go.mod h1:alTtZBo3j4AWFvUrAH6F5ZaHcTj4G5Y01nHz8dkU6vU=
github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464 h1:VmnMMMheFXwLV0noxYhbJbLmkV4iaVW3xNnj6xcCNHo=
github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464/go.mod h1:Pe5BqN2DdIdChorAXl6bDaQd/wghpCleJfid2NoSli0=
-github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20240312131609-91e2a902d867 h1:uTwB5Qiqg99mK3sIT2ymPTz/H6lCIDEUzEIo5Ud59/8=
-github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20240312131609-91e2a902d867/go.mod h1:PnA8vx/7sDJ31xBXD1rnT6xx8GuvWEk9un0d8vVtAyU=
-github.com/Psiphon-Labs/qtls-go1-19 v0.0.0-20230608213623-d58aa73e519a h1:O8D+GcEoZwutcERaABP2AM3RDvswBVtNmBWvlBn5wiw=
-github.com/Psiphon-Labs/qtls-go1-19 v0.0.0-20230608213623-d58aa73e519a/go.mod h1:81bbD3bvEvi3BSamZb30PgvPvqwSLfEPqwwmq5sx7fc=
-github.com/Psiphon-Labs/qtls-go1-20 v0.0.0-20230608214729-dd57d6787acf h1:bGS+WxWdHHuf42hn3M1GFSJbzCgtKNVTuiRqwCo3zyc=
-github.com/Psiphon-Labs/qtls-go1-20 v0.0.0-20230608214729-dd57d6787acf/go.mod h1:wUiSd0qyefymNlikc99B2rRC01YPN1uUvDMytMOGmF8=
-github.com/Psiphon-Labs/quic-go v0.0.0-20230626192210-73f29effc9da h1:TI2+ExyFR3A0kPrFHfaM6y3RybP0HGfP9N1R8hfZzfk=
-github.com/Psiphon-Labs/quic-go v0.0.0-20230626192210-73f29effc9da/go.mod h1:wTIxqsKVrEQIxVIIYOEHuscY+PM3h6Wz79u5aF60fo0=
-github.com/Psiphon-Labs/tls-tris v0.0.0-20230824155421-58bf6d336a9a h1:BOfU6ghaMsT/c40sWHmf3PXNwIendYXzL6tRv6NbPog=
-github.com/Psiphon-Labs/tls-tris v0.0.0-20230824155421-58bf6d336a9a/go.mod h1:v3y9GXFo9Sf2mO6auD2ExGG7oDgrK8TI7eb49ZnUxrE=
+github.com/Psiphon-Labs/psiphon-tls v0.0.0-20240424193802-52b2602ec60c h1:+SEszyxW7yu+smufzSlAszj/WmOYJ054DJjb5jllulc=
+github.com/Psiphon-Labs/psiphon-tls v0.0.0-20240424193802-52b2602ec60c/go.mod h1:AaKKoshr8RI1LZTheeNDtNuZ39qNVPWVK4uir2c2XIs=
+github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20240424194431-3612a5a6fb4c h1:fGq255KuBSpc9Odea+VhbMSbQcW07uWfypbUU0QWjVM=
+github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20240424194431-3612a5a6fb4c/go.mod h1:M5yXGzsfrz2dPQtdjssbwYNpnWlhAeIDXwX7FrG5uv8=
+github.com/Psiphon-Labs/quic-go v0.0.0-20240424181006-45545f5e1536 h1:pM5ex1QufkHV8lDR6Tc1Crk1bW5lYZjrFIJGZNBWE9k=
+github.com/Psiphon-Labs/quic-go v0.0.0-20240424181006-45545f5e1536/go.mod h1:2MTiPsgoOqWs3Bo6Xr3ElMBX6zzfjd3YkDFpQJLwHdQ=
github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
@@ -53,16 +51,16 @@ github.com/bifurcation/mint v0.0.0-20180306135233-198357931e61/go.mod h1:zVt7zX3
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY=
github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic=
github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
-github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
+github.com/cloudflare/circl v1.3.8 h1:j+V8jJt09PoeMFIu2uh5JUyEaIHTXVOHslFoLNAKqwI=
+github.com/cloudflare/circl v1.3.8/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
github.com/cognusion/go-cache-lru v0.0.0-20170419142635-f73e2280ecea h1:9C2rdYRp8Vzwhm3sbFX0yYfB+70zKFRjn7cnPCucHSw=
@@ -84,6 +82,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA=
github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc=
github.com/deckarep/golang-set v0.0.0-20171013212420-1d4478f51bed h1:njG8LmGD6JCWJu4bwIKmkOHvch70UOEIqczl5vp7Gok=
+github.com/deckarep/golang-set v0.0.0-20171013212420-1d4478f51bed/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo=
github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8=
github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE=
@@ -93,6 +92,7 @@ github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkz
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
@@ -103,8 +103,8 @@ github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d h1:wi6jN5LVt/ljaBG4ue7
github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4=
github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
-github.com/dop251/goja_nodejs v0.0.0-20231122114759-e84d9a924c5c h1:hLoodLRD4KLWIH8eyAQCLcH8EqIrjac7fCkp/fHnvuQ=
-github.com/dop251/goja_nodejs v0.0.0-20231122114759-e84d9a924c5c/go.mod h1:bhGPmCgCCTSRfiMYWjpS46IDo9EUZXlsuUaPXSWGbv0=
+github.com/dop251/goja_nodejs v0.0.0-20240418154818-2aae10d4cbcf h1:2JoVYP9iko8uuIW33BQafzaylDixXbdXCRw/vCoxL+s=
+github.com/dop251/goja_nodejs v0.0.0-20240418154818-2aae10d4cbcf/go.mod h1:bhGPmCgCCTSRfiMYWjpS46IDo9EUZXlsuUaPXSWGbv0=
github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
@@ -113,18 +113,20 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
github.com/elazarl/goproxy v0.0.0-20200809112317-0581fc3aee2d h1:rtM8HsT3NG37YPjz8sYSbUSdElP9lUsQENYzJDZDUBE=
+github.com/elazarl/goproxy v0.0.0-20200809112317-0581fc3aee2d/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
github.com/elazarl/goproxy/ext v0.0.0-20200809112317-0581fc3aee2d h1:st1tmvy+4duoRj+RaeeJoECWCWM015fBtf/4aR+hhqk=
+github.com/elazarl/goproxy/ext v0.0.0-20200809112317-0581fc3aee2d/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
-github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
+github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
github.com/florianl/go-nfqueue v1.1.1-0.20200829120558-a2f196e98ab0 h1:7ZJyJV4KiWBijCCzUPvVaqxsDxO36+KD0XKBdEN3I+8=
+github.com/florianl/go-nfqueue v1.1.1-0.20200829120558-a2f196e98ab0/go.mod h1:2z3Tfqwv2ueuK6h563xUHRcCh1mv38wS9EjiWiesk84=
github.com/flynn/noise v1.0.1 h1:vPp/jdQLXC6ppsXSj/pM3W1BIJ5FEHE2TulSJBpb43Y=
github.com/flynn/noise v1.0.1/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk=
github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI=
github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs=
@@ -132,30 +134,29 @@ github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpj
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
-github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
+github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
+github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
+github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
-github.com/gobuffalo/logger v1.0.6 h1:nnZNpxYo0zx+Aj9RfMPBm+x9zAU2OayFh/xrAWi34HU=
-github.com/gobuffalo/packd v1.0.1 h1:U2wXfRr4E9DH8IdsDLlRFwTZTK7hLfq9qT/QHXGVe/0=
-github.com/gobuffalo/packr/v2 v2.8.3 h1:xE1yzvnO56cUC0sTpKR3DIbxZgB54AftTFMhB2XEWlY=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/gobwas/glob v0.2.4-0.20180402141543-f00a7392b439 h1:T6zlOdzrYuHf6HUKujm9bzkzbZ5Iv/xf6rs8BHZDpoI=
+github.com/gobwas/glob v0.2.4-0.20180402141543-f00a7392b439/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68=
-github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
+github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4=
+github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -167,7 +168,6 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
@@ -183,8 +183,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg=
-github.com/google/pprof v0.0.0-20231212022811-ec68065c825e h1:bwOy7hAFd0C91URzMIEBfr6BAz29yk7Qj0cy6S7DJlU=
-github.com/google/pprof v0.0.0-20231212022811-ec68065c825e/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
+github.com/google/pprof v0.0.0-20240509144519-723abb6459b7 h1:velgFPYr1X9TDwLIfkV7fWqsFlf7TeP11M/7kPd/dVI=
+github.com/google/pprof v0.0.0-20240509144519-723abb6459b7/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
@@ -193,6 +193,7 @@ github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
github.com/grafov/m3u8 v0.12.0 h1:T6iTwTsSEtMcwkayef+FJO8kj+Sglr4Lh81Zj8Ked/4=
@@ -251,16 +252,17 @@ github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv
github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/josharian/native v1.0.0 h1:Ts/E8zCSEsG17dUqv7joXJFybuMLjQfWE04tsBODTxk=
+github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI=
github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
-github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
-github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid/v2 v2.0.14/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
@@ -276,6 +278,7 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
@@ -290,10 +293,8 @@ github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s=
github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI=
-github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY=
-github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI=
github.com/marusama/semaphore v0.0.0-20171214154724-565ffd8e868a h1:6SRny9FLB1eWasPyDUqBQnMi9NhXU01XIlB0ao89YoI=
+github.com/marusama/semaphore v0.0.0-20171214154724-565ffd8e868a/go.mod h1:TmeOqAKoDinfPfSohs14CO3VcEf7o+Bem6JiNe05yrQ=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
@@ -309,13 +310,13 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
-github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
-github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
github.com/mdlayher/netlink v1.4.2-0.20210930205308-a81a8c23d40a h1:yk5OmRew64lWdeNanQ3l0hDgUt1E8MfipPhh/GO9Tuw=
+github.com/mdlayher/netlink v1.4.2-0.20210930205308-a81a8c23d40a/go.mod h1:qw8F9IVzxa0GpqhVAfOw8DNyo7ec/jxI6bPWPEg1MV4=
github.com/mdlayher/socket v0.0.0-20210624160740-9dbe287ded84 h1:L1jnQ6o+K3M574eez7eTxbsia6H1SfJaVpaXY33L37Q=
+github.com/mdlayher/socket v0.0.0-20210624160740-9dbe287ded84/go.mod h1:GAFlyu4/XV68LkQKYzKhIo/WW7j3Zi0YRAz/BOoanUc=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
-github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4=
-github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY=
+github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs=
+github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -325,84 +326,77 @@ github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8
github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
github.com/mroth/weightedrand v1.0.0 h1:V8JeHChvl2MP1sAoXq4brElOcza+jxLkRuwvtQu8L3E=
github.com/mroth/weightedrand v1.0.0/go.mod h1:3p2SIcC8al1YMzGhAIoXD+r9olo/g/cdJgAD905gyNE=
-github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
-github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
-github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs=
-github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM=
+github.com/onsi/ginkgo/v2 v2.17.3 h1:oJcvKpIb7/8uLpDDtnQuf18xVnwKp8DTD7DQ6gTd/MU=
+github.com/onsi/ginkgo/v2 v2.17.3/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
-github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
-github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
+github.com/onsi/gomega v1.33.0 h1:snPCflnZrpMsy94p4lXVEkHo12lmPnc3vY5XBbreexE=
+github.com/onsi/gomega v1.33.0/go.mod h1:+925n5YtiFsLzzafLUHzVMBpvvRAzrydIBiSIxjX3wY=
github.com/ooni/netem v0.0.0-20240208095707-608dcbcd82b8 h1:kJ2wn19lIP/y9ng85BbFRdWKHK6Er116Bbt5uhqHVD4=
github.com/ooni/netem v0.0.0-20240208095707-608dcbcd82b8/go.mod h1:b/wAvTR5n92Vk2b0SBmuMU0xO4ZGVrsXtU7zjTby7vw=
-github.com/ooni/oocrypto v0.5.8 h1:eoixlF81pQFGAD6LR5IWz5aIwgnolyYnkPti7eQh5n8=
-github.com/ooni/oocrypto v0.5.8/go.mod h1:3CMXIpx7vpoZp73fzUuLvuZApfFlKmdrkKJu3QC4hhs=
-github.com/ooni/oohttp v0.6.8 h1:yqLXhYO0VXSo56impKOSlVVY+r7TbirJNiash3kkeeg=
-github.com/ooni/oohttp v0.6.8/go.mod h1:2IGLODw3KeJozKrwhGLVHWC7Ocihi8l5+HbhqLmxt6I=
-github.com/ooni/probe-assets v0.22.0 h1:Nxl7X+HdTamGKKzA4g0OKZY7zOPTY8IcHe/G9A/zrcc=
-github.com/ooni/probe-assets v0.22.0/go.mod h1:m0k2FFzcLfFm7dhgyYkLCUR3R0CoRPr0jcjctDS2+gU=
+github.com/ooni/oocrypto v0.6.1 h1:D0fGokmHoVKGBy39RxPxK77ov0Ob9Z5pdx4vKA6vpWk=
+github.com/ooni/oocrypto v0.6.1/go.mod h1:mGlPZeI3jV1gnVQ3xs5WYNo8IoYlyB/p/x79P58hhog=
+github.com/ooni/oohttp v0.7.2 h1:MtZFqq2vd0/ppYTjhKMSO5K6sj2NxVjrO7ldElWtHq4=
+github.com/ooni/oohttp v0.7.2/go.mod h1:6mjMEE8uA2wODu93EABmtmbjy0/YuORNNcRY9Dw2ncw=
+github.com/ooni/probe-assets v0.23.0 h1:QNagQEgyRpM5tC0yHhUU3x0+L4gjr8p8nzMl+TUI3Vo=
+github.com/ooni/probe-assets v0.23.0/go.mod h1:m0k2FFzcLfFm7dhgyYkLCUR3R0CoRPr0jcjctDS2+gU=
github.com/oschwald/geoip2-golang v1.9.0 h1:uvD3O6fXAXs+usU+UGExshpdP13GAqp4GBrzN7IgKZc=
+github.com/oschwald/geoip2-golang v1.9.0/go.mod h1:BHK6TvDyATVQhKNbQBdrj9eAvuwOMi2zSFXizL3K81Y=
github.com/oschwald/maxminddb-golang v1.12.0 h1:9FnTOD0YOhP7DGxGsq4glzpGy5+w7pq50AS6wALUMYs=
github.com/oschwald/maxminddb-golang v1.12.0/go.mod h1:q0Nob5lTCqyQ8WT6FYgS1L7PXKVVbgiymefNwIjPzgY=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pborman/getopt/v2 v2.1.0 h1:eNfR+r+dWLdWmV8g5OlpyrTYHkhVNxHBdN2cCrJmOEA=
github.com/pborman/getopt/v2 v2.1.0/go.mod h1:4NtW75ny4eBw9fO1bhtNdYTlZKYX5/tBLtsOpwKIKd0=
github.com/pebbe/zmq4 v1.2.10 h1:wQkqRZ3CZeABIeidr3e8uQZMMH5YAykA/WN0L5zkd1c=
+github.com/pebbe/zmq4 v1.2.10/go.mod h1:nqnPueOapVhE2wItZ0uOErngczsJdLOGkebMxaO8r48=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
-github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8=
-github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0=
+github.com/pion/datachannel v1.5.6 h1:1IxKJntfSlYkpUj8LlYRSWpYiTTC02nUrOE8T3DqGeg=
+github.com/pion/datachannel v1.5.6/go.mod h1:1eKT6Q85pRnr2mHiWHxJwO50SfZRtWHTsNIVb/NfGW4=
github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=
-github.com/pion/dtls/v2 v2.2.8 h1:BUroldfiIbV9jSnC6cKOMnyiORRWrWWpV11JUyEu5OA=
-github.com/pion/dtls/v2 v2.2.8/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=
-github.com/pion/ice/v2 v2.3.11 h1:rZjVmUwyT55cmN8ySMpL7rsS8KYsJERsrxJLLxpKhdw=
-github.com/pion/ice/v2 v2.3.11/go.mod h1:hPcLC3kxMa+JGRzMHqQzjoSj3xtE9F+eoncmXLlCL4E=
-github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc=
-github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y=
+github.com/pion/dtls/v2 v2.2.11 h1:9U/dpCYl1ySttROPWJgqWKEylUdT0fXp/xst6JwY5Ks=
+github.com/pion/dtls/v2 v2.2.11/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE=
+github.com/pion/ice/v2 v2.3.24 h1:RYgzhH/u5lH0XO+ABatVKCtRd+4U1GEaCXSMjNr13tI=
+github.com/pion/ice/v2 v2.3.24/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw=
+github.com/pion/interceptor v0.1.29 h1:39fsnlP1U8gw2JzOFWdfCU82vHvhW9o0rZnZF56wF+M=
+github.com/pion/interceptor v0.1.29/go.mod h1:ri+LGNjRUc5xUNtDEPzfdkmSqISixVTBF/z/Zms/6T4=
github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms=
-github.com/pion/mdns v0.0.8/go.mod h1:hYE72WX8WDveIhg7fmXgMKivD3Puklk0Ymzog0lSyaI=
-github.com/pion/mdns v0.0.9 h1:7Ue5KZsqq8EuqStnpPWV33vYYEH0+skdDN5L7EiEsI4=
-github.com/pion/mdns v0.0.9/go.mod h1:2JA5exfxwzXiCihmxpTKgFUpiQws2MnipoPK09vecIc=
+github.com/pion/mdns v0.0.12 h1:CiMYlY+O0azojWDmxdNr7ADGrnZ+V6Ilfner+6mSVK8=
+github.com/pion/mdns v0.0.12/go.mod h1:VExJjv8to/6Wqm1FXK+Ii/Z9tsVk/F5sD/N70cnYFbk=
github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
-github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I=
github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4=
-github.com/pion/rtcp v1.2.13 h1:+EQijuisKwm/8VBs8nWllr0bIndR7Lf7cZG200mpbNo=
-github.com/pion/rtcp v1.2.13/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4=
-github.com/pion/rtp v1.8.2/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU=
-github.com/pion/rtp v1.8.3 h1:VEHxqzSVQxCkKDSHro5/4IUUG1ea+MFdqR2R3xSpNU8=
+github.com/pion/rtcp v1.2.14 h1:KCkGV3vJ+4DAJmvP0vaQShsb0xkRfWkO540Gy102KyE=
+github.com/pion/rtcp v1.2.14/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4=
github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU=
-github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0=
-github.com/pion/sctp v1.8.8/go.mod h1:igF9nZBrjh5AtmKc7U30jXltsFHicFCXSmWA2GWRaWs=
-github.com/pion/sctp v1.8.9 h1:TP5ZVxV5J7rz7uZmbyvnUvsn7EJ2x/5q9uhsTtXbI3g=
-github.com/pion/sctp v1.8.9/go.mod h1:cMLT45jqw3+jiJCrtHVwfQLnfR0MGZ4rgOJwUOIqLkI=
-github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw=
-github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw=
+github.com/pion/rtp v1.8.6 h1:MTmn/b0aWWsAzux2AmP8WGllusBVw4NPYPVFFd7jUPw=
+github.com/pion/rtp v1.8.6/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU=
+github.com/pion/sctp v1.8.13/go.mod h1:YKSgO/bO/6aOMP9LCie1DuD7m+GamiK2yIiPM6vH+GA=
+github.com/pion/sctp v1.8.16 h1:PKrMs+o9EMLRvFfXq59WFsC+V8mN1wnKzqrv+3D/gYY=
+github.com/pion/sctp v1.8.16/go.mod h1:P6PbDVA++OJMrVNg2AL3XtYHV4uD6dvfyOovCgMs0PE=
+github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY=
+github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M=
github.com/pion/srtp/v2 v2.0.18 h1:vKpAXfawO9RtTRKZJbG4y0v1b11NZxQnxRl85kGuUlo=
github.com/pion/srtp/v2 v2.0.18/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA=
github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4=
github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8=
-github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40=
-github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI=
github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g=
github.com/pion/transport/v2 v2.2.2/go.mod h1:OJg3ojoBJopjEeECq2yJdXH9YVrUJ1uQ++NjXLOUorc=
github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=
-github.com/pion/transport/v2 v2.2.4 h1:41JJK6DZQYSeVLxILA2+F4ZkKb4Xd/tFJZRFZQ9QAlo=
github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=
-github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM=
+github.com/pion/transport/v2 v2.2.5 h1:iyi25i/21gQck4hfRhomF6SktmUQjRsRW4WJdhfc3Kc=
+github.com/pion/transport/v2 v2.2.5/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=
github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0=
+github.com/pion/transport/v3 v3.0.2 h1:r+40RJR25S9w3jbA6/5uEPTzcdn7ncyU44RWCbHkLg4=
+github.com/pion/transport/v3 v3.0.2/go.mod h1:nIToODoOlb5If2jF9y2Igfx3PFYWfuXi37m0IlWa/D0=
github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
-github.com/pion/turn/v2 v2.1.4 h1:2xn8rduI5W6sCZQkEnIUDAkrBQNl2eYIBCHMZ3QMmP8=
-github.com/pion/turn/v2 v2.1.4/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
-github.com/pion/webrtc/v3 v3.2.23 h1:GbqEuxBbVLFhXk0GwxKAoaIJYiEa9TyoZPEZC+2HZxM=
-github.com/pion/webrtc/v3 v3.2.23/go.mod h1:1CaT2fcZzZ6VZA+O1i9yK2DU4EOcXVvSbWG9pr5jefs=
+github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc=
+github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
+github.com/pion/webrtc/v3 v3.2.40 h1:Wtfi6AZMQg+624cvCXUuSmrKWepSB7zfgYDOYqsSOVU=
+github.com/pion/webrtc/v3 v3.2.40/go.mod h1:M1RAe3TNTD1tzyvqHrbVODfwdPGSXOUo/OgpoGGJqFY=
github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -410,23 +404,22 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY=
-github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
-github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
+github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg=
+github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
+github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
-github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
-github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM=
-github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
-github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
-github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE=
+github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U=
+github.com/prometheus/procfs v0.14.0 h1:Lw4VdGGoKEZilJsayHf0B+9YgLGREba2C6xr+Fdfq6s=
+github.com/prometheus/procfs v0.14.0/go.mod h1:XL+Iwz8k8ZabyZfMFHPiilCniixqQarAy5Mu67pHlNQ=
github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
-github.com/quic-go/qtls-go1-20 v0.4.1 h1:D33340mCNDAIKBqXuAvexTNMUByrYmFYVfKfDN5nfFs=
-github.com/quic-go/qtls-go1-20 v0.4.1/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
-github.com/quic-go/quic-go v0.40.1 h1:X3AGzUNFs0jVuO3esAGnTfvdgvL4fq655WaOi1snv1Q=
-github.com/quic-go/quic-go v0.40.1/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpTx31vx60c=
-github.com/refraction-networking/conjure v0.7.10 h1:QGH2wna/9cxu760a/RbE6GhEaElGk7Uagj0epJeprZg=
-github.com/refraction-networking/conjure v0.7.10/go.mod h1:7KuAtYfSL0K0WpCScjN9YKiOZ4AQ/8IzSjUtVwWbSv8=
+github.com/quic-go/quic-go v0.43.1 h1:fLiMNfQVe9q2JvSsiXo4fXOEguXHGGl9+6gLp4RPeZQ=
+github.com/quic-go/quic-go v0.43.1/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M=
+github.com/refraction-networking/conjure v0.7.11-0.20240130155008-c8df96195ab2 h1:m2ZH6WV69otVmBpWbk8et3MypHFsjcYXTNrknQKS/PY=
+github.com/refraction-networking/conjure v0.7.11-0.20240130155008-c8df96195ab2/go.mod h1:7KuAtYfSL0K0WpCScjN9YKiOZ4AQ/8IzSjUtVwWbSv8=
github.com/refraction-networking/ed25519 v0.1.2 h1:08kJZUkAlY7a7cZGosl1teGytV+QEoNxPO7NnRvAB+g=
github.com/refraction-networking/ed25519 v0.1.2/go.mod h1:nxYLUAYt/hmNpAh64PNSQ/tQ9gTIB89wCaGKJlRtZ9I=
github.com/refraction-networking/gotapdance v1.7.10 h1:vPtvuihP95SqrnnpX//KI1PTqrXCvNnOQslrG4gxsRY=
@@ -448,15 +441,15 @@ github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
-github.com/rubenv/sql-migrate v1.5.2 h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0=
-github.com/rubenv/sql-migrate v1.5.2/go.mod h1:H38GW8Vqf8F0Su5XignRyaRcbXbJunSWxs+kmzlg0Is=
+github.com/rubenv/sql-migrate v1.6.1 h1:bo6/sjsan9HaXAsNxYP/jCEDUGibHp8JmOBw7NTGRos=
+github.com/rubenv/sql-migrate v1.6.1/go.mod h1:tPzespupJS0jacLfhbwto/UjSX+8h2FdWB7ar+QlHa0=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735 h1:7YvPJVmEeFHR1Tj9sZEYsmarJEQfMVYpd/Vyy/A8dqE=
+github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/schollz/progressbar/v3 v3.14.2 h1:EducH6uNLIWsr560zSV1KrTeUb/wZGAHqyMFIEa99ks=
github.com/schollz/progressbar/v3 v3.14.2/go.mod h1:aQAZQnhF4JGFtRJiw/eobaXpsqpVQAftEQ+hLGXaRc4=
-github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM=
github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY=
github.com/sergeyfrolov/bsbuffer v0.0.0-20180903213811-94e85abb8507 h1:ML7ZNtcln5UBo5Wv7RIv9Xg3Pr5VuRCWLFXEwda54Y4=
@@ -472,6 +465,7 @@ github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsR
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -490,6 +484,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -500,8 +495,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a h1:SJy1Pu0eH1C29XwJucQo73FrleVK6t4kYz4NVhp34Yw=
@@ -535,7 +531,6 @@ github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae h1:J0GxkO96kL4WF+A
github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62kEgmN++bm9BVICuT/e8yiLI2KFobd/TRFsE=
github.com/xtaci/smux v1.5.24 h1:77emW9dtnOxxOQ5ltR+8BbsX1kzcOxQ5gB+aaV9hXOY=
github.com/xtaci/smux v1.5.24/go.mod h1:OMlQbT5vcgl2gb49mFkYo6SMf+zP3rcjcwQz7ZU7IGY=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec h1:FpfFs4EhNehiVfzQttTuxanPIT43FtkkCFypIod8LHo=
@@ -554,8 +549,8 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
-go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo=
-go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
+go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
+go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
@@ -583,16 +578,18 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
-golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb h1:c0vyKkb6yr3KR7jEfJaOSv4lG7xPkbN6r52aJz1d8a8=
-golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
+golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
+golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
golang.org/x/exp/typeparams v0.0.0-20230522175609-2e198f4a06a1 h1:pnP8r+W8Fm7XJ8CWtXi4S9oJmPBTrkfYN/dNbaPj6Y4=
golang.org/x/exp/typeparams v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -602,11 +599,10 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
-golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -616,34 +612,31 @@ golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
-golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
+golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -656,17 +649,12 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -677,8 +665,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -687,23 +673,25 @@ golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo=
golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -711,15 +699,14 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -735,11 +722,10 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
-golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
+golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw=
+golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -755,6 +741,7 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
+google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -763,8 +750,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
-google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
+google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -778,9 +765,7 @@ gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3M
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -794,6 +779,7 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.4.2 h1:6qXr+R5w+ktL5UkwEbPp+fEvfyoMPche6GkOpGHZcLc=
+honnef.co/go/tools v0.4.2/go.mod h1:36ZgoUOrqOk1GxwHhyryEkq8FQWkUO2xGuSMhUCcdvA=
modernc.org/b v1.0.4/go.mod h1:Oqc2xtmGT0tvBUsPZIanirLhxBCQZhM7Lu3TlzBj9w8=
modernc.org/b v1.1.0/go.mod h1:yF+wmBAFjebNdVqZNTeNfmnLaLqq91wozvDLcuXz+ck=
modernc.org/db v1.0.8/go.mod h1:L8Az96H46DF2+BGeaS6+WiEqLORR2sjp0yBn6LA/lAQ=
diff --git a/pkg/checkincache/checkincache.go b/pkg/checkincache/checkincache.go
index 566f0d041..c9dad2ef1 100644
--- a/pkg/checkincache/checkincache.go
+++ b/pkg/checkincache/checkincache.go
@@ -10,8 +10,8 @@ import (
"github.com/ooni/probe-engine/pkg/runtimex"
)
-// checkInFlagsState is the state created by check-in flags.
-const checkInFlagsState = "checkinflags.state"
+// CheckInFlagsState is the state created by check-in flags.
+const CheckInFlagsState = "checkinflags.state"
// checkInFlagsWrapper is the struct wrapping the check-in flags.
//
@@ -37,13 +37,13 @@ func Store(kvStore model.KeyValueStore, resp *model.OOAPICheckInResult) error {
}
data, err := json.Marshal(wrapper)
runtimex.PanicOnError(err, "json.Marshal unexpectedly failed")
- return kvStore.Set(checkInFlagsState, data)
+ return kvStore.Set(CheckInFlagsState, data)
}
// GetFeatureFlag returns the value of a check-in feature flag. In case of any
// error this function will always return a false value.
func GetFeatureFlag(kvStore model.KeyValueStore, name string) bool {
- data, err := kvStore.Get(checkInFlagsState)
+ data, err := kvStore.Get(CheckInFlagsState)
if err != nil {
return false // as documented
}
diff --git a/pkg/checkincache/checkincache_test.go b/pkg/checkincache/checkincache_test.go
index 401eb67cd..ff8f7a9b5 100644
--- a/pkg/checkincache/checkincache_test.go
+++ b/pkg/checkincache/checkincache_test.go
@@ -28,7 +28,7 @@ func TestStore(t *testing.T) {
t.Fatal(err)
}
var wrapper checkInFlagsWrapper
- data, err := memstore.Get(checkInFlagsState)
+ data, err := memstore.Get(CheckInFlagsState)
if err != nil {
t.Fatal(err)
}
diff --git a/pkg/cmd/apitool/main.go b/pkg/cmd/apitool/main.go
index db50285cf..438c75e48 100644
--- a/pkg/cmd/apitool/main.go
+++ b/pkg/cmd/apitool/main.go
@@ -17,7 +17,6 @@ import (
"sync/atomic"
"github.com/apex/log"
- "github.com/ooni/probe-engine/pkg/httpx"
"github.com/ooni/probe-engine/pkg/kvstore"
"github.com/ooni/probe-engine/pkg/model"
"github.com/ooni/probe-engine/pkg/netxlite"
@@ -32,16 +31,14 @@ func newclient() probeservices.Client {
txp := netx.NewHTTPTransportStdlib(log.Log)
ua := fmt.Sprintf("apitool/%s ooniprobe-engine/%s", version.Version, version.Version)
return probeservices.Client{
- APIClientTemplate: httpx.APIClientTemplate{
- BaseURL: *backend,
- HTTPClient: &http.Client{Transport: txp},
- Logger: log.Log,
- UserAgent: ua,
- },
+ BaseURL: *backend,
+ HTTPClient: &http.Client{Transport: txp},
KVStore: &kvstore.Memory{},
+ Logger: log.Log,
LoginCalls: &atomic.Int64{},
RegisterCalls: &atomic.Int64{},
StateFile: probeservices.NewStateFile(&kvstore.Memory{}),
+ UserAgent: ua,
}
}
diff --git a/pkg/cmd/buildtool/android_test.go b/pkg/cmd/buildtool/android_test.go
index 1386c4b79..010889802 100644
--- a/pkg/cmd/buildtool/android_test.go
+++ b/pkg/cmd/buildtool/android_test.go
@@ -702,12 +702,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) {
expect: []buildtooltest.ExecExpectations{{
Env: []string{},
Argv: []string{
- "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.2.1.tar.gz",
+ "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.0.tar.gz",
},
}, {
Env: []string{},
Argv: []string{
- "tar", "-xf", "openssl-3.2.1.tar.gz",
+ "tar", "-xf", "openssl-3.3.0.tar.gz",
},
}, {
Env: []string{},
@@ -757,12 +757,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) {
}, {
Env: []string{},
Argv: []string{
- "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.2.1.tar.gz",
+ "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.0.tar.gz",
},
}, {
Env: []string{},
Argv: []string{
- "tar", "-xf", "openssl-3.2.1.tar.gz",
+ "tar", "-xf", "openssl-3.3.0.tar.gz",
},
}, {
Env: []string{},
@@ -812,12 +812,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) {
}, {
Env: []string{},
Argv: []string{
- "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.2.1.tar.gz",
+ "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.0.tar.gz",
},
}, {
Env: []string{},
Argv: []string{
- "tar", "-xf", "openssl-3.2.1.tar.gz",
+ "tar", "-xf", "openssl-3.3.0.tar.gz",
},
}, {
Env: []string{},
@@ -867,12 +867,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) {
}, {
Env: []string{},
Argv: []string{
- "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.2.1.tar.gz",
+ "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.0.tar.gz",
},
}, {
Env: []string{},
Argv: []string{
- "tar", "-xf", "openssl-3.2.1.tar.gz",
+ "tar", "-xf", "openssl-3.3.0.tar.gz",
},
}, {
Env: []string{},
@@ -1738,12 +1738,12 @@ func TestAndroidBuildCdepsTor(t *testing.T) {
expect: []buildtooltest.ExecExpectations{{
Env: []string{},
Argv: []string{
- "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.10.tar.gz",
+ "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.11.tar.gz",
},
}, {
Env: []string{},
Argv: []string{
- "tar", "-xf", "tor-0.4.8.10.tar.gz",
+ "tar", "-xf", "tor-0.4.8.11.tar.gz",
},
}, {
Env: []string{},
@@ -1827,12 +1827,12 @@ func TestAndroidBuildCdepsTor(t *testing.T) {
}, {
Env: []string{},
Argv: []string{
- "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.10.tar.gz",
+ "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.11.tar.gz",
},
}, {
Env: []string{},
Argv: []string{
- "tar", "-xf", "tor-0.4.8.10.tar.gz",
+ "tar", "-xf", "tor-0.4.8.11.tar.gz",
},
}, {
Env: []string{},
@@ -1916,12 +1916,12 @@ func TestAndroidBuildCdepsTor(t *testing.T) {
}, {
Env: []string{},
Argv: []string{
- "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.10.tar.gz",
+ "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.11.tar.gz",
},
}, {
Env: []string{},
Argv: []string{
- "tar", "-xf", "tor-0.4.8.10.tar.gz",
+ "tar", "-xf", "tor-0.4.8.11.tar.gz",
},
}, {
Env: []string{},
@@ -2005,12 +2005,12 @@ func TestAndroidBuildCdepsTor(t *testing.T) {
}, {
Env: []string{},
Argv: []string{
- "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.10.tar.gz",
+ "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.11.tar.gz",
},
}, {
Env: []string{},
Argv: []string{
- "tar", "-xf", "tor-0.4.8.10.tar.gz",
+ "tar", "-xf", "tor-0.4.8.11.tar.gz",
},
}, {
Env: []string{},
diff --git a/pkg/cmd/buildtool/cdepsopenssl.go b/pkg/cmd/buildtool/cdepsopenssl.go
index c0a6e87ba..8f24439b1 100644
--- a/pkg/cmd/buildtool/cdepsopenssl.go
+++ b/pkg/cmd/buildtool/cdepsopenssl.go
@@ -27,13 +27,13 @@ func cdepsOpenSSLBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencie
defer restore()
// See https://github.com/Homebrew/homebrew-core/blob/master/Formula/o/openssl@3.rb
- cdepsMustFetch("https://www.openssl.org/source/openssl-3.2.1.tar.gz")
+ cdepsMustFetch("https://www.openssl.org/source/openssl-3.3.0.tar.gz")
deps.VerifySHA256( // must be mockable
- "83c7329fe52c850677d75e5d0b0ca245309b97e8ecbcfdc1dfdc4ab9fac35b39",
- "openssl-3.2.1.tar.gz",
+ "53e66b043322a606abf0087e7699a0e033a37fa13feb9742df35c3a33b18fb02",
+ "openssl-3.3.0.tar.gz",
)
- must.Run(log.Log, "tar", "-xf", "openssl-3.2.1.tar.gz")
- _ = deps.MustChdir("openssl-3.2.1") // must be mockable
+ must.Run(log.Log, "tar", "-xf", "openssl-3.3.0.tar.gz")
+ _ = deps.MustChdir("openssl-3.3.0") // must be mockable
mydir := filepath.Join(topdir, "CDEPS", "openssl")
for _, patch := range cdepsMustListPatches(mydir) {
diff --git a/pkg/cmd/buildtool/cdepstor.go b/pkg/cmd/buildtool/cdepstor.go
index b88ab7b0e..8e7ca852f 100644
--- a/pkg/cmd/buildtool/cdepstor.go
+++ b/pkg/cmd/buildtool/cdepstor.go
@@ -27,13 +27,13 @@ func cdepsTorBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencies) {
defer restore()
// See https://github.com/Homebrew/homebrew-core/blob/master/Formula/t/tor.rb
- cdepsMustFetch("https://www.torproject.org/dist/tor-0.4.8.10.tar.gz")
+ cdepsMustFetch("https://www.torproject.org/dist/tor-0.4.8.11.tar.gz")
deps.VerifySHA256( // must be mockable
- "e628b4fab70edb4727715b23cf2931375a9f7685ac08f2c59ea498a178463a86",
- "tor-0.4.8.10.tar.gz",
+ "8f2bdf90e63380781235aa7d604e159570f283ecee674670873d8bb7052c8e07",
+ "tor-0.4.8.11.tar.gz",
)
- must.Run(log.Log, "tar", "-xf", "tor-0.4.8.10.tar.gz")
- _ = deps.MustChdir("tor-0.4.8.10") // must be mockable
+ must.Run(log.Log, "tar", "-xf", "tor-0.4.8.11.tar.gz")
+ _ = deps.MustChdir("tor-0.4.8.11") // must be mockable
mydir := filepath.Join(topdir, "CDEPS", "tor")
for _, patch := range cdepsMustListPatches(mydir) {
diff --git a/pkg/cmd/buildtool/ios_test.go b/pkg/cmd/buildtool/ios_test.go
index fa8da588b..c7eed181f 100644
--- a/pkg/cmd/buildtool/ios_test.go
+++ b/pkg/cmd/buildtool/ios_test.go
@@ -349,12 +349,12 @@ func TestIOSBuildCdepsOpenSSL(t *testing.T) {
expect: []buildtooltest.ExecExpectations{{
Env: []string{},
Argv: []string{
- "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.2.1.tar.gz",
+ "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.0.tar.gz",
},
}, {
Env: []string{},
Argv: []string{
- "tar", "-xf", "openssl-3.2.1.tar.gz",
+ "tar", "-xf", "openssl-3.3.0.tar.gz",
},
}, {
Env: []string{},
@@ -399,12 +399,12 @@ func TestIOSBuildCdepsOpenSSL(t *testing.T) {
}, {
Env: []string{},
Argv: []string{
- "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.2.1.tar.gz",
+ "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.0.tar.gz",
},
}, {
Env: []string{},
Argv: []string{
- "tar", "-xf", "openssl-3.2.1.tar.gz",
+ "tar", "-xf", "openssl-3.3.0.tar.gz",
},
}, {
Env: []string{},
@@ -449,12 +449,12 @@ func TestIOSBuildCdepsOpenSSL(t *testing.T) {
}, {
Env: []string{},
Argv: []string{
- "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.2.1.tar.gz",
+ "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.0.tar.gz",
},
}, {
Env: []string{},
Argv: []string{
- "tar", "-xf", "openssl-3.2.1.tar.gz",
+ "tar", "-xf", "openssl-3.3.0.tar.gz",
},
}, {
Env: []string{},
@@ -1142,12 +1142,12 @@ func TestIOSBuildCdepsTor(t *testing.T) {
expect: []buildtooltest.ExecExpectations{{
Env: []string{},
Argv: []string{
- "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.10.tar.gz",
+ "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.11.tar.gz",
},
}, {
Env: []string{},
Argv: []string{
- "tar", "-xf", "tor-0.4.8.10.tar.gz",
+ "tar", "-xf", "tor-0.4.8.11.tar.gz",
},
}, {
Env: []string{},
@@ -1232,12 +1232,12 @@ func TestIOSBuildCdepsTor(t *testing.T) {
}, {
Env: []string{},
Argv: []string{
- "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.10.tar.gz",
+ "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.11.tar.gz",
},
}, {
Env: []string{},
Argv: []string{
- "tar", "-xf", "tor-0.4.8.10.tar.gz",
+ "tar", "-xf", "tor-0.4.8.11.tar.gz",
},
}, {
Env: []string{},
@@ -1322,12 +1322,12 @@ func TestIOSBuildCdepsTor(t *testing.T) {
}, {
Env: []string{},
Argv: []string{
- "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.10.tar.gz",
+ "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.11.tar.gz",
},
}, {
Env: []string{},
Argv: []string{
- "tar", "-xf", "tor-0.4.8.10.tar.gz",
+ "tar", "-xf", "tor-0.4.8.11.tar.gz",
},
}, {
Env: []string{},
diff --git a/pkg/cmd/buildtool/linuxcdeps_test.go b/pkg/cmd/buildtool/linuxcdeps_test.go
index 2180c4f25..58879dd10 100644
--- a/pkg/cmd/buildtool/linuxcdeps_test.go
+++ b/pkg/cmd/buildtool/linuxcdeps_test.go
@@ -92,12 +92,12 @@ func TestLinuxCdepsBuildMain(t *testing.T) {
expect: []buildtooltest.ExecExpectations{{
Env: []string{},
Argv: []string{
- "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.2.1.tar.gz",
+ "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.0.tar.gz",
},
}, {
Env: []string{},
Argv: []string{
- "tar", "-xf", "openssl-3.2.1.tar.gz",
+ "tar", "-xf", "openssl-3.3.0.tar.gz",
},
}, {
Env: []string{},
@@ -322,12 +322,12 @@ func TestLinuxCdepsBuildMain(t *testing.T) {
expect: []buildtooltest.ExecExpectations{{
Env: []string{},
Argv: []string{
- "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.10.tar.gz",
+ "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.11.tar.gz",
},
}, {
Env: []string{},
Argv: []string{
- "tar", "-xf", "tor-0.4.8.10.tar.gz",
+ "tar", "-xf", "tor-0.4.8.11.tar.gz",
},
}, {
Env: []string{},
diff --git a/pkg/cmd/e2epostprocess/main.go b/pkg/cmd/e2epostprocess/main.go
index 4f857a5a9..75849af22 100644
--- a/pkg/cmd/e2epostprocess/main.go
+++ b/pkg/cmd/e2epostprocess/main.go
@@ -48,7 +48,7 @@ func main() {
}
var found int
for _, file := range files {
- data, err := os.ReadFile(file)
+ data, err := os.ReadFile(file) // #nosec G304 - this is working as intended
fatalOnError(err)
measurements := bytes.Split(data, []byte("\n"))
for _, measurement := range measurements {
@@ -72,7 +72,7 @@ func main() {
options = append(options, *entry.Input)
}
log.Printf("run: go %s", strings.Join(options, " "))
- cmd := execabs.Command("go", options...)
+ cmd := execabs.Command("go", options...) // #nosec G204 - this is working as intended
cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
err = cmd.Run()
fatalOnError(err)
diff --git a/pkg/cmd/gardener/internal/dnsfix/dnsfix.go b/pkg/cmd/gardener/internal/dnsfix/dnsfix.go
index 20a2461dd..922dae567 100644
--- a/pkg/cmd/gardener/internal/dnsfix/dnsfix.go
+++ b/pkg/cmd/gardener/internal/dnsfix/dnsfix.go
@@ -46,7 +46,7 @@ func (s *Subcommand) Main() {
// walk through each entry
for _, entry := range entries {
- bar.Add(1)
+ _ = bar.Add(1)
s.processEntry(entry)
}
}
diff --git a/pkg/cmd/gardener/internal/dnsreport/dnsreport.go b/pkg/cmd/gardener/internal/dnsreport/dnsreport.go
index 795b906d0..f34fa8343 100644
--- a/pkg/cmd/gardener/internal/dnsreport/dnsreport.go
+++ b/pkg/cmd/gardener/internal/dnsreport/dnsreport.go
@@ -220,7 +220,7 @@ func (s *Subcommand) measureEntries(ctx context.Context, db *sql.DB, entries []*
// walk through each entry until we're interrupted by the context
for idx := 0; idx < len(entries) && ctx.Err() == nil; idx++ {
- bar.Add(1)
+ _ = bar.Add(1)
s.measureSingleEntry(db, entries[idx])
}
}
diff --git a/pkg/cmd/gardener/internal/testlists/testlists.go b/pkg/cmd/gardener/internal/testlists/testlists.go
index f43fefe5a..ee734cf8c 100644
--- a/pkg/cmd/gardener/internal/testlists/testlists.go
+++ b/pkg/cmd/gardener/internal/testlists/testlists.go
@@ -146,7 +146,7 @@ func emit(filepath string, all []*Entry, och chan<- *Entry) {
progressbar.OptionSetWriter(os.Stdout),
)
for _, entry := range all {
- bar.Add(1)
+ _ = bar.Add(1)
och <- entry
}
}
@@ -198,7 +198,7 @@ func csvReadAndFilter(filepath string, shouldKeep func(URL string) bool) [][]str
// csvWriteBack writes records back to a given file.
func csvWriteBack(filename string, records [][]string) {
- filep := runtimex.Try1(os.Create(filename))
+ filep := runtimex.Try1(os.Create(filename)) // #nosec G304 - this is working as intended
writer := csv.NewWriter(filep)
runtimex.Try0(writer.WriteAll(records))
runtimex.Try0(writer.Error())
diff --git a/pkg/cmd/ghgen/utils.go b/pkg/cmd/ghgen/utils.go
index 441a77f4e..16336b25c 100644
--- a/pkg/cmd/ghgen/utils.go
+++ b/pkg/cmd/ghgen/utils.go
@@ -152,7 +152,7 @@ func mustClose(c io.Closer) {
func generateWorkflowFile(name string, jobs []Job) {
filename := filepath.Join(".github", "workflows", name+".yml")
- fp, err := os.Create(filename)
+ fp, err := os.Create(filename) // #nosec G304 - this is working as intended
runtimex.PanicOnError(err, "os.Create failed")
defer mustClose(fp)
mustFprintf(fp, "# File generated by `go run ./internal/cmd/ghgen`; DO NOT EDIT.\n")
@@ -171,7 +171,7 @@ func generateWorkflowFile(name string, jobs []Job) {
mustFprintf(fp, "\n")
mustFprintf(fp, "jobs:\n")
for _, job := range jobs {
- job.Action(fp, &job)
+ job.Action(fp, &job) // #nosec G601 -- job.Action is synchronous and does not retain job
}
mustFprintf(fp, "# End of autogenerated file\n")
}
diff --git a/pkg/cmd/miniooni/consent.go b/pkg/cmd/miniooni/consent.go
index 622d5e54d..aeac23520 100644
--- a/pkg/cmd/miniooni/consent.go
+++ b/pkg/cmd/miniooni/consent.go
@@ -27,7 +27,7 @@ func acquireUserConsent(miniooniDir string, currentOptions *Options) {
// maybeWriteConsentFile writes the consent file iff the yes argument is true
func maybeWriteConsentFile(yes bool, filepath string) (err error) {
if yes {
- err = os.WriteFile(filepath, []byte("\n"), 0644)
+ err = os.WriteFile(filepath, []byte("\n"), 0600)
}
return
}
diff --git a/pkg/cmd/miniooni/javascript.go b/pkg/cmd/miniooni/javascript.go
index 36f8a46ce..4ea2d63de 100644
--- a/pkg/cmd/miniooni/javascript.go
+++ b/pkg/cmd/miniooni/javascript.go
@@ -10,7 +10,7 @@ import (
)
// registerJavaScript registers the javascript subcommand
-func registerJavaScript(rootCmd *cobra.Command, globalOptions *Options) {
+func registerJavaScript(rootCmd *cobra.Command, _ *Options) {
subCmd := &cobra.Command{
Use: "javascript",
Short: "Very experimental command to run JavaScript snippets",
diff --git a/pkg/cmd/miniooni/main.go b/pkg/cmd/miniooni/main.go
index 9fa9d6428..626111802 100644
--- a/pkg/cmd/miniooni/main.go
+++ b/pkg/cmd/miniooni/main.go
@@ -40,6 +40,8 @@ type Options struct {
RepeatEvery int64
ReportFile string
SnowflakeRendezvous string
+ SoftwareName string
+ SoftwareVersion string
TorArgs []string
TorBinary string
Tunnel string
@@ -133,6 +135,20 @@ func main() {
"rendezvous method for --tunnel=torsf (one of: \"domain_fronting\" and \"amp\")",
)
+ flags.StringVar(
+ &globalOptions.SoftwareName,
+ "software-name",
+ "miniooni",
+ "Set the name of the application",
+ )
+
+ flags.StringVar(
+ &globalOptions.SoftwareVersion,
+ "software-version",
+ version.Version,
+ "Set the version of the application",
+ )
+
flags.StringSliceVar(
&globalOptions.TorArgs,
"tor-args",
@@ -268,7 +284,7 @@ func registerAllExperiments(rootCmd *cobra.Command, globalOptions *Options) {
// nothing
}
- if doc := documentationForOptions(name, factory); doc != "" {
+ if doc := documentationForOptions(factory); doc != "" {
flags.StringSliceVarP(
&globalOptions.ExtraOptions,
"option",
@@ -356,7 +372,7 @@ func mainSingleIteration(logger model.Logger, experimentName string, currentOpti
sess := newSessionOrPanic(ctx, currentOptions, miniooniDir, logger)
defer func() {
- sess.Close()
+ _ = sess.Close()
log.Infof("whole session: recv %s, sent %s",
humanize.SI(sess.KibiBytesReceived()*1024, "byte"),
humanize.SI(sess.KibiBytesSent()*1024, "byte"),
@@ -376,7 +392,7 @@ func mainSingleIteration(logger model.Logger, experimentName string, currentOpti
runx(ctx, sess, experimentName, annotations, extraOptions, currentOptions)
}
-func documentationForOptions(name string, factory *registry.Factory) string {
+func documentationForOptions(factory *registry.Factory) string {
var sb strings.Builder
options, err := factory.Options()
if err != nil || len(options) < 1 {
diff --git a/pkg/cmd/miniooni/main_test.go b/pkg/cmd/miniooni/main_test.go
index 8199c2fb2..3d146c71e 100644
--- a/pkg/cmd/miniooni/main_test.go
+++ b/pkg/cmd/miniooni/main_test.go
@@ -1,12 +1,18 @@
package main
-import "testing"
+import (
+ "testing"
+
+ "github.com/ooni/probe-engine/pkg/version"
+)
func TestSimple(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
MainWithConfiguration("example", &Options{
- Yes: true,
+ SoftwareName: "miniooni",
+ SoftwareVersion: version.Version,
+ Yes: true,
})
}
diff --git a/pkg/cmd/miniooni/oonirun.go b/pkg/cmd/miniooni/oonirun.go
index aad71a263..c1a4972e9 100644
--- a/pkg/cmd/miniooni/oonirun.go
+++ b/pkg/cmd/miniooni/oonirun.go
@@ -43,7 +43,7 @@ func ooniRunMain(ctx context.Context,
}
}
for _, filename := range currentOptions.InputFilePaths {
- data, err := os.ReadFile(filename)
+ data, err := os.ReadFile(filename) // #nosec G304 - this is working as intended
if err != nil {
logger.Warnf("oonirun: reading OONI Run v2 descriptor failed: %s", err.Error())
continue
diff --git a/pkg/cmd/miniooni/session.go b/pkg/cmd/miniooni/session.go
index 5abbaf681..211a01820 100644
--- a/pkg/cmd/miniooni/session.go
+++ b/pkg/cmd/miniooni/session.go
@@ -12,12 +12,6 @@ import (
"github.com/ooni/probe-engine/pkg/legacy/kvstore2dir"
"github.com/ooni/probe-engine/pkg/model"
"github.com/ooni/probe-engine/pkg/runtimex"
- "github.com/ooni/probe-engine/pkg/version"
-)
-
-const (
- softwareName = "miniooni"
- softwareVersion = version.Version
)
// newSessionOrPanic creates and starts a new session or panics on failure
@@ -44,8 +38,8 @@ func newSessionOrPanic(ctx context.Context, currentOptions *Options,
Logger: logger,
ProxyURL: proxyURL,
SnowflakeRendezvous: currentOptions.SnowflakeRendezvous,
- SoftwareName: softwareName,
- SoftwareVersion: softwareVersion,
+ SoftwareName: currentOptions.SoftwareName,
+ SoftwareVersion: currentOptions.SoftwareVersion,
TorArgs: currentOptions.TorArgs,
TorBinary: currentOptions.TorBinary,
TunnelDir: tunnelDir,
diff --git a/pkg/cmd/oohelper/internal/client.go b/pkg/cmd/oohelper/internal/client.go
index 48fb819f3..47682d08f 100644
--- a/pkg/cmd/oohelper/internal/client.go
+++ b/pkg/cmd/oohelper/internal/client.go
@@ -108,7 +108,8 @@ func (oo OOClient) Do(ctx context.Context, config OOConfig) (*CtrlResponse, erro
"Accept-Language": {model.HTTPHeaderAcceptLanguage},
"User-Agent": {model.HTTPHeaderUserAgent},
},
- TCPConnect: endpoints,
+ TCPConnect: endpoints,
+ XQUICEnabled: true,
}
data, err := json.Marshal(creq)
runtimex.PanicOnError(err, "oohelper: cannot marshal control request")
diff --git a/pkg/cmd/oohelperd/main.go b/pkg/cmd/oohelperd/main.go
index 0a171c2bd..223fe99d6 100644
--- a/pkg/cmd/oohelperd/main.go
+++ b/pkg/cmd/oohelperd/main.go
@@ -60,7 +60,7 @@ func shutdown(srv *http.Server, wg *sync.WaitGroup) {
defer wg.Done()
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
- srv.Shutdown(ctx)
+ _ = srv.Shutdown(ctx)
}
func main() {
@@ -100,12 +100,16 @@ func main() {
} else {
w.Header().Set("WWW-Authenticate", "Basic realm=metrics")
w.WriteHeader(401)
- w.Write([]byte("401 Unauthorized\n"))
+ _, _ = w.Write([]byte("401 Unauthorized\n"))
}
})
// create a listening server for serving ooniprobe requests
- srv := &http.Server{Addr: *apiEndpoint, Handler: mux}
+ srv := &http.Server{
+ Addr: *apiEndpoint,
+ Handler: mux,
+ ReadHeaderTimeout: 8 * time.Second,
+ }
listener, err := net.Listen("tcp", *apiEndpoint)
runtimex.PanicOnError(err, "net.Listen failed")
@@ -121,7 +125,11 @@ func main() {
pprofMux := http.NewServeMux()
pprofMux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
pprofMux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
- pprofSrv := &http.Server{Addr: *pprofEndpoint, Handler: pprofMux}
+ pprofSrv := &http.Server{
+ Addr: *pprofEndpoint,
+ Handler: pprofMux,
+ ReadHeaderTimeout: 8 * time.Second,
+ }
go pprofSrv.ListenAndServe()
log.Infof("serving CPU profile at http://%s/debug/pprof/profile", *pprofEndpoint)
log.Infof("serving execution traces at http://%s/debug/pprof/trace", *pprofEndpoint)
diff --git a/pkg/cmd/oonireport/oonireport.go b/pkg/cmd/oonireport/oonireport.go
index 10175a21f..971ad2487 100644
--- a/pkg/cmd/oonireport/oonireport.go
+++ b/pkg/cmd/oonireport/oonireport.go
@@ -15,7 +15,6 @@ import (
"github.com/ooni/probe-engine/pkg/engine"
"github.com/ooni/probe-engine/pkg/fsx"
"github.com/ooni/probe-engine/pkg/model"
- "github.com/ooni/probe-engine/pkg/probeservices"
"github.com/ooni/probe-engine/pkg/runtimex"
"github.com/ooni/probe-engine/pkg/version"
"github.com/pborman/getopt/v2"
@@ -43,8 +42,7 @@ const (
)
var (
- path string
- control bool
+ path string
)
func fatalIfFalse(cond bool, msg string) {
@@ -55,7 +53,7 @@ func fatalIfFalse(cond bool, msg string) {
func readLines(path string) []string {
// open measurement file
- file, err := os.Open(path)
+ file, err := os.Open(path) // #nosec G304 - this is working as intended
runtimex.PanicOnError(err, "Open file error.")
defer file.Close()
@@ -89,11 +87,8 @@ func newSession(ctx context.Context) *engine.Session {
}
// new Submitter creates a probe services client and submitter
-func newSubmitter(sess *engine.Session, ctx context.Context) *probeservices.Submitter {
- psc, err := sess.NewProbeServicesClient(ctx)
- runtimex.PanicOnError(err, "error occurred while creating client")
- submitter := probeservices.NewSubmitter(psc, sess.Logger())
- return submitter
+func newSubmitter(sess *engine.Session, ctx context.Context) model.Submitter {
+ return runtimex.Try1(sess.NewSubmitter(ctx))
}
// toMeasurement loads an input string as model.Measurement
@@ -106,7 +101,7 @@ func toMeasurement(s string) *model.Measurement {
// submitAll submits the measurements in input. Returns the count of submitted measurements, both
// on success and on error, and the error that occurred (nil on success).
-func submitAll(ctx context.Context, lines []string, subm *probeservices.Submitter) (int, error) {
+func submitAll(ctx context.Context, lines []string, subm model.Submitter) (int, error) {
submitted := 0
for _, line := range lines {
mm := toMeasurement(line)
diff --git a/pkg/cmd/ooporthelper/main.go b/pkg/cmd/ooporthelper/main.go
index 618d77158..cf7b0a0d4 100644
--- a/pkg/cmd/ooporthelper/main.go
+++ b/pkg/cmd/ooporthelper/main.go
@@ -27,7 +27,7 @@ func init() {
func shutdown(ctx context.Context, l net.Listener) {
<-ctx.Done()
- l.Close()
+ _ = l.Close()
}
// TODO(DecFox): Add the ability of an echo service to generate some traffic
diff --git a/pkg/cmd/tinyjafar/main.go b/pkg/cmd/tinyjafar/main.go
index c2086e54d..90ca3352f 100644
--- a/pkg/cmd/tinyjafar/main.go
+++ b/pkg/cmd/tinyjafar/main.go
@@ -153,7 +153,7 @@ func mainWithArgs(writer io.Writer, sigChan <-chan os.Signal, args ...string) {
fset := flag.NewFlagSet("tinyjafar", flag.ExitOnError)
cfg.initFlags(fset)
- fset.Parse(args)
+ runtimex.Try0(fset.Parse(args))
cs := newCmdSet()
cs.handleDropIP(cfg)
diff --git a/pkg/database/actions.go b/pkg/database/actions.go
index 9a4110efb..be53faef9 100644
--- a/pkg/database/actions.go
+++ b/pkg/database/actions.go
@@ -112,7 +112,7 @@ func (d *Database) GetMeasurementJSON(msmtID int64) (map[string]interface{}, err
return nil, errors.New("cannot access measurement file")
}
measurementFilePath := measurement.DatabaseMeasurement.MeasurementFilePath.String
- b, err := os.ReadFile(measurementFilePath)
+ b, err := os.ReadFile(measurementFilePath) // #nosec G304 - this is working as intended
if err != nil {
return nil, err
}
@@ -198,11 +198,10 @@ func (d *Database) DeleteResult(resultID int64) error {
return err
}
if err := res.Delete(); err != nil {
- log.WithError(err).Error("failed to delete the result directory")
+ log.WithError(err).Error("failed to delete the result")
return err
}
- os.RemoveAll(result.MeasurementDir)
- return nil
+ return os.RemoveAll(result.MeasurementDir)
}
// UpdateUploadedStatus implements WritableDatabase.UpdateUploadedStatus
@@ -337,7 +336,10 @@ func (d *Database) CreateOrUpdateURL(urlStr string, categoryCode string, country
return err
} else {
url.CategoryCode = sql.NullString{String: categoryCode, Valid: true}
- res.Update(url)
+ if err := res.Update(url); err != nil {
+ log.WithError(err).Error("Failed to update the database")
+ return err
+ }
}
return nil
diff --git a/pkg/engine/experiment.go b/pkg/engine/experiment.go
index b2cbe3af6..6560d9019 100644
--- a/pkg/engine/experiment.go
+++ b/pkg/engine/experiment.go
@@ -6,10 +6,8 @@ package engine
import (
"context"
- "encoding/json"
"errors"
"net/http"
- "os"
"runtime"
"time"
@@ -185,16 +183,6 @@ func (e *experiment) MeasureWithContext(
return
}
-// SaveMeasurement implements Experiment.SaveMeasurement.
-func (e *experiment) SaveMeasurement(measurement *model.Measurement, filePath string) error {
- return e.saveMeasurement(
- measurement, filePath, json.Marshal, os.OpenFile,
- func(fp *os.File, b []byte) (int, error) {
- return fp.Write(b)
- },
- )
-}
-
// SubmitAndUpdateMeasurementContext implements Experiment.SubmitAndUpdateMeasurementContext.
func (e *experiment) SubmitAndUpdateMeasurementContext(
ctx context.Context, measurement *model.Measurement) error {
@@ -250,7 +238,7 @@ func (e *experiment) OpenReportContext(ctx context.Context) error {
e.byteCounter,
),
}
- client, err := e.session.NewProbeServicesClient(ctx)
+ client, err := e.session.newProbeServicesClient(ctx)
if err != nil {
e.session.logger.Debugf("%+v", err)
return err
@@ -278,24 +266,3 @@ func (e *experiment) newReportTemplate() model.OOAPIReportTemplate {
TestVersion: e.testVersion,
}
}
-
-func (e *experiment) saveMeasurement(
- measurement *model.Measurement, filePath string,
- marshal func(v interface{}) ([]byte, error),
- openFile func(name string, flag int, perm os.FileMode) (*os.File, error),
- write func(fp *os.File, b []byte) (n int, err error),
-) error {
- data, err := marshal(measurement)
- if err != nil {
- return err
- }
- data = append(data, byte('\n'))
- filep, err := openFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
- if err != nil {
- return err
- }
- if _, err := write(filep, data); err != nil {
- return err
- }
- return filep.Close()
-}
diff --git a/pkg/engine/experiment_integration_test.go b/pkg/engine/experiment_integration_test.go
index 40fd94056..9bf34c571 100644
--- a/pkg/engine/experiment_integration_test.go
+++ b/pkg/engine/experiment_integration_test.go
@@ -3,15 +3,13 @@ package engine
import (
"context"
"encoding/json"
- "errors"
- "io/ioutil"
"net/http"
"net/http/httptest"
"os"
- "path/filepath"
"strings"
"testing"
+ "github.com/google/go-cmp/cmp"
"github.com/ooni/probe-engine/pkg/model"
"github.com/ooni/probe-engine/pkg/registry"
)
@@ -264,18 +262,24 @@ func runexperimentflow(t *testing.T, experiment model.Experiment, input string)
measurement.AddAnnotations(map[string]string{
"probe-engine-ci": "yes",
})
- data, err := json.Marshal(measurement)
+ savedMeasurement, err := json.Marshal(measurement)
if err != nil {
t.Fatal(err)
}
- if data == nil {
+ if savedMeasurement == nil {
t.Fatal("data is nil")
}
+ tempfile, err := os.CreateTemp("", "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ filename := tempfile.Name()
+ tempfile.Close()
err = experiment.SubmitAndUpdateMeasurementContext(ctx, measurement)
if err != nil {
t.Fatal(err)
}
- err = experiment.SaveMeasurement(measurement, "/tmp/experiment.jsonl")
+ err = SaveMeasurement(measurement, filename)
if err != nil {
t.Fatal(err)
}
@@ -289,54 +293,13 @@ func runexperimentflow(t *testing.T, experiment model.Experiment, input string)
if sk == nil {
t.Fatal("got nil summary keys")
}
-}
-
-func TestSaveMeasurementErrors(t *testing.T) {
- if testing.Short() {
- t.Skip("skip test in short mode")
- }
- sess := newSessionForTesting(t)
- defer sess.Close()
- builder, err := sess.NewExperimentBuilder("example")
+ loadedMeasurement, err := os.ReadFile(filename)
if err != nil {
t.Fatal(err)
}
- exp := builder.NewExperiment().(*experiment)
- dirname, err := ioutil.TempDir("", "ooniprobe-engine-save-measurement")
- if err != nil {
- t.Fatal(err)
- }
- filename := filepath.Join(dirname, "report.jsonl")
- m := new(model.Measurement)
- err = exp.saveMeasurement(
- m, filename, func(v interface{}) ([]byte, error) {
- return nil, errors.New("mocked error")
- }, os.OpenFile, func(fp *os.File, b []byte) (int, error) {
- return fp.Write(b)
- },
- )
- if err == nil {
- t.Fatal("expected an error here")
- }
- err = exp.saveMeasurement(
- m, filename, json.Marshal,
- func(name string, flag int, perm os.FileMode) (*os.File, error) {
- return nil, errors.New("mocked error")
- }, func(fp *os.File, b []byte) (int, error) {
- return fp.Write(b)
- },
- )
- if err == nil {
- t.Fatal("expected an error here")
- }
- err = exp.saveMeasurement(
- m, filename, json.Marshal, os.OpenFile,
- func(fp *os.File, b []byte) (int, error) {
- return 0, errors.New("mocked error")
- },
- )
- if err == nil {
- t.Fatal("expected an error here")
+ withFinalNewline := append(savedMeasurement, '\n')
+ if diff := cmp.Diff(withFinalNewline, loadedMeasurement); diff != "" {
+ t.Fatal(diff)
}
}
diff --git a/pkg/engine/inputloader.go b/pkg/engine/inputloader.go
index 23ee22124..271ed17ff 100644
--- a/pkg/engine/inputloader.go
+++ b/pkg/engine/inputloader.go
@@ -27,8 +27,7 @@ var (
// InputLoaderSession is the session according to an InputLoader. We
// introduce this abstraction because it helps us with testing.
type InputLoaderSession interface {
- CheckIn(ctx context.Context,
- config *model.OOAPICheckInConfig) (*model.OOAPICheckInResultNettests, error)
+ CheckIn(ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error)
}
// InputLoaderLogger is the logger according to an InputLoader.
@@ -146,7 +145,7 @@ func (il *InputLoader) loadOptional() ([]model.OOAPIURLInfo, error) {
}
// loadStrictlyRequired implements the InputStrictlyRequired policy.
-func (il *InputLoader) loadStrictlyRequired(ctx context.Context) ([]model.OOAPIURLInfo, error) {
+func (il *InputLoader) loadStrictlyRequired(_ context.Context) ([]model.OOAPIURLInfo, error) {
inputs, err := il.loadLocal()
if err != nil || len(inputs) > 0 {
return inputs, err
@@ -242,7 +241,7 @@ func staticInputForExperiment(name string) ([]model.OOAPIURLInfo, error) {
}
// loadOrStaticDefault implements the InputOrStaticDefault policy.
-func (il *InputLoader) loadOrStaticDefault(ctx context.Context) ([]model.OOAPIURLInfo, error) {
+func (il *InputLoader) loadOrStaticDefault(_ context.Context) ([]model.OOAPIURLInfo, error) {
inputs, err := il.loadLocal()
if err != nil || len(inputs) > 0 {
return inputs, err
@@ -328,12 +327,12 @@ func (il *InputLoader) checkIn(
return nil, err
}
// Note: safe to assume that reply is not nil if err is nil
- if reply.WebConnectivity != nil && len(reply.WebConnectivity.URLs) > 0 {
- reply.WebConnectivity.URLs = il.preventMistakes(
- reply.WebConnectivity.URLs, config.WebConnectivity.CategoryCodes,
+ if reply.Tests.WebConnectivity != nil && len(reply.Tests.WebConnectivity.URLs) > 0 {
+ reply.Tests.WebConnectivity.URLs = il.preventMistakes(
+ reply.Tests.WebConnectivity.URLs, config.WebConnectivity.CategoryCodes,
)
}
- return reply, nil
+ return &reply.Tests, nil
}
// preventMistakes makes the code more robust with respect to any possible
diff --git a/pkg/engine/inputloader_test.go b/pkg/engine/inputloader_test.go
index 264eb345f..2811b72a1 100644
--- a/pkg/engine/inputloader_test.go
+++ b/pkg/engine/inputloader_test.go
@@ -446,7 +446,7 @@ func TestInputLoaderReadfileScannerFailure(t *testing.T) {
type InputLoaderMockableSession struct {
// Output contains the output of CheckIn. It should
// be nil when Error is not-nil.
- Output *model.OOAPICheckInResultNettests
+ Output *model.OOAPICheckInResult
// Error is the error to be returned by CheckIn. It
// should be nil when Output is not-nil.
@@ -455,7 +455,7 @@ type InputLoaderMockableSession struct {
// CheckIn implements InputLoaderSession.CheckIn.
func (sess *InputLoaderMockableSession) CheckIn(
- ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResultNettests, error) {
+ ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) {
if sess.Output == nil && sess.Error == nil {
return nil, errors.New("both Output and Error are nil")
}
@@ -480,7 +480,9 @@ func TestInputLoaderCheckInFailure(t *testing.T) {
func TestInputLoaderCheckInSuccessWithNilWebConnectivity(t *testing.T) {
il := &InputLoader{
Session: &InputLoaderMockableSession{
- Output: &model.OOAPICheckInResultNettests{},
+ Output: &model.OOAPICheckInResult{
+ Tests: model.OOAPICheckInResultNettests{},
+ },
},
}
out, err := il.loadRemote(context.Background())
@@ -495,8 +497,10 @@ func TestInputLoaderCheckInSuccessWithNilWebConnectivity(t *testing.T) {
func TestInputLoaderCheckInSuccessWithNoURLs(t *testing.T) {
il := &InputLoader{
Session: &InputLoaderMockableSession{
- Output: &model.OOAPICheckInResultNettests{
- WebConnectivity: &model.OOAPICheckInInfoWebConnectivity{},
+ Output: &model.OOAPICheckInResult{
+ Tests: model.OOAPICheckInResultNettests{
+ WebConnectivity: &model.OOAPICheckInInfoWebConnectivity{},
+ },
},
},
}
@@ -521,9 +525,11 @@ func TestInputLoaderCheckInSuccessWithSomeURLs(t *testing.T) {
}}
il := &InputLoader{
Session: &InputLoaderMockableSession{
- Output: &model.OOAPICheckInResultNettests{
- WebConnectivity: &model.OOAPICheckInInfoWebConnectivity{
- URLs: expect,
+ Output: &model.OOAPICheckInResult{
+ Tests: model.OOAPICheckInResultNettests{
+ WebConnectivity: &model.OOAPICheckInInfoWebConnectivity{
+ URLs: expect,
+ },
},
},
},
diff --git a/pkg/engine/savemeasurement.go b/pkg/engine/savemeasurement.go
new file mode 100644
index 000000000..a9729cde8
--- /dev/null
+++ b/pkg/engine/savemeasurement.go
@@ -0,0 +1,39 @@
+package engine
+
+import (
+ "encoding/json"
+ "os"
+
+ "github.com/ooni/probe-engine/pkg/model"
+)
+
+// SaveMeasurement saves a measurement on the specified file path.
+func SaveMeasurement(measurement *model.Measurement, filePath string) error {
+ return saveMeasurement(
+ measurement, filePath, json.Marshal, os.OpenFile,
+ func(fp *os.File, b []byte) (int, error) {
+ return fp.Write(b)
+ },
+ )
+}
+
+func saveMeasurement(
+ measurement *model.Measurement, filePath string,
+ marshal func(v interface{}) ([]byte, error),
+ openFile func(name string, flag int, perm os.FileMode) (*os.File, error),
+ write func(fp *os.File, b []byte) (n int, err error),
+) error {
+ data, err := marshal(measurement)
+ if err != nil {
+ return err
+ }
+ data = append(data, byte('\n'))
+ filep, err := openFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
+ if err != nil {
+ return err
+ }
+ if _, err := write(filep, data); err != nil {
+ return err
+ }
+ return filep.Close()
+}
diff --git a/pkg/engine/savemeasurement_test.go b/pkg/engine/savemeasurement_test.go
new file mode 100644
index 000000000..0da00294b
--- /dev/null
+++ b/pkg/engine/savemeasurement_test.go
@@ -0,0 +1,85 @@
+package engine
+
+import (
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/must"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/testingx"
+)
+
+func TestSaveMeasurementSuccess(t *testing.T) {
+ // get temporary file where to write the measurement
+ filep, err := os.CreateTemp("", "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ filename := filep.Name()
+ filep.Close()
+
+ // create and fake-fill the measurement
+ m := &model.Measurement{}
+ ff := &testingx.FakeFiller{}
+ ff.Fill(m)
+
+ // write the measurement to disk
+ if err := SaveMeasurement(m, filename); err != nil {
+ t.Fatal(err)
+ }
+
+ // marshal the measurement to JSON with extra \n at the end
+ expect := append(must.MarshalJSON(m), '\n')
+
+ // read the measurement from file
+ got := runtimex.Try1(os.ReadFile(filename))
+
+ // make sure what we read matches what we expect
+ if diff := cmp.Diff(expect, got); diff != "" {
+ t.Fatal(diff)
+ }
+}
+
+func TestSaveMeasurementErrors(t *testing.T) {
+ dirname, err := os.MkdirTemp("", "ooniprobe-engine-save-measurement")
+ if err != nil {
+ t.Fatal(err)
+ }
+ filename := filepath.Join(dirname, "report.jsonl")
+ m := new(model.Measurement)
+ err = saveMeasurement(
+ m, filename, func(v interface{}) ([]byte, error) {
+ return nil, errors.New("mocked error")
+ }, os.OpenFile, func(fp *os.File, b []byte) (int, error) {
+ return fp.Write(b)
+ },
+ )
+ if err == nil {
+ t.Fatal("expected an error here")
+ }
+ err = saveMeasurement(
+ m, filename, json.Marshal,
+ func(name string, flag int, perm os.FileMode) (*os.File, error) {
+ return nil, errors.New("mocked error")
+ }, func(fp *os.File, b []byte) (int, error) {
+ return fp.Write(b)
+ },
+ )
+ if err == nil {
+ t.Fatal("expected an error here")
+ }
+ err = saveMeasurement(
+ m, filename, json.Marshal, os.OpenFile,
+ func(fp *os.File, b []byte) (int, error) {
+ return 0, errors.New("mocked error")
+ },
+ )
+ if err == nil {
+ t.Fatal("expected an error here")
+ }
+}
diff --git a/pkg/engine/session.go b/pkg/engine/session.go
index c034d54e0..1f4c14662 100644
--- a/pkg/engine/session.go
+++ b/pkg/engine/session.go
@@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
- "io/ioutil"
"net/url"
"os"
"sync"
@@ -155,7 +154,7 @@ func NewSession(ctx context.Context, config SessionConfig) (*Session, error) {
// use the temporary directory on the current system. This should
// work on Desktop. We tested that it did also work on iOS, but
// we have also seen on 2020-06-10 that it does not work on Android.
- tempDir, err := ioutil.TempDir(config.TempDir, "ooniengine")
+ tempDir, err := os.MkdirTemp(config.TempDir, "ooniengine")
if err != nil {
return nil, err
}
@@ -265,9 +264,7 @@ func (s *Session) KibiBytesSent() float64 {
//
// The return value is either the check-in response or an error.
func (s *Session) CheckIn(
- ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResultNettests, error) {
- // TODO(bassosimone): consider refactoring this function to return
- // the whole check-in response to the caller.
+ ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) {
if err := s.maybeLookupLocationContext(ctx); err != nil {
return nil, err
}
@@ -300,7 +297,7 @@ func (s *Session) CheckIn(
if err != nil {
return nil, err
}
- return &resp.Tests, nil
+ return resp, nil
}
// maybeLookupLocationContext is a wrapper for MaybeLookupLocationContext that calls
@@ -321,7 +318,7 @@ func (s *Session) newProbeServicesClientForCheckIn(
if s.testNewProbeServicesClientForCheckIn != nil {
return s.testNewProbeServicesClientForCheckIn(ctx)
}
- client, err := s.NewProbeServicesClient(ctx)
+ client, err := s.newProbeServicesClient(ctx)
if err != nil {
return nil, err
}
@@ -341,7 +338,7 @@ func (s *Session) Close() error {
// doClose implements Close. This function is called just once.
func (s *Session) doClose() {
// make sure we close open connections and persist stats to the key-value store
- s.network.Close()
+ _ = s.network.Close()
s.resolver.CloseIdleConnections()
if s.tunnel != nil {
@@ -367,7 +364,7 @@ func (s *Session) DefaultHTTPClient() model.HTTPClient {
// FetchTorTargets fetches tor targets from the API.
func (s *Session) FetchTorTargets(
ctx context.Context, cc string) (map[string]model.OOAPITorTarget, error) {
- clnt, err := s.NewOrchestraClient(ctx)
+ clnt, err := s.newOrchestraClient(ctx)
if err != nil {
return nil, err
}
@@ -384,16 +381,6 @@ func (s *Session) Logger() model.Logger {
return s.logger
}
-// MaybeLookupLocation is a caching location lookup call.
-func (s *Session) MaybeLookupLocation() error {
- return s.MaybeLookupLocationContext(context.Background())
-}
-
-// MaybeLookupBackends is a caching OONI backends lookup call.
-func (s *Session) MaybeLookupBackends() error {
- return s.MaybeLookupBackendsContext(context.Background())
-}
-
// ErrAlreadyUsingProxy indicates that we cannot create a tunnel with
// a specific name because we already configured a proxy.
var ErrAlreadyUsingProxy = errors.New(
@@ -411,12 +398,12 @@ func (s *Session) NewExperimentBuilder(name string) (model.ExperimentBuilder, er
return eb, nil
}
-// NewProbeServicesClient creates a new client for talking with the
+// newProbeServicesClient creates a new client for talking with the
// OONI probe services. This function will benchmark the available
// probe services, and select the fastest. In case all probe services
// seem to be down, we try again applying circumvention tactics.
// This function will fail IMMEDIATELY if given a cancelled context.
-func (s *Session) NewProbeServicesClient(ctx context.Context) (*probeservices.Client, error) {
+func (s *Session) newProbeServicesClient(ctx context.Context) (*probeservices.Client, error) {
if ctx.Err() != nil {
return nil, ctx.Err() // helps with testing
}
@@ -433,21 +420,18 @@ func (s *Session) NewProbeServicesClient(ctx context.Context) (*probeservices.Cl
}
// NewSubmitter creates a new submitter instance.
-func (s *Session) NewSubmitter(ctx context.Context) (Submitter, error) {
- psc, err := s.NewProbeServicesClient(ctx)
+func (s *Session) NewSubmitter(ctx context.Context) (model.Submitter, error) {
+ psc, err := s.newProbeServicesClient(ctx)
if err != nil {
return nil, err
}
return probeservices.NewSubmitter(psc, s.Logger()), nil
}
-// NewOrchestraClient creates a new orchestra client. This client is registered
+// newOrchestraClient creates a new orchestra client. This client is registered
// and logged in with the OONI orchestra. An error is returned on failure.
-//
-// This function is DEPRECATED. New code SHOULD NOT use it. It will eventually
-// be made private or entirely removed from the codebase.
-func (s *Session) NewOrchestraClient(ctx context.Context) (*probeservices.Client, error) {
- clnt, err := s.NewProbeServicesClient(ctx)
+func (s *Session) newOrchestraClient(ctx context.Context) (*probeservices.Client, error) {
+ clnt, err := s.newProbeServicesClient(ctx)
if err != nil {
return nil, err
}
@@ -665,9 +649,9 @@ func (s *Session) MaybeLookupBackendsContext(ctx context.Context) error {
return nil
}
-// LookupLocationContext performs a location lookup. If you want memoisation
+// doLookupLocationContext performs a location lookup. If you want memoisation
// of the results, you should use MaybeLookupLocationContext.
-func (s *Session) LookupLocationContext(ctx context.Context) (*enginelocate.Results, error) {
+func (s *Session) doLookupLocationContext(ctx context.Context) (*enginelocate.Results, error) {
task := enginelocate.NewTask(enginelocate.Config{
Logger: s.Logger(),
Resolver: s.resolver,
@@ -682,7 +666,7 @@ func (s *Session) lookupLocationContext(ctx context.Context) (*enginelocate.Resu
if s.testLookupLocationContext != nil {
return s.testLookupLocationContext(ctx)
}
- return s.LookupLocationContext(ctx)
+ return s.doLookupLocationContext(ctx)
}
// MaybeLookupLocationContext is like MaybeLookupLocation but with a context
diff --git a/pkg/engine/session_integration_test.go b/pkg/engine/session_integration_test.go
index 642b25acd..c25d513bf 100644
--- a/pkg/engine/session_integration_test.go
+++ b/pkg/engine/session_integration_test.go
@@ -146,7 +146,7 @@ func newSessionForTestingNoLookups(t *testing.T) *Session {
func newSessionForTestingNoBackendsLookup(t *testing.T) *Session {
sess := newSessionForTestingNoLookups(t)
- if err := sess.MaybeLookupLocation(); err != nil {
+ if err := sess.MaybeLookupLocationContext(context.Background()); err != nil {
t.Fatal(err)
}
log.Infof("Platform: %s", sess.Platform())
@@ -164,7 +164,7 @@ func newSessionForTestingNoBackendsLookup(t *testing.T) *Session {
func newSessionForTesting(t *testing.T) *Session {
sess := newSessionForTestingNoBackendsLookup(t)
- if err := sess.MaybeLookupBackends(); err != nil {
+ if err := sess.MaybeLookupBackendsContext(context.Background()); err != nil {
t.Fatal(err)
}
return sess
@@ -245,7 +245,7 @@ func TestBouncerError(t *testing.T) {
if sess.ProxyURL() == nil {
t.Fatal("expected to see explicit proxy here")
}
- if err := sess.MaybeLookupBackends(); err == nil {
+ if err := sess.MaybeLookupBackendsContext(context.Background()); err == nil {
t.Fatal("expected an error here")
}
}
@@ -260,7 +260,7 @@ func TestMaybeLookupBackendsNewClientError(t *testing.T) {
Address: "httpo://jehhrikjjqrlpufu.onion",
}}
defer sess.Close()
- err := sess.MaybeLookupBackends()
+ err := sess.MaybeLookupBackendsContext(context.Background())
if !errors.Is(err, ErrAllProbeServicesFailed) {
t.Fatal("not the error we expected")
}
@@ -272,7 +272,7 @@ func TestSessionLocationLookup(t *testing.T) {
}
sess := newSessionForTestingNoLookups(t)
defer sess.Close()
- if err := sess.MaybeLookupLocation(); err != nil {
+ if err := sess.MaybeLookupLocationContext(context.Background()); err != nil {
t.Fatal(err)
}
if sess.ProbeASNString() == model.DefaultProbeASNString {
@@ -419,7 +419,7 @@ func TestAllProbeServicesUnsupported(t *testing.T) {
Address: "mascetti",
Type: "antani",
})
- err = sess.MaybeLookupBackends()
+ err = sess.MaybeLookupBackendsContext(context.Background())
if !errors.Is(err, ErrAllProbeServicesFailed) {
t.Fatal("unexpected error")
}
@@ -447,7 +447,7 @@ func TestNewOrchestraClientMaybeLookupBackendsFailure(t *testing.T) {
sess.testMaybeLookupBackendsContext = func(ctx context.Context) error {
return errMocked
}
- client, err := sess.NewOrchestraClient(context.Background())
+ client, err := sess.newOrchestraClient(context.Background())
if !errors.Is(err, errMocked) {
t.Fatal("not the error we expected", err)
}
@@ -465,7 +465,7 @@ func TestNewOrchestraClientMaybeLookupLocationFailure(t *testing.T) {
sess.testMaybeLookupLocationContext = func(ctx context.Context) error {
return errMocked
}
- client, err := sess.NewOrchestraClient(context.Background())
+ client, err := sess.newOrchestraClient(context.Background())
if !errors.Is(err, errMocked) {
t.Fatalf("not the error we expected: %+v", err)
}
@@ -482,7 +482,7 @@ func TestNewOrchestraClientProbeServicesNewClientFailure(t *testing.T) {
sess.selectedProbeServiceHook = func(svc *model.OOAPIService) {
svc.Type = "antani" // should really not be supported for a long time
}
- client, err := sess.NewOrchestraClient(context.Background())
+ client, err := sess.newOrchestraClient(context.Background())
if !errors.Is(err, probeservices.ErrUnsupportedEndpoint) {
t.Fatal("not the error we expected")
}
diff --git a/pkg/engine/session_internal_test.go b/pkg/engine/session_internal_test.go
index 9241f1e68..a2ea161ca 100644
--- a/pkg/engine/session_internal_test.go
+++ b/pkg/engine/session_internal_test.go
@@ -65,24 +65,24 @@ func (c *mockableProbeServicesClientForCheckIn) CheckIn(
}
func TestSessionCheckInSuccessful(t *testing.T) {
- results := &model.OOAPICheckInResultNettests{
- WebConnectivity: &model.OOAPICheckInInfoWebConnectivity{
- ReportID: "xxx-x-xx",
- URLs: []model.OOAPIURLInfo{{
- CategoryCode: "NEWS",
- CountryCode: "IT",
- URL: "https://www.repubblica.it/",
- }, {
- CategoryCode: "NEWS",
- CountryCode: "IT",
- URL: "https://www.unita.it/",
- }},
+ results := &model.OOAPICheckInResult{
+ Tests: model.OOAPICheckInResultNettests{
+ WebConnectivity: &model.OOAPICheckInInfoWebConnectivity{
+ ReportID: "xxx-x-xx",
+ URLs: []model.OOAPIURLInfo{{
+ CategoryCode: "NEWS",
+ CountryCode: "IT",
+ URL: "https://www.repubblica.it/",
+ }, {
+ CategoryCode: "NEWS",
+ CountryCode: "IT",
+ URL: "https://www.unita.it/",
+ }},
+ },
},
}
mockedClnt := &mockableProbeServicesClientForCheckIn{
- Results: &model.OOAPICheckInResult{
- Tests: *results,
- },
+ Results: results,
}
s := &Session{
location: &enginelocate.Results{
diff --git a/pkg/engine/session_nopsiphon.go b/pkg/engine/session_nopsiphon.go
index cc03326d5..472aeb470 100644
--- a/pkg/engine/session_nopsiphon.go
+++ b/pkg/engine/session_nopsiphon.go
@@ -9,7 +9,7 @@ import (
// FetchPsiphonConfig fetches psiphon config from the API.
func (s *Session) FetchPsiphonConfig(ctx context.Context) ([]byte, error) {
- clnt, err := s.NewOrchestraClient(ctx)
+ clnt, err := s.newOrchestraClient(ctx)
if err != nil {
return nil, err
}
diff --git a/pkg/enginelocate/cloudflare.go b/pkg/enginelocate/cloudflare.go
index 8cf5fa957..3857123bf 100644
--- a/pkg/enginelocate/cloudflare.go
+++ b/pkg/enginelocate/cloudflare.go
@@ -2,32 +2,46 @@ package enginelocate
import (
"context"
- "net/http"
+ "net"
"regexp"
"strings"
- "github.com/ooni/probe-engine/pkg/httpx"
+ "github.com/ooni/probe-engine/pkg/httpclientx"
"github.com/ooni/probe-engine/pkg/model"
)
func cloudflareIPLookup(
ctx context.Context,
- httpClient *http.Client,
+ httpClient model.HTTPClient,
logger model.Logger,
userAgent string,
resolver model.Resolver,
) (string, error) {
- data, err := (&httpx.APIClientTemplate{
- BaseURL: "https://www.cloudflare.com",
- HTTPClient: httpClient,
- Logger: logger,
- UserAgent: model.HTTPHeaderUserAgent,
- }).WithBodyLogging().Build().FetchResource(ctx, "/cdn-cgi/trace")
+ // get the raw response body
+ data, err := httpclientx.GetRaw(
+ ctx,
+ httpclientx.NewEndpoint("https://www.cloudflare.com/cdn-cgi/trace"),
+ &httpclientx.Config{
+ Authorization: "", // not needed
+ Client: httpClient,
+ Logger: logger,
+ UserAgent: userAgent,
+ })
+
+ // handle the error case
if err != nil {
return model.DefaultProbeIP, err
}
+
+ // find the IP addr
r := regexp.MustCompile("(?:ip)=(.*)")
ip := strings.Trim(string(r.Find(data)), "ip=")
- logger.Debugf("cloudflare: body: %s", ip)
+
+ // make sure the IP addr is valid
+ if net.ParseIP(ip) == nil {
+ return model.DefaultProbeIP, ErrInvalidIPAddress
+ }
+
+ // done!
return ip, nil
}
diff --git a/pkg/enginelocate/cloudflare_test.go b/pkg/enginelocate/cloudflare_test.go
index 98d1007ed..8303c487f 100644
--- a/pkg/enginelocate/cloudflare_test.go
+++ b/pkg/enginelocate/cloudflare_test.go
@@ -2,32 +2,234 @@ package enginelocate
import (
"context"
+ "errors"
"net"
"net/http"
+ "net/url"
"testing"
"github.com/apex/log"
+ "github.com/ooni/probe-engine/pkg/mocks"
"github.com/ooni/probe-engine/pkg/model"
"github.com/ooni/probe-engine/pkg/netxlite"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/testingx"
)
+// cloudflareRealisticresponse is a realistic response returned by cloudflare
+// with the IP address modified to belong to a public institution.
+var cloudflareRealisticResponse = []byte(`
+fl=270f47
+h=www.cloudflare.com
+ip=130.192.91.211
+ts=1713946961.154
+visit_scheme=https
+uag=Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:125.0) Gecko/20100101 Firefox/125.0
+colo=MXP
+sliver=none
+http=http/3
+loc=IT
+tls=TLSv1.3
+sni=plaintext
+warp=off
+gateway=off
+rbi=off
+kex=X25519
+`)
+
func TestIPLookupWorksUsingcloudlflare(t *testing.T) {
- if testing.Short() {
- t.Skip("skip test in short mode")
- }
-
- netx := &netxlite.Netx{}
- ip, err := cloudflareIPLookup(
- context.Background(),
- http.DefaultClient,
- log.Log,
- model.HTTPHeaderUserAgent,
- netx.NewStdlibResolver(model.DiscardLogger),
- )
- if err != nil {
- t.Fatal(err)
- }
- if net.ParseIP(ip) == nil {
- t.Fatalf("not an IP address: '%s'", ip)
- }
+
+ // We want to make sure the real server gives us an IP address.
+ t.Run("is working as intended when using the real server", func(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skip test in short mode")
+ }
+
+ // figure out the IP address using cloudflare
+ netx := &netxlite.Netx{}
+ ip, err := cloudflareIPLookup(
+ context.Background(),
+ http.DefaultClient,
+ log.Log,
+ model.HTTPHeaderUserAgent,
+ netx.NewStdlibResolver(model.DiscardLogger),
+ )
+
+ // we expect this call to succeed
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // we expect to get back a valid IPv4/IPv6 address
+ if net.ParseIP(ip) == nil {
+ t.Fatalf("not an IP address: '%s'", ip)
+ }
+ })
+
+ // But we also want to make sure everything is working as intended when using
+ // a local HTTP server, as well as that we can handle errors, so that we can run
+ // tests in short mode. This is done with the tests below.
+
+ t.Run("is working as intended when using a fake server", func(t *testing.T) {
+ // create a fake server returning an hardcoded IP address.
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write(cloudflareRealisticResponse)
+ }))
+ defer srv.Close()
+
+ // create an HTTP client that uses the fake server.
+ client := &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ // rewrite the request URL to be the one of the fake server
+ req.URL = runtimex.Try1(url.Parse(srv.URL))
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // figure out the IP address using cloudflare
+ netx := &netxlite.Netx{}
+ ip, err := cloudflareIPLookup(
+ context.Background(),
+ client,
+ log.Log,
+ model.HTTPHeaderUserAgent,
+ netx.NewStdlibResolver(model.DiscardLogger),
+ )
+
+ // we expect this call to succeed
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // we expect to get back a valid IPv4/IPv6 address
+ if net.ParseIP(ip) == nil {
+ t.Fatalf("not an IP address: '%s'", ip)
+ }
+
+ // we expect to see exactly the IP address that we want to see
+ if ip != "130.192.91.211" {
+ t.Fatal("unexpected IP address", ip)
+ }
+ })
+
+ t.Run("correctly handles network errors", func(t *testing.T) {
+ // create a fake server resetting the connection for the client.
+ srv := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer srv.Close()
+
+ // create an HTTP client that uses the fake server.
+ client := &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ // rewrite the request URL to be the one of the fake server
+ req.URL = runtimex.Try1(url.Parse(srv.URL))
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // figure out the IP address using cloudflare
+ netx := &netxlite.Netx{}
+ ip, err := cloudflareIPLookup(
+ context.Background(),
+ client,
+ log.Log,
+ model.HTTPHeaderUserAgent,
+ netx.NewStdlibResolver(model.DiscardLogger),
+ )
+
+ // we expect to see ECONNRESET here
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // the returned IP address should be the default one
+ if ip != model.DefaultProbeIP {
+ t.Fatal("unexpected IP address", ip)
+ }
+ })
+
+ t.Run("correctly handles parsing errors", func(t *testing.T) {
+ // create a fake server returnning different keys
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`ipx=130.192.91.211`)) // note: different key name
+ }))
+ defer srv.Close()
+
+ // create an HTTP client that uses the fake server.
+ client := &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ // rewrite the request URL to be the one of the fake server
+ req.URL = runtimex.Try1(url.Parse(srv.URL))
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // figure out the IP address using cloudflare
+ netx := &netxlite.Netx{}
+ ip, err := cloudflareIPLookup(
+ context.Background(),
+ client,
+ log.Log,
+ model.HTTPHeaderUserAgent,
+ netx.NewStdlibResolver(model.DiscardLogger),
+ )
+
+ // we expect to see an error indicating there's no IP address in the response
+ if !errors.Is(err, ErrInvalidIPAddress) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // the returned IP address should be the default one
+ if ip != model.DefaultProbeIP {
+ t.Fatal("unexpected IP address", ip)
+ }
+ })
+
+ t.Run("correctly handles the case where the IP address is invalid", func(t *testing.T) {
+ // create a fake server returnning different keys
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`ip=foobarbaz`)) // note: invalid IP address
+ }))
+ defer srv.Close()
+
+ // create an HTTP client that uses the fake server.
+ client := &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ // rewrite the request URL to be the one of the fake server
+ req.URL = runtimex.Try1(url.Parse(srv.URL))
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // figure out the IP address using cloudflare
+ netx := &netxlite.Netx{}
+ ip, err := cloudflareIPLookup(
+ context.Background(),
+ client,
+ log.Log,
+ model.HTTPHeaderUserAgent,
+ netx.NewStdlibResolver(model.DiscardLogger),
+ )
+
+ // we expect to see an error indicating there's no IP address in the response
+ if !errors.Is(err, ErrInvalidIPAddress) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // the returned IP address should be the default one
+ if ip != model.DefaultProbeIP {
+ t.Fatal("unexpected IP address", ip)
+ }
+ })
}
diff --git a/pkg/enginelocate/fake_test.go b/pkg/enginelocate/fake_test.go
deleted file mode 100644
index a15382010..000000000
--- a/pkg/enginelocate/fake_test.go
+++ /dev/null
@@ -1,28 +0,0 @@
-package enginelocate
-
-import (
- "net/http"
- "time"
-
- "github.com/ooni/probe-engine/pkg/netxlite"
-)
-
-type FakeTransport struct {
- Err error
- Resp *http.Response
-}
-
-func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
- time.Sleep(10 * time.Microsecond)
- if req.Body != nil {
- netxlite.ReadAllContext(req.Context(), req.Body)
- req.Body.Close()
- }
- if txp.Err != nil {
- return nil, txp.Err
- }
- txp.Resp.Request = req // non thread safe but it doesn't matter
- return txp.Resp, nil
-}
-
-func (txp FakeTransport) CloseIdleConnections() {}
diff --git a/pkg/enginelocate/invalid_test.go b/pkg/enginelocate/invalid_test.go
index 241a28d5a..0bec574b4 100644
--- a/pkg/enginelocate/invalid_test.go
+++ b/pkg/enginelocate/invalid_test.go
@@ -2,14 +2,13 @@ package enginelocate
import (
"context"
- "net/http"
"github.com/ooni/probe-engine/pkg/model"
)
func invalidIPLookup(
ctx context.Context,
- httpClient *http.Client,
+ httpClient model.HTTPClient,
logger model.Logger,
userAgent string,
resolver model.Resolver,
diff --git a/pkg/enginelocate/iplookup.go b/pkg/enginelocate/iplookup.go
index d5564d3e0..9103195a0 100644
--- a/pkg/enginelocate/iplookup.go
+++ b/pkg/enginelocate/iplookup.go
@@ -9,8 +9,8 @@ import (
"net/http"
"time"
+ "github.com/ooni/probe-engine/pkg/legacy/multierror"
"github.com/ooni/probe-engine/pkg/model"
- "github.com/ooni/probe-engine/pkg/multierror"
"github.com/ooni/probe-engine/pkg/netxlite"
)
@@ -25,7 +25,7 @@ var (
)
type lookupFunc func(
- ctx context.Context, client *http.Client,
+ ctx context.Context, client model.HTTPClient,
logger model.Logger, userAgent string,
resolver model.Resolver,
) (string, error)
@@ -68,7 +68,7 @@ type ipLookupClient struct {
}
func makeSlice() []method {
- r := rand.New(rand.NewSource(time.Now().UnixNano()))
+ r := rand.New(rand.NewSource(time.Now().UnixNano())) // #nosec G404 -- not really important
ret := make([]method, len(methods))
perm := r.Perm(len(methods))
for idx, randIdx := range perm {
diff --git a/pkg/enginelocate/stun.go b/pkg/enginelocate/stun.go
index 5005af451..0b43298c1 100644
--- a/pkg/enginelocate/stun.go
+++ b/pkg/enginelocate/stun.go
@@ -3,7 +3,6 @@ package enginelocate
import (
"context"
"net"
- "net/http"
"github.com/ooni/probe-engine/pkg/model"
"github.com/ooni/probe-engine/pkg/netxlite"
@@ -46,7 +45,7 @@ func stunIPLookup(ctx context.Context, config stunConfig) (string, error) {
}
clnt, err := newClient(conn)
if err != nil {
- conn.Close()
+ _ = conn.Close()
return model.DefaultProbeIP, err
}
defer clnt.Close()
@@ -86,7 +85,7 @@ func stunIPLookup(ctx context.Context, config stunConfig) (string, error) {
func stunEkigaIPLookup(
ctx context.Context,
- httpClient *http.Client,
+ httpClient model.HTTPClient,
logger model.Logger,
userAgent string,
resolver model.Resolver,
@@ -100,7 +99,7 @@ func stunEkigaIPLookup(
func stunGoogleIPLookup(
ctx context.Context,
- httpClient *http.Client,
+ httpClient model.HTTPClient,
logger model.Logger,
userAgent string,
resolver model.Resolver,
diff --git a/pkg/enginelocate/ubuntu.go b/pkg/enginelocate/ubuntu.go
index 2071932a3..8fc5f3610 100644
--- a/pkg/enginelocate/ubuntu.go
+++ b/pkg/enginelocate/ubuntu.go
@@ -3,9 +3,9 @@ package enginelocate
import (
"context"
"encoding/xml"
- "net/http"
+ "net"
- "github.com/ooni/probe-engine/pkg/httpx"
+ "github.com/ooni/probe-engine/pkg/httpclientx"
"github.com/ooni/probe-engine/pkg/model"
)
@@ -16,25 +16,32 @@ type ubuntuResponse struct {
func ubuntuIPLookup(
ctx context.Context,
- httpClient *http.Client,
+ httpClient model.HTTPClient,
logger model.Logger,
userAgent string,
resolver model.Resolver,
) (string, error) {
- data, err := (&httpx.APIClientTemplate{
- BaseURL: "https://geoip.ubuntu.com/",
- HTTPClient: httpClient,
- Logger: logger,
- UserAgent: userAgent,
- }).WithBodyLogging().Build().FetchResource(ctx, "/lookup")
+ // read the HTTP response and parse as XML
+ v, err := httpclientx.GetXML[*ubuntuResponse](
+ ctx,
+ httpclientx.NewEndpoint("https://geoip.ubuntu.com/lookup"),
+ &httpclientx.Config{
+ Authorization: "", // not needed
+ Client: httpClient,
+ Logger: logger,
+ UserAgent: userAgent,
+ })
+
+ // handle the error case
if err != nil {
return model.DefaultProbeIP, err
}
- logger.Debugf("ubuntu: body: %s", string(data))
- var v ubuntuResponse
- err = xml.Unmarshal(data, &v)
- if err != nil {
- return model.DefaultProbeIP, err
+
+ // make sure the IP addr is valid
+ if net.ParseIP(v.IP) == nil {
+ return model.DefaultProbeIP, ErrInvalidIPAddress
}
+
+ // handle the success case
return v.IP, nil
}
diff --git a/pkg/enginelocate/ubuntu_test.go b/pkg/enginelocate/ubuntu_test.go
index c59039521..08ef5c619 100644
--- a/pkg/enginelocate/ubuntu_test.go
+++ b/pkg/enginelocate/ubuntu_test.go
@@ -2,56 +2,268 @@ package enginelocate
import (
"context"
- "io"
+ "errors"
"net"
"net/http"
+ "net/url"
"strings"
"testing"
"github.com/apex/log"
+ "github.com/ooni/probe-engine/pkg/mocks"
"github.com/ooni/probe-engine/pkg/model"
"github.com/ooni/probe-engine/pkg/netxlite"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/testingx"
)
-func TestUbuntuParseError(t *testing.T) {
- netx := &netxlite.Netx{}
- ip, err := ubuntuIPLookup(
- context.Background(),
- &http.Client{Transport: FakeTransport{
- Resp: &http.Response{
- StatusCode: 200,
- Body: io.NopCloser(strings.NewReader("<")),
- },
- }},
- log.Log,
- model.HTTPHeaderUserAgent,
- netx.NewStdlibResolver(model.DiscardLogger),
- )
- if err == nil || !strings.HasPrefix(err.Error(), "XML syntax error") {
- t.Fatalf("not the error we expected: %+v", err)
- }
- if ip != model.DefaultProbeIP {
- t.Fatalf("not the expected IP address: %s", ip)
- }
-}
+// ubuntuRealisticresponse is a realistic response returned by cloudflare
+// with the IP address modified to belong to a public institution.
+var ubuntuRealisticresponse = []byte(`
+
+130.192.91.211
+OK
+IT
+ITA
+Italy
+09
+Lombardia
+Sesto San Giovanni
+20099
+45.5349
+9.2295
+0
+Europe/Rome
+
+`)
func TestIPLookupWorksUsingUbuntu(t *testing.T) {
- if testing.Short() {
- t.Skip("skip test in short mode")
- }
-
- netx := &netxlite.Netx{}
- ip, err := ubuntuIPLookup(
- context.Background(),
- http.DefaultClient,
- log.Log,
- model.HTTPHeaderUserAgent,
- netx.NewStdlibResolver(model.DiscardLogger),
- )
- if err != nil {
- t.Fatal(err)
- }
- if net.ParseIP(ip) == nil {
- t.Fatalf("not an IP address: '%s'", ip)
- }
+
+ // We want to make sure the real server gives us an IP address.
+ t.Run("is working as intended when using the real server", func(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skip test in short mode")
+ }
+
+ netx := &netxlite.Netx{}
+ ip, err := ubuntuIPLookup(
+ context.Background(),
+ http.DefaultClient,
+ log.Log,
+ model.HTTPHeaderUserAgent,
+ netx.NewStdlibResolver(model.DiscardLogger),
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ if net.ParseIP(ip) == nil {
+ t.Fatalf("not an IP address: '%s'", ip)
+ }
+ })
+
+ // But we also want to make sure everything is working as intended when using
+ // a local HTTP server, as well as that we can handle errors, so that we can run
+ // tests in short mode. This is done with the tests below.
+
+ t.Run("is working as intended when using a fake server", func(t *testing.T) {
+ // create a fake server returning an hardcoded IP address.
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write(ubuntuRealisticresponse)
+ }))
+ defer srv.Close()
+
+ // create an HTTP client that uses the fake server.
+ client := &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ // rewrite the request URL to be the one of the fake server
+ req.URL = runtimex.Try1(url.Parse(srv.URL))
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // figure out the IP address using ubuntu
+ netx := &netxlite.Netx{}
+ ip, err := ubuntuIPLookup(
+ context.Background(),
+ client,
+ log.Log,
+ model.HTTPHeaderUserAgent,
+ netx.NewStdlibResolver(model.DiscardLogger),
+ )
+
+ // we expect this call to succeed
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // we expect to get back a valid IPv4/IPv6 address
+ if net.ParseIP(ip) == nil {
+ t.Fatalf("not an IP address: '%s'", ip)
+ }
+
+ // we expect to see exactly the IP address that we want to see
+ if ip != "130.192.91.211" {
+ t.Fatal("unexpected IP address", ip)
+ }
+ })
+
+ t.Run("correctly handles network errors", func(t *testing.T) {
+ srv := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer srv.Close()
+
+ // create an HTTP client that uses the fake server.
+ client := &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ // rewrite the request URL to be the one of the fake server
+ req.URL = runtimex.Try1(url.Parse(srv.URL))
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // figure out the IP address using ubuntu
+ netx := &netxlite.Netx{}
+ ip, err := ubuntuIPLookup(
+ context.Background(),
+ client,
+ log.Log,
+ model.HTTPHeaderUserAgent,
+ netx.NewStdlibResolver(model.DiscardLogger),
+ )
+
+ // we expect to see ECONNRESET here
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // the returned IP address should be the default one
+ if ip != model.DefaultProbeIP {
+ t.Fatal("unexpected IP address", ip)
+ }
+ })
+
+ t.Run("correctly handles parsing errors", func(t *testing.T) {
+ // create a fake server returnning different keys
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`<`)) // note: invalid XML
+ }))
+ defer srv.Close()
+
+ // create an HTTP client that uses the fake server.
+ client := &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ // rewrite the request URL to be the one of the fake server
+ req.URL = runtimex.Try1(url.Parse(srv.URL))
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // figure out the IP address using ubuntu
+ netx := &netxlite.Netx{}
+ ip, err := ubuntuIPLookup(
+ context.Background(),
+ client,
+ log.Log,
+ model.HTTPHeaderUserAgent,
+ netx.NewStdlibResolver(model.DiscardLogger),
+ )
+
+ // we expect to see an XML parsing error here
+ if err == nil || !strings.HasPrefix(err.Error(), "XML syntax error") {
+ t.Fatalf("not the error we expected: %+v", err)
+ }
+
+ // the returned IP address should be the default one
+ if ip != model.DefaultProbeIP {
+ t.Fatal("unexpected IP address", ip)
+ }
+ })
+
+ t.Run("correctly handles missing IP address in a valid XML document", func(t *testing.T) {
+ // create a fake server returnning different keys
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(``)) // note: missing IP address
+ }))
+ defer srv.Close()
+
+ // create an HTTP client that uses the fake server.
+ client := &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ // rewrite the request URL to be the one of the fake server
+ req.URL = runtimex.Try1(url.Parse(srv.URL))
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // figure out the IP address using ubuntu
+ netx := &netxlite.Netx{}
+ ip, err := ubuntuIPLookup(
+ context.Background(),
+ client,
+ log.Log,
+ model.HTTPHeaderUserAgent,
+ netx.NewStdlibResolver(model.DiscardLogger),
+ )
+
+ // we expect to see an error indicating there's no IP address in the response
+ if !errors.Is(err, ErrInvalidIPAddress) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // the returned IP address should be the default one
+ if ip != model.DefaultProbeIP {
+ t.Fatal("unexpected IP address", ip)
+ }
+ })
+
+ t.Run("correctly handles the case where the IP address is invalid", func(t *testing.T) {
+ // create a fake server returnning different keys
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`foobarbaz`)) // note: not an IP address
+ }))
+ defer srv.Close()
+
+ // create an HTTP client that uses the fake server.
+ client := &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ // rewrite the request URL to be the one of the fake server
+ req.URL = runtimex.Try1(url.Parse(srv.URL))
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // figure out the IP address using ubuntu
+ netx := &netxlite.Netx{}
+ ip, err := ubuntuIPLookup(
+ context.Background(),
+ client,
+ log.Log,
+ model.HTTPHeaderUserAgent,
+ netx.NewStdlibResolver(model.DiscardLogger),
+ )
+
+ // we expect to see an error indicating there's no IP address in the response
+ if !errors.Is(err, ErrInvalidIPAddress) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // the returned IP address should be the default one
+ if ip != model.DefaultProbeIP {
+ t.Fatal("unexpected IP address", ip)
+ }
+ })
}
diff --git a/pkg/enginenetx/DESIGN.md b/pkg/enginenetx/DESIGN.md
new file mode 100644
index 000000000..fe1185d6f
--- /dev/null
+++ b/pkg/enginenetx/DESIGN.md
@@ -0,0 +1,571 @@
+# Engine Network Extensions
+
+This file documents the [./internal/enginenetx](.) package design. The content is current
+as of [probe-cli#1552](https://github.com/ooni/probe-cli/pull/1552).
+
+## Table of Contents
+
+- [Goals & Assumptions](#goals--assumptions)
+- [High-Level API](#high-level-api)
+- [Creating TLS Connections](#creating-tls-connections)
+- [Dialing Tactics](#dialing-tactics)
+- [Dialing Algorithm](#dialing-algorithm)
+- [Dialing Policies](#dialing-policies)
+ - [dnsPolicy](#dnspolicy)
+ - [userPolicy](#userpolicy)
+ - [statsPolicy](#statspolicy)
+ - [bridgePolicy](#bridgepolicy)
+- [Managing Stats](#managing-stats)
+- [Real-World Scenarios](#real-world-scenarios)
+- [Limitations and Future Work](#limitations-and-future-work)
+
+## Goals & Assumptions
+
+We define "bridge" an IP address with the following properties:
+
+1. the IP address is not expected to change frequently;
+
+2. the IP address listens on port 443 and accepts _any_ incoming SNI;
+
+3. the webserver on port 443 provides unified access to
+[OONI API services](https://docs.ooni.org/backend/ooniapi/services/).
+
+We also assume that the Web Connectivity test helpers (TH) could accept any SNIs.
+
+We also define "tactic" a tactic to perform a TLS handshake either with a
+bridge or with a TH. We also define "policy" the collection of algorithms for
+producing tactics for performing TLS handshakes.
+
+Considering all of this, this package aims to:
+
+1. overcome DNS-based censorship for "api.ooni.io" by hardcoding known-good
+bridges IP addresses inside the codebase to be used as a fallback;
+
+2. overcome SNI-based censorship for "api.ooni.io" and test helpers by choosing
+from a pre-defined list of SNIs as a fallback;
+
+3. remember and use tactics for creating TLS connections that worked previously
+and attempt to use them as a fallback;
+
+4. for the trivial case, an uncensored API backend, communication to the API
+should use the simplest way possible. This naturally leads to the fact that
+it should recover ~quickly if the conditions change (e.g., if a bridge
+is discontinued);
+
+5. for users in censored regions it should be possible to use
+tactics to overcome the restrictions;
+
+6. when using tactics, try to defer sending the true `SNI` on the wire,
+therefore trying to avoid triggering potential residual censorship blocking
+a given TCP endpoint for some time regardless of what `SNI` is being used next;
+
+7. allow users to force specific bridges and SNIs by editing
+`$OONI_HOME/engine/bridges.conf`.
+
+The rest of this document explains how we designed for achieving these goals.
+
+## High-Level API
+
+The purpose of the `enginenetx` package is to provide a `*Network` object from
+which consumers can obtain a `model.HTTPTransport` and `*http.Client` to use
+for HTTP operations:
+
+```Go
+func (n *Network) HTTPTransport() model.HTTPTransport
+func (n *Network) NewHTTPClient() *http.Client
+```
+
+**Listing 1.** `*enginenetx.Network` HTTP APIs.
+
+The `HTTPTransport` method returns a `*Network` field containing an HTTP
+transport with custom TLS connection establishment tactics depending on the
+configured policies.
+
+The `NewHTTPClient` method wraps such a transport into an `*http.Client`.
+
+## Creating TLS Connections
+
+In [network.go](network.go), `newHTTPSDialerPolicy` configures the dialing policy
+depending on the arguments passed to `NewNetwork`:
+
+1. if the `proxyURL` argument is not `nil`, we use the `dnsPolicy` alone, since
+we assume that the proxy knows how to do circumvention.
+
+2. othwerwise, we compose policies as illustrated by the following diagram:
+
+```
+ +---------------+ +-----------------+
+ | statsPolicyV2 | | bridgesPolicyV2 |
+ +------------------+ +---------------+ +-----------------+
+ | dnsPolicy | | |
+ +------------------+ | P | F
+ | | |
+ V V V
+ +-------------------+ +----------------------------------+
+ | testHelpersPolicy | | mixPolicyInterleave<3> |
+ +-------------------+ +----------------------------------+
+ | |
+ | P | F
+ | |
+ V V
+ +--------------+ +--------------------------------------+
+ | userPolicyV2 | | mixPolicyInterleave<3> |
+ +--------------+ +--------------------------------------+
+ | |
+ | P | F
+ | |
+ V V
+ +-----------------------------------+
+ | mixPolicyEitherOr |
+ +-----------------------------------+
+ |
+ V
+```
+
+**Diagram 1.** Sequence of policies constructed when not using a proxy.
+
+In the above diagram, each block is a policy and each arrow is a Go channel. We
+mark "primary" channels with "P" and "fallback" channels with "F".
+
+Here's what each policy does:
+
+1. `mixPolicyEitherOr`: if the primary channel returns tactics, just return
+them, otherwise, just return tactics from the fallback.
+
+2. `userPolicyV2`: returns tactics defined inside the `bridges.conf` file.
+
+3. `mixPolicyInterleave<3>` read three from the primary, then three from
+the fallback, then three from the primary, and continue alternating between
+channels until both of them have been drained.
+
+4. `testHelpersPolicy`: pass through each tactic it receives, then, if
+the domain is a test helper domain, also generate tactics with additional
+SNIs different from the test helper SNI.
+
+5. `dnsPolicy`: use the DNS to generate tactics where the domain name
+is also sent on the wire as the SNI.
+
+6. `statsPolicyV2`: generate tactics based on what we know to be working.
+
+7. `bridgesPolicyV2`: generate tactics using known bridges IP addresses
+and SNIs different from the `api.ooni.io` SNI.
+
+Until [probe-cli#1552](https://github.com/ooni/probe-cli/pull/1552), the whole
+policy situation was much simpler and linear, but we changed that in such a
+pull request to ensure the code was giving priority to DNS results.
+
+## Dialing Tactics
+
+Each policy implements the following interface
+(defined in [httpsdialer.go](httpsdialer.go)):
+
+```Go
+type httpsDialerPolicy interface {
+ LookupTactics(ctx context.Context, domain, port string) <-chan *httpsDialerTactic
+}
+```
+
+**Listing 2.** Interface implemented by policies.
+
+The `LookupTactics` operation is _conceptually_ similar to
+[net.Resolver.LookupHost](https://pkg.go.dev/net#Resolver.LookupHost), because
+both operations map a domain name to IP addresses to connect to. However,
+there are also some key differences, namely:
+
+1. `LookupTactics` is domain _and_ port specific, while `LookupHost`
+only takes in input the domain name to resolve;
+
+2. `LookupTactics` returns _a stream_ of viable "tactics", while `LookupHost`
+returns a list of IP addresses (we define "stream" a channel where a background
+goroutine posts content and which is closed when done).
+
+The second point, in particular, is crucial. The design of `LookupTactics` is
+such that we can start attempting to dial as soon as we have some tactics
+to try. A composed `httpsDialerPolicy` can, in fact, start multiple child `LookupTactics`
+operations and then return tactics to the caller as soon as some are ready, without
+blocking dialing until _all_ the child operations are complete.
+
+Also, as you may have guessed, the `dnsPolicy` is a policy that, under the hood,
+eventually calls [net.Resolver.LookupHost](https://pkg.go.dev/net#Resolver.LookupHost)
+to get IP addresses using the DNS used by the `*engine.Session` type. (Typically, such a
+resolver, in turn, composes several DNS-over-HTTPS resolvers with the fallback
+`getaddrinfo` resolver, and remembers which resolvers work.)
+
+A "tactic" looks like this:
+
+```Go
+type httpsDialerTactic struct {
+ Address string
+
+ Port string
+
+ SNI string
+
+ VerifyHostname string
+}
+```
+
+**Listing 3.** Structure describing a tactic.
+
+Here's an explanation of why we have each field in the struct:
+
+- `Address` and `Port` qualify the TCP endpoint;
+
+- `SNI` is the `SNI` to send as part of the TLS ClientHello;
+
+- `VerifyHostname` is the hostname to use for TLS certificate verification.
+
+The separation of `SNI` and `VerifyHostname` is what allows us to send an innocuous
+SNI over the network and then verify the certificate using the real SNI after a
+`skipVerify=true` TLS handshake has completed. (Obviously, for this trick to work,
+the HTTPS server we're using must be okay with receiving unrelated SNIs.)
+
+## Dialing Algorithm
+
+Creating TLS connections is implemented by `(*httpsDialer).DialTLSContext`, also
+part of [httpsdialer.go](httpsdialer.go).
+
+This method _morally_ does the following in ~parallel:
+
+```mermaid
+stateDiagram-v2
+ tacticsGenerator --> skipDuplicate
+ skipDuplicate --> computeHappyEyeballsDelay
+ computeHappyEyeballsDelay --> tcpConnect
+ tcpConnect --> tlsHandshake
+ tlsHandshake --> verifyCertificate
+```
+
+**Diagram 2.** Sequence of operations when dialing TLS connections.
+
+Such a diagram roughly corresponds to this Go ~pseudo-code:
+
+```Go
+func (hd *httpsDialer) DialTLSContext(
+ ctx context.Context, network string, endpoint string) (net.Conn, error) {
+ // map to ensure we don't have duplicate tactics
+ uniq := make(map[string]int)
+
+ // time when we started dialing
+ t0 := time.Now()
+
+ // index of each dialing attempt
+ idx := 0
+
+ // [...] omitting code to get hostname and port from endpoint [...]
+
+ // fetch tactics asynchronously
+ for tx := range hd.policy.LookupTactics(ctx, hostname, port) {
+
+ // avoid using the same tactic more than once
+ summary := tx.tacticSummaryKey()
+ if uniq[summary] > 0 {
+ continue
+ }
+ uniq[summary]++
+
+ // compute the happy eyeballs deadline
+ deadline := t0.Add(happyEyeballsDelay(idx))
+ idx++
+
+ // dial in a background goroutine so this code runs in parallel
+ go func(tx *httpsDialerTactic, deadline time.Duration) {
+ // wait for deadline
+ if delta := time.Until(deadline); delta > 0 {
+ time.Sleep(delta)
+ }
+
+ // dial TCP
+ conn, err := tcpConnect(tx.Address, tx.Port)
+
+ // [...] omitting error handling and passing error to DialTLSContext [...]
+
+ // handshake
+ tconn, err := tlsHandshake(conn, tx.SNI, false /* skip verification */)
+
+ // [...] omitting error handling and passing error to DialTLSContext [...]
+
+ // make sure the hostname's OK
+ err := verifyHostname(tconn, tx.VerifyHostname)
+
+ // [...] omitting error handling and passing error or conn to DialTLSContext [...]
+
+ }(tx, deadline)
+ }
+
+ // [...] omitting code to decide whether to return a conn or an error [...]
+}
+```
+
+**Listing 4.** Algorithm implementing dialing TLS connections.
+
+This simplified algorithm differs for the real implementation in that we
+have omitted the following (boring) details:
+
+1. code to obtain `hostname` and `port` from `endpoint` (e.g., code to extract
+`"x.org"` and `"443"` from `"x.org:443"`);
+
+2. code to pass back a connection or an error from a background
+goroutine to the `DialTLSContext` method;
+
+3. code to decide whether to return a `net.Conn` or an `error`;
+
+4. the fact that `DialTLSContext` uses a goroutine pool rather than creating a
+goroutine for each tactic;
+
+5. the fact that, as soon as we successfully have a connection, we
+immediately cancel any other parallel attempts.
+
+The `happyEyeballsDelay` function (in [happyeyeballs.go](happyeyeballs.go)) is
+such that we generate the following delays:
+
+| idx | delay (s) |
+| --- | --------- |
+| 1 | 0 |
+| 2 | 1 |
+| 4 | 2 |
+| 4 | 4 |
+| 5 | 8 |
+| 6 | 16 |
+| 7 | 24 |
+| 8 | 32 |
+| ... | ... |
+
+**Table 1.** Happy-eyeballs-like delays.
+
+That is, we exponentially increase the delay until `8s`, then we linearly increase by `8s`. We
+aim to space attempts to accommodate for slow access networks
+and/or access network experiencing temporary failures to deliver packets. However,
+we also aim to have dialing parallelism, to reduce the overall time to connect
+when we're experiencing many timeouts when attempting to dial.
+
+(We chose 1s as the baseline delay because that would be ~three happy-eyeballs delays as
+implemented by the Go standard library, and overall a TCP connect followed by a TLS
+handshake should roughly amount to three round trips.)
+
+Additionally, the `*httpsDialer` algorithm keeps statistics
+using an `httpsDialerEventsHandler` type:
+
+```Go
+type httpsDialerEventsHandler interface {
+ OnStarting(tactic *httpsDialerTactic)
+ OnTCPConnectError(ctx context.Context, tactic *httpsDialerTactic, err error)
+ OnTLSHandshakeError(ctx context.Context, tactic *httpsDialerTactic, err error)
+ OnTLSVerifyError(tactic *httpsDialerTactic, err error)
+ OnSuccess(tactic *httpsDialerTactic)
+}
+```
+
+**Listing 5.** Interface for collecting statistics.
+
+These statistics contribute to construct knowledge about the network
+conditions and influence the generation of tactics.
+
+## Dialing Policies
+
+### dnsPolicy
+
+The `dnsPolicy` is implemented by [dnspolicy.go](dnspolicy.go).
+
+Its `LookupTactics` algorithm is quite simple:
+
+1. we short circuit the cases in which the `domain` argument
+contains an IP address to "resolve" exactly that IP address (thus emulating
+what `getaddrinfo` would do when asked to "resolve" an IP address);
+
+2. for each resolved address, we generate tactics where the `SNI` and
+`VerifyHostname` equal the `domain`.
+
+If `httpsDialer` uses this policy as its only policy, the operation it
+performs are morally equivalent to normally dialing for TLS.
+
+### userPolicy
+
+The `userPolicy` is implemented by [userpolicy.go](userpolicy.go).
+
+When constructing a `userPolicy` with `newUserPolicy` and read user policies
+from the `$OONI_HOME/engine/bridges.conf` file.
+
+As of 2024-04-16, the structure of `bridges.conf` is like in the following example:
+
+```JavaScript
+{
+ "DomainEndpoints": {
+ "api.ooni.io:443": [{
+ "Address": "162.55.247.208",
+ "Port": "443",
+ "SNI": "www.example.com",
+ "VerifyHostname": "api.ooni.io"
+ }, {
+ /* omitted */
+ }]
+ },
+ "Version": 3
+}
+```
+
+**Listing 6.** Sample `bridges.conf` content.
+
+This example instructs to use the given tactic(s) when establishing a TLS connection to
+`"api.ooni.io:443"`. If `bridges.conf` does not contain any entry, then this policy
+would not know how to dial for a specific address and port.
+
+The `newUserPolicy` constructor reads this file from disk on startup
+and keeps its content in memory.
+
+`LookupTactics` will:
+
+1. check whether there's an entry for the given `domain` and `port`
+inside the `DomainEndpoints` map;
+
+2. if there are no entries, return zero tactics.
+
+3. otherwise return all the tactic entries.
+
+As shown in Diagram 1, because `userPolicy` is user-configured, we _entirely bypass_ the
+fallback policy when there's an user-configured entry.
+
+### statsPolicy
+
+The `statsPolicy` is implemented by [statspolicy.go](statspolicy.go).
+
+The general idea of this policy is that it depends on
+a `*statsManager` that keeps persistent stats about tactics.
+
+If we have stats about working tactics, we return them via the
+channel, otherwise, there's nothing that we can return.
+
+### bridgePolicy
+
+The `bridgePolicy` is implemented by [bridgespolicy.go](bridgespolicy.go) and
+rests on the assumptions made explicit above. That is:
+
+1. that there is at least one _bridge_ for "api.ooni.io";
+
+2. that the Web Connectivity Test Helpers accepts any SNI.
+
+This policy will just generate tactics using well known IP addresses
+and innocuous SNIs. When we are dialing for a domain different from
+"api.ooni.io", this policy would return no tactics through the channel.
+
+## Managing Stats
+
+The [statsmanager.go](statsmanager.go) file implements the `*statsManager`.
+
+We initialize the `*statsManager` by calling `newStatsManager` with a stats-trim
+interval of 30 seconds in `NewNetwork` in [network.go](network.go).
+
+The `*statsManager` keeps stats at `$OONI_HOME/engine/httpsdialerstats.state`.
+
+In `newStatsManager`, we attempt to read this file using `loadStatsContainer` and, if
+not present, we fall back to create empty stats with `newStatsContainer`.
+
+While creating the `*statsManager` we also spawn a goroutine that trims the stats
+at every stats-trimming interval by calling `(*statsManager).trim`. In turn, `trim`
+calls `statsContainerPruneEntries`, which eventually:
+
+1. removes entries not modified for more than one week;
+
+2. sorts entries and only keeps the top 10 entries.
+
+More specifically we sort entries using this algorithm:
+
+1. by decreasing success rate; then
+
+2. by decreasing number of successes; then
+
+3. by decreasing last update time.
+
+Likewise, calling `(*statsManager).Close` invokes `statsContainerPruneEntries`, and
+then ensures that we write `$OONI_HOME/engine/httpsdialerstats.state`.
+
+This way, subsequent OONI Probe runs could load the stats that are more likely
+to work and `statsPolicy` can take advantage of this information.
+
+The overall structure of `httpsdialerstats.state` is roughly the following:
+
+```JavaScript
+{
+ "DomainEndpoints": {
+ "api.ooni.io:443": {
+ "Tactics": {
+ "162.55.247.208:443 sni=api.trademe.co.nz verify=api.ooni.io": {
+ "CountStarted": 58,
+ "CountTCPConnectError": 0,
+ "CountTCPConnectInterrupt": 0,
+ "CountTCPConnectSuccess": 58,
+ "CountTLSHandshakeError": 0,
+ "CountTLSHandshakeInterrupt": 0,
+ "CountTLSVerificationError": 0,
+ "CountSuccess": 58,
+ "HistoTCPConnectError": {},
+ "HistoTLSHandshakeError": {},
+ "HistoTLSVerificationError": {},
+ "LastUpdated": "2024-04-15T10:38:53.575561+02:00",
+ "Tactic": {
+ "Address": "162.55.247.208",
+ "InitialDelay": 0,
+ "Port": "443",
+ "SNI": "api.trademe.co.nz",
+ "VerifyHostname": "api.ooni.io"
+ }
+ },
+ /* ... */
+ }
+ }
+ }
+ "Version": 5
+}
+```
+
+**Listing 7.** Content of the stats state as cached on disk.
+
+That is, the `DomainEndpoints` map contains contains an entry for each
+TLS endpoint and, in turn, such an entry contains tactics indexed by
+a summary string to speed up looking them up.
+
+For each tactic, we keep counters and histograms, the time when the
+entry had been updated last, and the tactic itself.
+
+The `*statsManager` implements `httpsDialerEventsHandler`, which means
+that it has callbacks invoked by the `*httpsDialer` for interesting
+events regarding dialing (e.g., whether TCP connect failed).
+
+These callbacks basically create or update stats by locking a mutex
+and updating the relevant counters and histograms.
+
+## Real-World Scenarios
+
+Because we always prioritize the DNS, the bridge becoming unavailable
+has no impact on uncensored probes given that we try bridge based
+strategies after we have tried all the DNS based strategies.
+
+## Limitations and Future Work
+
+1. We should integrate the [engineresolver](../engineresolver/) package with this package
+more tightly: doing that would allow users to configure the order in which we use DNS-over-HTTPS
+resolvers (see [probe#2675](https://github.com/ooni/probe/issues/2675)).
+
+2. We lack a mechanism to dynamically distribute new bridges IP addresses to probes using,
+for example, the check-in API and possibly other mechanisms. Lacking this functionality, our
+bridge strategy is incomplete since it rests on a single bridge being available. What's
+more, if this bridge disappears or is IP blocked, all the probes will have one slow bootstrap
+and probes where DNS is not working will stop working (see
+[probe#2500](https://github.com/ooni/probe/issues/2500)).
+
+3. We should consider adding TLS ClientHello fragmentation as a tactic.
+
+4. We should add support for HTTP/3 bridges.
+
+5. We should redesign the dialing algorithm to react immediately to previous
+failures rather than waiting the proper happy-eyeball time, like we also
+did for the [httpclientx](../httpclientx/) package.
+
+6. A previous implementation of this design had and explicit `InitialDelay`
+field for a tactic. We are currently not using this field as we rewrite
+the happy eyeballs delay unconditionally. Perhaps, we should keep the original
+field value when reading user policies, to give users more control.
+
+7. We should consider using existing knowledge from the stats to change
+the SNI being used when using the DNS. This would make our knowledge about
+what is working and not working much more effective than now.
diff --git a/pkg/enginenetx/bridgespolicy.go b/pkg/enginenetx/bridgespolicy.go
index 0b282630d..1b15aafea 100644
--- a/pkg/enginenetx/bridgespolicy.go
+++ b/pkg/enginenetx/bridgespolicy.go
@@ -11,100 +11,27 @@ import (
"time"
)
-// bridgesPolicy is a policy where we use bridges for communicating
+// bridgesPolicyV2 is a policy where we use bridges for communicating
// with the OONI backend, i.e., api.ooni.io.
//
// A bridge is an IP address that can route traffic from and to
// the OONI backend and accepts any SNI.
//
// The zero value is invalid; please, init MANDATORY fields.
-type bridgesPolicy struct {
- // Fallback is the MANDATORY fallback policy.
- Fallback httpsDialerPolicy
-}
+//
+// This is v2 of the bridgesPolicy because the previous implementation
+// incorporated mixing logic, while now the mixing happens outside
+// of this policy, thus giving us much more flexibility.
+type bridgesPolicyV2 struct{}
-var _ httpsDialerPolicy = &bridgesPolicy{}
+var _ httpsDialerPolicy = &bridgesPolicyV2{}
// LookupTactics implements httpsDialerPolicy.
-func (p *bridgesPolicy) LookupTactics(ctx context.Context, domain, port string) <-chan *httpsDialerTactic {
- out := make(chan *httpsDialerTactic)
-
- go func() {
- defer close(out) // tell the parent when we're done
- index := 0
-
- // emit bridges related tactics first which are empty if there are
- // no bridges for the givend domain and port
- for tx := range p.bridgesTacticsForDomain(domain, port) {
- tx.InitialDelay = happyEyeballsDelay(index)
- index += 1
- out <- tx
- }
-
- // now fallback to get more tactics (typically here the fallback
- // uses the DNS and obtains some extra tactics)
- //
- // we wrap whatever the underlying policy returns us with some
- // extra logic for better communicating with test helpers
- for tx := range p.maybeRewriteTestHelpersTactics(p.Fallback.LookupTactics(ctx, domain, port)) {
- tx.InitialDelay = happyEyeballsDelay(index)
- index += 1
- out <- tx
- }
- }()
-
- return out
-}
-
-var bridgesPolicyTestHelpersDomains = []string{
- "0.th.ooni.org",
- "1.th.ooni.org",
- "2.th.ooni.org",
- "3.th.ooni.org",
- "d33d1gs9kpq1c5.cloudfront.net",
-}
-
-// TODO(bassosimone): this would be slices.Contains when we'll use go1.21
-func bridgesPolicySlicesContains(slice []string, value string) bool {
- for _, entry := range slice {
- if value == entry {
- return true
- }
- }
- return false
-}
-
-func (p *bridgesPolicy) maybeRewriteTestHelpersTactics(input <-chan *httpsDialerTactic) <-chan *httpsDialerTactic {
- out := make(chan *httpsDialerTactic)
-
- go func() {
- defer close(out) // tell the parent when we're done
-
- for tactic := range input {
- // When we're not connecting to a TH, pass the policy down the chain unmodified
- if !bridgesPolicySlicesContains(bridgesPolicyTestHelpersDomains, tactic.VerifyHostname) {
- out <- tactic
- continue
- }
-
- // This is the case where we're connecting to a test helper. Let's try
- // to produce policies hiding the SNI to censoring middleboxes.
- for _, sni := range p.bridgesDomainsInRandomOrder() {
- out <- &httpsDialerTactic{
- Address: tactic.Address,
- InitialDelay: 0,
- Port: tactic.Port,
- SNI: sni,
- VerifyHostname: tactic.VerifyHostname,
- }
- }
- }
- }()
-
- return out
+func (p *bridgesPolicyV2) LookupTactics(ctx context.Context, domain, port string) <-chan *httpsDialerTactic {
+ return bridgesTacticsForDomain(domain, port)
}
-func (p *bridgesPolicy) bridgesTacticsForDomain(domain, port string) <-chan *httpsDialerTactic {
+func bridgesTacticsForDomain(domain, port string) <-chan *httpsDialerTactic {
out := make(chan *httpsDialerTactic)
go func() {
@@ -115,11 +42,11 @@ func (p *bridgesPolicy) bridgesTacticsForDomain(domain, port string) <-chan *htt
return
}
- for _, ipAddr := range p.bridgesAddrs() {
- for _, sni := range p.bridgesDomainsInRandomOrder() {
+ for _, ipAddr := range bridgesAddrs() {
+ for _, sni := range bridgesDomainsInRandomOrder() {
out <- &httpsDialerTactic{
Address: ipAddr,
- InitialDelay: 0,
+ InitialDelay: 0, // set when dialing
Port: port,
SNI: sni,
VerifyHostname: domain,
@@ -131,23 +58,23 @@ func (p *bridgesPolicy) bridgesTacticsForDomain(domain, port string) <-chan *htt
return out
}
-func (p *bridgesPolicy) bridgesDomainsInRandomOrder() (out []string) {
- out = p.bridgesDomains()
- r := rand.New(rand.NewSource(time.Now().UnixNano()))
+func bridgesDomainsInRandomOrder() (out []string) {
+ out = bridgesDomains()
+ r := rand.New(rand.NewSource(time.Now().UnixNano())) // #nosec G404 -- not really important
r.Shuffle(len(out), func(i, j int) {
out[i], out[j] = out[j], out[i]
})
return
}
-func (p *bridgesPolicy) bridgesAddrs() (out []string) {
+func bridgesAddrs() (out []string) {
return append(
out,
"162.55.247.208",
)
}
-func (p *bridgesPolicy) bridgesDomains() (out []string) {
+func bridgesDomains() (out []string) {
// See https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/-/issues/40273
return append(
out,
diff --git a/pkg/enginenetx/bridgespolicy_test.go b/pkg/enginenetx/bridgespolicy_test.go
index ebf6e708b..243ac4828 100644
--- a/pkg/enginenetx/bridgespolicy_test.go
+++ b/pkg/enginenetx/bridgespolicy_test.go
@@ -2,29 +2,14 @@ package enginenetx
import (
"context"
- "errors"
"testing"
-
- "github.com/ooni/probe-engine/pkg/mocks"
- "github.com/ooni/probe-engine/pkg/model"
)
-func TestBeaconsPolicy(t *testing.T) {
- t.Run("for domains for which we don't have bridges and DNS failure", func(t *testing.T) {
- expected := errors.New("mocked error")
- p := &bridgesPolicy{
- Fallback: &dnsPolicy{
- Logger: model.DiscardLogger,
- Resolver: &mocks.Resolver{
- MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
- return nil, expected
- },
- },
- },
- }
+func TestBridgesPolicyV2(t *testing.T) {
+ t.Run("for domains for which we don't have bridges", func(t *testing.T) {
+ p := &bridgesPolicyV2{}
- ctx := context.Background()
- tactics := p.LookupTactics(ctx, "www.example.com", "443")
+ tactics := p.LookupTactics(context.Background(), "www.example.com", "443")
var count int
for range tactics {
@@ -36,73 +21,30 @@ func TestBeaconsPolicy(t *testing.T) {
}
})
- t.Run("for domains for which we don't have bridges and DNS success", func(t *testing.T) {
- p := &bridgesPolicy{
- Fallback: &dnsPolicy{
- Logger: model.DiscardLogger,
- Resolver: &mocks.Resolver{
- MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
- return []string{"93.184.216.34"}, nil
- },
- },
- },
- }
-
- ctx := context.Background()
- tactics := p.LookupTactics(ctx, "www.example.com", "443")
-
- var count int
- for tactic := range tactics {
- count++
-
- if tactic.Port != "443" {
- t.Fatal("the port should always be 443")
- }
- if tactic.Address != "93.184.216.34" {
- t.Fatal("the host should always be 93.184.216.34")
- }
-
- if tactic.SNI != "www.example.com" {
- t.Fatal("the SNI field should always be like `www.example.com`")
- }
-
- if tactic.VerifyHostname != "www.example.com" {
- t.Fatal("the VerifyHostname field should always be like `www.example.com`")
- }
- }
-
- if count != 1 {
- t.Fatal("expected to see one tactic")
- }
- })
-
t.Run("for the api.ooni.io domain", func(t *testing.T) {
- expected := errors.New("mocked error")
- p := &bridgesPolicy{
- Fallback: &dnsPolicy{
- Logger: model.DiscardLogger,
- Resolver: &mocks.Resolver{
- MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
- return nil, expected
- },
- },
- },
- }
+ p := &bridgesPolicyV2{}
- ctx := context.Background()
- tactics := p.LookupTactics(ctx, "api.ooni.io", "443")
+ tactics := p.LookupTactics(context.Background(), "api.ooni.io", "443")
var count int
for tactic := range tactics {
count++
+ // for each generated tactic, make sure we're getting the
+ // expected value for each of the fields
+
if tactic.Port != "443" {
t.Fatal("the port should always be 443")
}
+
if tactic.Address != "162.55.247.208" {
t.Fatal("the host should always be 162.55.247.208")
}
+ if tactic.InitialDelay != 0 {
+ t.Fatal("unexpected InitialDelay")
+ }
+
if tactic.SNI == "api.ooni.io" {
t.Fatal("we should not see the `api.ooni.io` SNI on the wire")
}
@@ -116,49 +58,4 @@ func TestBeaconsPolicy(t *testing.T) {
t.Fatal("expected to see at least one tactic")
}
})
-
- t.Run("for test helper domains", func(t *testing.T) {
- for _, domain := range bridgesPolicyTestHelpersDomains {
- t.Run(domain, func(t *testing.T) {
- expectedAddrs := []string{"164.92.180.7"}
-
- p := &bridgesPolicy{
- Fallback: &dnsPolicy{
- Logger: model.DiscardLogger,
- Resolver: &mocks.Resolver{
- MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
- return expectedAddrs, nil
- },
- },
- },
- }
-
- ctx := context.Background()
- index := 0
- for tactics := range p.LookupTactics(ctx, domain, "443") {
-
- if tactics.Address != "164.92.180.7" {
- t.Fatal("unexpected .Address")
- }
-
- if tactics.InitialDelay != happyEyeballsDelay(index) {
- t.Fatal("unexpected .InitialDelay")
- }
- index++
-
- if tactics.Port != "443" {
- t.Fatal("unexpected .Port")
- }
-
- if tactics.SNI == domain {
- t.Fatal("unexpected .Domain")
- }
-
- if tactics.VerifyHostname != domain {
- t.Fatal("unexpected .VerifyHostname")
- }
- }
- })
- }
- })
}
diff --git a/pkg/enginenetx/dnspolicy.go b/pkg/enginenetx/dnspolicy.go
index 9fd97fb8f..f481359f8 100644
--- a/pkg/enginenetx/dnspolicy.go
+++ b/pkg/enginenetx/dnspolicy.go
@@ -16,8 +16,6 @@ import (
// given resolver and the domain as the SNI.
//
// The zero value is invalid; please, init all MANDATORY fields.
-//
-// This policy uses an Happy-Eyeballs-like algorithm.
type dnsPolicy struct {
// Logger is the MANDATORY logger.
Logger model.Logger
@@ -56,10 +54,10 @@ func (p *dnsPolicy) LookupTactics(
}
// The tactics we generate here have SNI == VerifyHostname == domain
- for idx, addr := range addrs {
+ for _, addr := range addrs {
tactic := &httpsDialerTactic{
Address: addr,
- InitialDelay: happyEyeballsDelay(idx),
+ InitialDelay: 0, // set when dialing
Port: port,
SNI: domain,
VerifyHostname: domain,
diff --git a/pkg/enginenetx/dnspolicy_test.go b/pkg/enginenetx/dnspolicy_test.go
index a3fbc1113..0eccc2dd1 100644
--- a/pkg/enginenetx/dnspolicy_test.go
+++ b/pkg/enginenetx/dnspolicy_test.go
@@ -54,6 +54,9 @@ func TestDNSPolicy(t *testing.T) {
if tactic.Address != "130.192.91.211" {
t.Fatal("invalid endpoint address")
}
+ if tactic.InitialDelay != 0 {
+ t.Fatal("unexpected .InitialDelay")
+ }
if tactic.Port != "443" {
t.Fatal("invalid endpoint port")
}
diff --git a/pkg/enginenetx/filter.go b/pkg/enginenetx/filter.go
new file mode 100644
index 000000000..998c1c98d
--- /dev/null
+++ b/pkg/enginenetx/filter.go
@@ -0,0 +1,76 @@
+package enginenetx
+
+// filterOutNilTactics filters out nil tactics.
+//
+// This function returns a channel where we emit the edited
+// tactics, and which we clone when we're done.
+func filterOutNilTactics(input <-chan *httpsDialerTactic) <-chan *httpsDialerTactic {
+ output := make(chan *httpsDialerTactic)
+ go func() {
+ defer close(output)
+ for tx := range input {
+ if tx != nil {
+ output <- tx
+ }
+ }
+ }()
+ return output
+}
+
+// filterOnlyKeepUniqueTactics only keeps unique tactics.
+//
+// This function returns a channel where we emit the edited
+// tactics, and which we clone when we're done.
+func filterOnlyKeepUniqueTactics(input <-chan *httpsDialerTactic) <-chan *httpsDialerTactic {
+ output := make(chan *httpsDialerTactic)
+ go func() {
+
+ // make sure we close output chan
+ defer close(output)
+
+ // useful to make sure we don't emit two equal policy in a single run
+ uniq := make(map[string]int)
+
+ for tx := range input {
+ // handle the case in which we already emitted a tactic
+ key := tx.tacticSummaryKey()
+ if uniq[key] > 0 {
+ continue
+ }
+ uniq[key]++
+
+ // emit the tactic
+ output <- tx
+ }
+
+ }()
+ return output
+}
+
+// filterAssignInitialDelays assigns initial delays to tactics.
+//
+// This function returns a channel where we emit the edited
+// tactics, and which we clone when we're done.
+func filterAssignInitialDelays(input <-chan *httpsDialerTactic) <-chan *httpsDialerTactic {
+ output := make(chan *httpsDialerTactic)
+ go func() {
+
+ // make sure we close output chan
+ defer close(output)
+
+ index := 0
+ for tx := range input {
+ // TODO(bassosimone): what do we do now about the user configured
+ // initial delays? Should we declare them as deprecated?
+
+ // rewrite the delays
+ tx.InitialDelay = happyEyeballsDelay(index)
+ index++
+
+ // emit the tactic
+ output <- tx
+ }
+
+ }()
+ return output
+}
diff --git a/pkg/enginenetx/filter_test.go b/pkg/enginenetx/filter_test.go
new file mode 100644
index 000000000..8ce8e08c7
--- /dev/null
+++ b/pkg/enginenetx/filter_test.go
@@ -0,0 +1,111 @@
+package enginenetx
+
+import (
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/testingx"
+)
+
+func TestFilterOutNilTactics(t *testing.T) {
+ inputs := []*httpsDialerTactic{
+ nil,
+ nil,
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "x.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "www.polito.it",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ nil,
+ }
+
+ expect := []*httpsDialerTactic{
+ inputs[2], inputs[4],
+ }
+
+ var output []*httpsDialerTactic
+ for tx := range filterOutNilTactics(streamTacticsFromSlice(inputs)) {
+ output = append(output, tx)
+ }
+
+ if diff := cmp.Diff(expect, output); diff != "" {
+ t.Fatal(diff)
+ }
+}
+
+func TestFilterOnlyKeepUniqueTactics(t *testing.T) {
+ templates := []*httpsDialerTactic{{
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "www.example.com",
+ VerifyHostname: "api.ooni.io",
+ }, {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "www.kernel.org",
+ VerifyHostname: "api.ooni.io",
+ }, {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "x.org",
+ VerifyHostname: "api.ooni.io",
+ }, {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "www.polito.it",
+ VerifyHostname: "api.ooni.io",
+ }}
+
+ inputs := []*httpsDialerTactic{
+ templates[2], templates[1], templates[1],
+ templates[2], templates[2], templates[1],
+ templates[0], templates[1], templates[0],
+ templates[2], templates[1], templates[2],
+ templates[1], templates[0], templates[1],
+ templates[3], // only once at the end
+ }
+
+ expect := []*httpsDialerTactic{
+ templates[2], templates[1], templates[0], templates[3],
+ }
+
+ var output []*httpsDialerTactic
+ for tx := range filterOnlyKeepUniqueTactics(streamTacticsFromSlice(inputs)) {
+ output = append(output, tx)
+ }
+
+ if diff := cmp.Diff(expect, output); diff != "" {
+ t.Fatal(diff)
+ }
+}
+
+func TestFilterAssignInitalDelays(t *testing.T) {
+ inputs := []*httpsDialerTactic{}
+ ff := &testingx.FakeFiller{}
+ ff.Fill(&inputs)
+ idx := 0
+ for tx := range filterAssignInitialDelays(streamTacticsFromSlice(inputs)) {
+ if tx.InitialDelay != happyEyeballsDelay(idx) {
+ t.Fatal("unexpected .InitialDelay", tx.InitialDelay, "for", idx)
+ }
+ idx++
+ }
+ if idx < 1 {
+ t.Fatal("expected to see at least one entry")
+ }
+}
diff --git a/pkg/enginenetx/httpsdialer.go b/pkg/enginenetx/httpsdialer.go
index 19f8f509d..ff2c76e20 100644
--- a/pkg/enginenetx/httpsdialer.go
+++ b/pkg/enginenetx/httpsdialer.go
@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"net"
+ "sync"
"sync/atomic"
"time"
@@ -97,7 +98,7 @@ type httpsDialerPolicy interface {
// httpsDialerEventsHandler handles events occurring while we try dialing TLS.
type httpsDialerEventsHandler interface {
- // These callbacks are invoked during the TLS handshake to inform this
+ // These callbacks are invoked during the TLS dialing to inform this
// interface about events that occurred. A policy SHOULD keep track of which
// addresses, SNIs, etc. work and return them more frequently.
//
@@ -202,18 +203,26 @@ func (hd *httpsDialer) DialTLSContext(ctx context.Context, network string, endpo
return nil, err
}
+ // TODO(bassosimone): this code should be refactored using the same
+ // pattern used by `./internal/httpclientx` to perform attempts faster
+ // in case there is an initial early failure.
+
// We need a cancellable context to interrupt the tactics emitter early when we
// immediately get a valid response and we don't need to use other tactics.
ctx, cancel := context.WithCancel(ctx)
defer cancel()
+ // Create structure for computing the zero dialing time once during
+ // the first dial, so that subsequent attempts use happy eyeballs based
+ // on the moment in which we tried the first dial.
+ t0 := &httpsDialerWorkerZeroTime{}
+
// The emitter will emit tactics and then close the channel when done. We spawn 16 workers
// that handle tactics in parallel and post results on the collector channel.
- emitter := hd.policy.LookupTactics(ctx, hostname, port)
+ emitter := httpsDialerFilterTactics(hd.policy.LookupTactics(ctx, hostname, port))
collector := make(chan *httpsDialerErrorOrConn)
joiner := make(chan any)
const parallelism = 16
- t0 := time.Now()
for idx := 0; idx < parallelism; idx++ {
go hd.worker(ctx, joiner, emitter, t0, collector)
}
@@ -236,8 +245,10 @@ func (hd *httpsDialer) DialTLSContext(ctx context.Context, network string, endpo
continue
}
- // Save the conn and tell goroutines to stop ASAP
+ // Save the conn
connv = append(connv, result.Conn)
+
+ // Interrupt other concurrent dialing attempts
cancel()
}
}
@@ -245,13 +256,57 @@ func (hd *httpsDialer) DialTLSContext(ctx context.Context, network string, endpo
return httpsDialerReduceResult(connv, errorv)
}
+// httpsDialerWorkerZeroTime contains the zero time used when dialing. We set this
+// zero time when we start the first dialing attempt, such that subsequent attempts
+// are correctly spaced starting from such a zero time.
+//
+// A previous approach was that we were taking the zero time when we started
+// getting tactics, but this approach was wrong, because it caused several tactics
+// to be ready, when the DNS lookup was slow.
+//
+// The zero value of this structure is ready to use.
+type httpsDialerWorkerZeroTime struct {
+ // mu provides mutual exclusion.
+ mu sync.Mutex
+
+ // t is the zero time.
+ t time.Time
+}
+
+// Get returns the zero dialing time. The first invocation of this method
+// saves the zero dialing time and subsquent invocations just return it.
+//
+// This method is safe to be called concurrently by goroutines.
+func (t0 *httpsDialerWorkerZeroTime) Get() time.Time {
+ defer t0.mu.Unlock()
+ t0.mu.Lock()
+ if t0.t.IsZero() {
+ t0.t = time.Now()
+ }
+ return t0.t
+}
+
+// httpsDialerFilterTactics filters the tactics to:
+//
+// 1. be paranoid and filter out nil tactics if any;
+//
+// 2. avoid emitting duplicate tactics as part of the same run;
+//
+// 3. rewrite the happy eyeball delays.
+//
+// This function returns a channel where we emit the edited
+// tactics, and which we clone when we're done.
+func httpsDialerFilterTactics(input <-chan *httpsDialerTactic) <-chan *httpsDialerTactic {
+ return filterAssignInitialDelays(filterOnlyKeepUniqueTactics(filterOutNilTactics(input)))
+}
+
// httpsDialerReduceResult returns either an established conn or an error, using [errDNSNoAnswer] in
// case the list of connections and the list of errors are empty.
func httpsDialerReduceResult(connv []model.TLSConn, errorv []error) (model.TLSConn, error) {
switch {
case len(connv) >= 1:
for _, c := range connv[1:] {
- c.Close()
+ _ = c.Close()
}
return connv[0], nil
@@ -271,7 +326,7 @@ func (hd *httpsDialer) worker(
ctx context.Context,
joiner chan<- any,
reader <-chan *httpsDialerTactic,
- t0 time.Time,
+ t0 *httpsDialerWorkerZeroTime,
writer chan<- *httpsDialerErrorOrConn,
) {
// let the parent know that we terminated
@@ -295,7 +350,7 @@ func (hd *httpsDialer) worker(
func (hd *httpsDialer) dialTLS(
ctx context.Context,
logger model.Logger,
- t0 time.Time,
+ t0 *httpsDialerWorkerZeroTime,
tactic *httpsDialerTactic,
) (model.TLSConn, error) {
// honor happy-eyeballs delays and wait for the tactic to be ready to run
@@ -303,6 +358,9 @@ func (hd *httpsDialer) dialTLS(
return nil, err
}
+ // for debugging let the user know which tactic is ready
+ logger.Infof("tactic '%+v' is ready", tactic)
+
// tell the observer that we're starting
hd.stats.OnStarting(tactic)
@@ -321,7 +379,7 @@ func (hd *httpsDialer) dialTLS(
// create TLS configuration
tlsConfig := &tls.Config{
- InsecureSkipVerify: true, // Note: we're going to verify at the end of the func!
+ InsecureSkipVerify: true, // #nosec G402 - we verify at end of func
NextProtos: []string{"h2", "http/1.1"},
RootCAs: hd.rootCAs,
ServerName: tactic.SNI,
@@ -342,7 +400,7 @@ func (hd *httpsDialer) dialTLS(
// handle handshake error
if err != nil {
hd.stats.OnTLSHandshakeError(ctx, tactic, err)
- tcpConn.Close()
+ _ = tcpConn.Close()
return nil, err
}
@@ -354,7 +412,7 @@ func (hd *httpsDialer) dialTLS(
// handle verification error
if err != nil {
hd.stats.OnTLSVerifyError(tactic, err)
- tlsConn.Close()
+ _ = tlsConn.Close()
return nil, err
}
@@ -369,10 +427,10 @@ func (hd *httpsDialer) dialTLS(
// return the context error if the context expires.
func httpsDialerTacticWaitReady(
ctx context.Context,
- t0 time.Time,
+ t0 *httpsDialerWorkerZeroTime,
tactic *httpsDialerTactic,
) error {
- deadline := t0.Add(tactic.InitialDelay)
+ deadline := t0.Get().Add(tactic.InitialDelay)
delta := time.Until(deadline)
if delta <= 0 {
return nil
diff --git a/pkg/enginenetx/httpsdialer_test.go b/pkg/enginenetx/httpsdialer_test.go
index 8535f8314..8e2e41f7d 100644
--- a/pkg/enginenetx/httpsdialer_test.go
+++ b/pkg/enginenetx/httpsdialer_test.go
@@ -632,3 +632,231 @@ func TestHTTPSDialerReduceResult(t *testing.T) {
}
})
}
+
+// Make sure that (1) we remove nils; (2) we avoid emitting duplicate tactics; (3) we fill
+// the happy-eyeballs delays for each entry we return.
+func TestHTTPSDialerFilterTactics(t *testing.T) {
+ // define the inputs vector including duplicates and nils
+ inputs := []*httpsDialerTactic{
+ nil,
+ nil,
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "x.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "www.polito.it",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ nil,
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "x.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "www.polito.it",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "x.com",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ nil,
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "kerneltrap.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ nil,
+ nil,
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "kerneltrap.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ nil,
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "freebsd.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ nil,
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "kerneltrap.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ nil,
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "dragonflybsd.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "kerneltrap.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ nil,
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "openbsd.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ nil,
+ {
+ Address: "130.192.91.231",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "openbsd.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ nil,
+ {
+ Address: "130.192.91.231",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "openbsd.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ nil,
+ {
+ Address: "130.192.91.231",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "netbsd.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ nil,
+ {
+ Address: "130.192.91.231",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "openbsd.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ nil,
+ }
+
+ // define the expectations
+ expect := []*httpsDialerTactic{
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "x.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ {
+ Address: "130.192.91.211",
+ InitialDelay: time.Second,
+ Port: "443",
+ SNI: "www.polito.it",
+ VerifyHostname: "api.ooni.io",
+ },
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 2 * time.Second,
+ Port: "443",
+ SNI: "x.com",
+ VerifyHostname: "api.ooni.io",
+ },
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 4 * time.Second,
+ Port: "443",
+ SNI: "kerneltrap.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 8 * time.Second,
+ Port: "443",
+ SNI: "freebsd.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 16 * time.Second,
+ Port: "443",
+ SNI: "dragonflybsd.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ {
+ Address: "130.192.91.211",
+ InitialDelay: 24 * time.Second,
+ Port: "443",
+ SNI: "openbsd.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ {
+ Address: "130.192.91.231",
+ InitialDelay: 32 * time.Second,
+ Port: "443",
+ SNI: "openbsd.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ {
+ Address: "130.192.91.231",
+ InitialDelay: 40 * time.Second,
+ Port: "443",
+ SNI: "netbsd.org",
+ VerifyHostname: "api.ooni.io",
+ },
+ }
+
+ // run the algorithm
+ var results []*httpsDialerTactic
+ for tx := range httpsDialerFilterTactics(streamTacticsFromSlice(inputs)) {
+ results = append(results, tx)
+ }
+
+ // compare the results
+ if diff := cmp.Diff(expect, results); diff != "" {
+ t.Fatal(diff)
+ }
+}
diff --git a/pkg/enginenetx/mixpolicy.go b/pkg/enginenetx/mixpolicy.go
new file mode 100644
index 000000000..697cfa360
--- /dev/null
+++ b/pkg/enginenetx/mixpolicy.go
@@ -0,0 +1,133 @@
+package enginenetx
+
+//
+// Mix policies - ability of mixing from a primary policy and a fallback policy
+// in a more flexible way than strictly falling back
+//
+
+import (
+ "context"
+
+ "github.com/ooni/probe-engine/pkg/optional"
+)
+
+// mixPolicyEitherOr reads from primary and only if primary does
+// not return any tactic, then it reads from fallback.
+type mixPolicyEitherOr struct {
+ // Primary is the primary policy.
+ Primary httpsDialerPolicy
+
+ // Fallback is the fallback policy.
+ Fallback httpsDialerPolicy
+}
+
+var _ httpsDialerPolicy = &mixPolicyEitherOr{}
+
+// LookupTactics implements httpsDialerPolicy.
+func (m *mixPolicyEitherOr) LookupTactics(ctx context.Context, domain string, port string) <-chan *httpsDialerTactic {
+ // create the output channel
+ output := make(chan *httpsDialerTactic)
+
+ go func() {
+ // make sure we eventually close the output channel
+ defer close(output)
+
+ // drain the primary policy
+ var count int
+ for tx := range m.Primary.LookupTactics(ctx, domain, port) {
+ output <- tx
+ count++
+ }
+
+ // if the primary worked, we're good
+ if count > 0 {
+ return
+ }
+
+ // drain the fallback policy
+ for tx := range m.Fallback.LookupTactics(ctx, domain, port) {
+ output <- tx
+ }
+ }()
+
+ return output
+}
+
+// mixPolicyInterleave interleaves policies by a given interleaving
+// factor. Say the interleave factor is N, then we first read N tactics
+// from the primary policy, then N from the fallback one, and we keep
+// going on like this until we've read all the tactics from both.
+type mixPolicyInterleave struct {
+ // Primary is the primary policy. We will read N from this
+ // policy first, then N from fallback, and so on.
+ Primary httpsDialerPolicy
+
+ // Fallback is the fallback policy.
+ Fallback httpsDialerPolicy
+
+ // Factor is the interleaving factor to use. If this value is
+ // zero, we behave like it was set to one.
+ Factor uint8
+}
+
+var _ httpsDialerPolicy = &mixPolicyInterleave{}
+
+// LookupTactics implements httpsDialerPolicy.
+func (p *mixPolicyInterleave) LookupTactics(ctx context.Context, domain string, port string) <-chan *httpsDialerTactic {
+ // create the output channel
+ output := make(chan *httpsDialerTactic)
+
+ go func() {
+ // make sure we eventually close the output channel
+ defer close(output)
+
+ // obtain the primary channel
+ primary := optional.Some(p.Primary.LookupTactics(ctx, domain, port))
+
+ // obtain the fallback channel
+ fallback := optional.Some(p.Fallback.LookupTactics(ctx, domain, port))
+
+ // loop until both channels are drained
+ for !primary.IsNone() || !fallback.IsNone() {
+ // take N from primary if possible
+ primary = p.maybeTakeN(primary, output)
+
+ // take N from secondary if possible
+ fallback = p.maybeTakeN(fallback, output)
+ }
+ }()
+
+ return output
+}
+
+// maybeTakeN takes N entries from input if it's not none. When input is not
+// none and reading from it indicates EOF, this function returns none. Otherwise,
+// it returns the same value given as input.
+func (p *mixPolicyInterleave) maybeTakeN(
+ input optional.Value[<-chan *httpsDialerTactic],
+ output chan<- *httpsDialerTactic,
+) optional.Value[<-chan *httpsDialerTactic] {
+ // make sure we've not already drained this channel
+ if !input.IsNone() {
+
+ // obtain the underlying channel
+ ch := input.Unwrap()
+
+ // take N entries from the channel
+ for idx := uint8(0); idx < max(1, p.Factor); idx++ {
+
+ // attempt to get the next tactic
+ tactic, good := <-ch
+
+ // handle the case where the channel has been drained
+ if !good {
+ return optional.None[<-chan *httpsDialerTactic]()
+ }
+
+ // emit the tactic
+ output <- tactic
+ }
+ }
+
+ return input
+}
diff --git a/pkg/enginenetx/mixpolicy_test.go b/pkg/enginenetx/mixpolicy_test.go
new file mode 100644
index 000000000..7927fd405
--- /dev/null
+++ b/pkg/enginenetx/mixpolicy_test.go
@@ -0,0 +1,370 @@
+package enginenetx
+
+import (
+ "context"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+)
+
+func TestMixPolicyEitherOr(t *testing.T) {
+ // testcase is a test case implemented by this function.
+ type testcase struct {
+ // name is the name of the test case
+ name string
+
+ // primary is the primary policy to use
+ primary httpsDialerPolicy
+
+ // fallback is the fallback policy to use
+ fallback httpsDialerPolicy
+
+ // domain is the domain to pass to LookupTactics
+ domain string
+
+ // port is the port to pass to LookupTactics
+ port string
+
+ // expect is the expectations in terms of tactics
+ expect []*httpsDialerTactic
+ }
+
+ // This is the list of tactics that we expect the primary
+ // policy to return when we're not using a null policy
+ expectedPrimaryTactics := []*httpsDialerTactic{{
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "shelob.polito.it",
+ VerifyHostname: "api.ooni.io",
+ }, {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "whitespider.polito.it",
+ VerifyHostname: "api.ooni.io",
+ }, {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "mirkwood.polito.it",
+ VerifyHostname: "api.ooni.io",
+ }, {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "highgarden.polito.it",
+ VerifyHostname: "api.ooni.io",
+ }}
+
+ // Create the non-null primary policy
+ primary := &userPolicyV2{
+ Root: &userPolicyRoot{
+ DomainEndpoints: map[string][]*httpsDialerTactic{
+ "api.ooni.io:443": expectedPrimaryTactics,
+ },
+ Version: userPolicyVersion,
+ },
+ }
+
+ // This is the list of tactics that we expect the fallback
+ // policy to return when we're not using a null policy
+ expectedFallbackTactics := []*httpsDialerTactic{{
+ Address: "130.192.91.231",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "kingslanding.polito.it",
+ VerifyHostname: "api.ooni.io",
+ }, {
+ Address: "130.192.91.231",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "pyke.polito.it",
+ VerifyHostname: "api.ooni.io",
+ }, {
+ Address: "130.192.91.231",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "winterfell.polito.it",
+ VerifyHostname: "api.ooni.io",
+ }}
+
+ // Create the non-null fallback policy
+ fallback := &userPolicyV2{
+ Root: &userPolicyRoot{
+ DomainEndpoints: map[string][]*httpsDialerTactic{
+ "api.ooni.io:443": expectedFallbackTactics,
+ },
+ Version: userPolicyVersion,
+ },
+ }
+
+ cases := []testcase{
+
+ // This test ensures that the code is WAI with two null policies
+ {
+ name: "with two null policies",
+ primary: &nullPolicy{},
+ fallback: &nullPolicy{},
+ domain: "api.ooni.io",
+ port: "443",
+ expect: nil,
+ },
+
+ // This test ensures that we get the content of the primary
+ // policy when the fallback policy is the null policy
+ {
+ name: "with the fallback policy being null",
+ primary: primary,
+ fallback: &nullPolicy{},
+ domain: "api.ooni.io",
+ port: "443",
+ expect: expectedPrimaryTactics,
+ },
+
+ // This test ensures that we get the content of the fallback
+ // policy when the primary policy is the null policy
+ {
+ name: "with the primary policy being null",
+ primary: &nullPolicy{},
+ fallback: fallback,
+ domain: "api.ooni.io",
+ port: "443",
+ expect: expectedFallbackTactics,
+ },
+
+ // This test ensures that we correctly only get the primary
+ // policy when both primary and fallback are set
+ {
+ name: "with both policies being nonnull",
+ primary: primary,
+ fallback: fallback,
+ domain: "api.ooni.io",
+ port: "443",
+ expect: expectedPrimaryTactics,
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+
+ // construct the mixPolicyEitherOr instance
+ p := &mixPolicyEitherOr{
+ Primary: tc.primary,
+ Fallback: tc.fallback,
+ }
+
+ // start looking up for tactics
+ outch := p.LookupTactics(context.Background(), tc.domain, tc.port)
+
+ // collect all the generated tactics
+ var got []*httpsDialerTactic
+ for entry := range outch {
+ got = append(got, entry)
+ }
+
+ // compare to expectations
+ if diff := cmp.Diff(tc.expect, got); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+ }
+}
+
+func TestMixPolicyInterleave(t *testing.T) {
+ // testcase is a test case implemented by this function.
+ type testcase struct {
+ // name is the name of the test case
+ name string
+
+ // primary is the primary policy to use
+ primary httpsDialerPolicy
+
+ // fallback is the fallback policy to use
+ fallback httpsDialerPolicy
+
+ // factor is the interleave factor
+ factor uint8
+
+ // domain is the domain to pass to LookupTactics
+ domain string
+
+ // port is the port to pass to LookupTactics
+ port string
+
+ // expect is the expectations in terms of tactics
+ expect []*httpsDialerTactic
+ }
+
+ // This is the list of tactics that we expect the primary
+ // policy to return when we're not using a null policy
+ expectedPrimaryTactics := []*httpsDialerTactic{{
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "shelob.polito.it",
+ VerifyHostname: "api.ooni.io",
+ }, {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "whitespider.polito.it",
+ VerifyHostname: "api.ooni.io",
+ }, {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "mirkwood.polito.it",
+ VerifyHostname: "api.ooni.io",
+ }, {
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "highgarden.polito.it",
+ VerifyHostname: "api.ooni.io",
+ }}
+
+ // Create the non-null primary policy
+ primary := &userPolicyV2{
+ Root: &userPolicyRoot{
+ DomainEndpoints: map[string][]*httpsDialerTactic{
+ "api.ooni.io:443": expectedPrimaryTactics,
+ },
+ Version: userPolicyVersion,
+ },
+ }
+
+ // This is the list of tactics that we expect the fallback
+ // policy to return when we're not using a null policy
+ expectedFallbackTactics := []*httpsDialerTactic{{
+ Address: "130.192.91.231",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "kingslanding.polito.it",
+ VerifyHostname: "api.ooni.io",
+ }, {
+ Address: "130.192.91.231",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "pyke.polito.it",
+ VerifyHostname: "api.ooni.io",
+ }, {
+ Address: "130.192.91.231",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "winterfell.polito.it",
+ VerifyHostname: "api.ooni.io",
+ }}
+
+ // Create the non-null fallback policy
+ fallback := &userPolicyV2{
+ Root: &userPolicyRoot{
+ DomainEndpoints: map[string][]*httpsDialerTactic{
+ "api.ooni.io:443": expectedFallbackTactics,
+ },
+ Version: userPolicyVersion,
+ },
+ }
+
+ cases := []testcase{
+
+ // This test ensures that the code is WAI with two null policies
+ {
+ name: "with two null policies",
+ primary: &nullPolicy{},
+ fallback: &nullPolicy{},
+ factor: 0,
+ domain: "api.ooni.io",
+ port: "443",
+ expect: nil,
+ },
+
+ // This test ensures that we get the content of the primary
+ // policy when the fallback policy is the null policy
+ {
+ name: "with the fallback policy being null",
+ primary: primary,
+ fallback: &nullPolicy{},
+ factor: 0,
+ domain: "api.ooni.io",
+ port: "443",
+ expect: expectedPrimaryTactics,
+ },
+
+ // This test ensures that we get the content of the fallback
+ // policy when the primary policy is the null policy
+ {
+ name: "with the primary policy being null",
+ primary: &nullPolicy{},
+ fallback: fallback,
+ factor: 0,
+ domain: "api.ooni.io",
+ port: "443",
+ expect: expectedFallbackTactics,
+ },
+
+ // This test ensures that we correctly interleave the tactics
+ {
+ name: "with both policies being nonnull and interleave being nonzero",
+ primary: primary,
+ fallback: fallback,
+ factor: 2,
+ domain: "api.ooni.io",
+ port: "443",
+ expect: []*httpsDialerTactic{
+ expectedPrimaryTactics[0],
+ expectedPrimaryTactics[1],
+ expectedFallbackTactics[0],
+ expectedFallbackTactics[1],
+ expectedPrimaryTactics[2],
+ expectedPrimaryTactics[3],
+ expectedFallbackTactics[2],
+ },
+ },
+
+ // This test ensures that we behave correctly when factor is zero
+ {
+ name: "with both policies being nonnull and interleave being zero",
+ primary: primary,
+ fallback: fallback,
+ factor: 0,
+ domain: "api.ooni.io",
+ port: "443",
+ expect: []*httpsDialerTactic{
+ expectedPrimaryTactics[0],
+ expectedFallbackTactics[0],
+ expectedPrimaryTactics[1],
+ expectedFallbackTactics[1],
+ expectedPrimaryTactics[2],
+ expectedFallbackTactics[2],
+ expectedPrimaryTactics[3],
+ },
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+
+ // construct the mixPolicyInterleave instance
+ p := &mixPolicyInterleave{
+ Primary: tc.primary,
+ Fallback: tc.fallback,
+ Factor: tc.factor,
+ }
+
+ // start looking up for tactics
+ outch := p.LookupTactics(context.Background(), tc.domain, tc.port)
+
+ // collect all the generated tactics
+ var got []*httpsDialerTactic
+ for entry := range outch {
+ got = append(got, entry)
+ }
+
+ // compare to expectations
+ if diff := cmp.Diff(tc.expect, got); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+ }
+}
diff --git a/pkg/enginenetx/mockspolicy_test.go b/pkg/enginenetx/mockspolicy_test.go
new file mode 100644
index 000000000..4fb6c49b1
--- /dev/null
+++ b/pkg/enginenetx/mockspolicy_test.go
@@ -0,0 +1,49 @@
+package enginenetx
+
+import (
+ "context"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/testingx"
+)
+
+// mocksPolicy is a mockable policy
+type mocksPolicy struct {
+ MockLookupTactics func(ctx context.Context, domain string, port string) <-chan *httpsDialerTactic
+}
+
+var _ httpsDialerPolicy = &mocksPolicy{}
+
+// LookupTactics implements httpsDialerPolicy.
+func (p *mocksPolicy) LookupTactics(ctx context.Context, domain string, port string) <-chan *httpsDialerTactic {
+ return p.MockLookupTactics(ctx, domain, port)
+}
+
+func TestMocksPolicy(t *testing.T) {
+ // create and fake fill a mocked tactic
+ var tx httpsDialerTactic
+ ff := &testingx.FakeFiller{}
+ ff.Fill(&tx)
+
+ // create a mocks policy
+ p := &mocksPolicy{
+ MockLookupTactics: func(ctx context.Context, domain, port string) <-chan *httpsDialerTactic {
+ output := make(chan *httpsDialerTactic, 1)
+ output <- &tx
+ close(output)
+ return output
+ },
+ }
+
+ // read the tactics emitted by the policy
+ var got []*httpsDialerTactic
+ for entry := range p.LookupTactics(context.Background(), "api.ooni.io", "443") {
+ got = append(got, entry)
+ }
+
+ // make sure we've got what we expect
+ if diff := cmp.Diff([]*httpsDialerTactic{&tx}, got); diff != "" {
+ t.Fatal(diff)
+ }
+}
diff --git a/pkg/enginenetx/network.go b/pkg/enginenetx/network.go
index 4f3e669b1..bdef9c679 100644
--- a/pkg/enginenetx/network.go
+++ b/pkg/enginenetx/network.go
@@ -93,7 +93,8 @@ func NewNetwork(
netx := &netxlite.Netx{}
dialer := netx.NewDialerWithResolver(logger, resolver)
- // Create manager for keeping track of statistics
+ // Create manager for keeping track of statistics. This implies creating a background
+ // goroutine that we'll need to close when we're done.
const trimInterval = 30 * time.Second
stats := newStatsManager(kvStore, logger, trimInterval)
@@ -112,21 +113,16 @@ func NewNetwork(
// Note that:
//
// - we're enabling compression, which is desiredable since this transport
- // is not made for measuring and compression is good(TM);
+ // is not made for measuring and compression is good(TM), also note that
+ // when the request uses Accept-Encoding, this kind of automatic management
+ // of compression is disabled, so there is no conflict.
//
// - if proxyURL is nil, the proxy option is equivalent to disabling
// the proxy, otherwise it means that we're using the ooni/oohttp library
// to dial for proxies, which has some restrictions.
//
- // In particular, the returned transport uses dialer for dialing with
- // cleartext proxies (e.g., socks5 and http) and httpsDialer for dialing
- // with encrypted proxies (e.g., https). After this has happened,
- // the code currently falls back to using the standard library's tls
- // client code for establishing TLS connections over the proxy. The main
- // implication here is that we're not using our custom mozilla CA for
- // validating TLS certificates, rather we're using the system's cert store.
- //
- // Fixing this issue is TODO(https://github.com/ooni/probe/issues/2536).
+ // - this code does not work as intended when using netem and proxies
+ // as documented by TODO(https://github.com/ooni/probe/issues/2536).
txp := netxlite.NewHTTPTransportWithOptions(
logger, dialer, httpsDialer,
netxlite.HTTPTransportOptionDisableCompression(false),
@@ -158,16 +154,41 @@ func newHTTPSDialerPolicy(
return &dnsPolicy{logger, resolver}
}
- // create a composed fallback TLS dialer policy
- fallback := &statsPolicy{
- Fallback: &bridgesPolicy{Fallback: &dnsPolicy{logger, resolver}},
- Stats: stats,
+ // create a policy interleaving stats policies and bridges policies
+ statsOrBridges := &mixPolicyInterleave{
+ Primary: &statsPolicyV2{
+ Stats: stats,
+ },
+ Fallback: &bridgesPolicyV2{},
+ Factor: 3,
+ }
+
+ // wrap the DNS policy with a policy that extends tactics for test
+ // helpers so that we also try using different SNIs.
+ dnsExt := &testHelpersPolicy{
+ Child: &dnsPolicy{logger, resolver},
+ }
+
+ // compose dnsExt and statsOrBridges such that dnsExt has
+ // priority in the selection of tactics
+ composed := &mixPolicyInterleave{
+ Primary: dnsExt,
+ Fallback: statsOrBridges,
+ Factor: 3,
}
- // make sure we honor a user-provided policy
- policy, err := newUserPolicy(kvStore, fallback)
+ // attempt to load a user-provided dialing policy
+ primary, err := newUserPolicyV2(kvStore)
+
+ // on error, just use composed
if err != nil {
- return fallback
+ return composed
+ }
+
+ // otherwise, finish creating the dialing policy
+ policy := &mixPolicyEitherOr{
+ Primary: primary,
+ Fallback: composed,
}
return policy
diff --git a/pkg/enginenetx/network_internal_test.go b/pkg/enginenetx/network_internal_test.go
index 0325e2950..b4198bf39 100644
--- a/pkg/enginenetx/network_internal_test.go
+++ b/pkg/enginenetx/network_internal_test.go
@@ -7,6 +7,7 @@ import (
"net/url"
"sync"
"testing"
+ "time"
"github.com/apex/log"
"github.com/google/go-cmp/cmp"
@@ -214,7 +215,7 @@ func TestNetworkUnit(t *testing.T) {
}
// Make sure we get the correct policy type depending on how we call newHTTPSDialerPolicy
-func TestNewHTTPSDialerPolicy(t *testing.T) {
+func TestNewHTTPSDialerPolicyTypes(t *testing.T) {
// testcase is a test case implemented by this function
type testcase struct {
// name is the name of the test case
@@ -229,53 +230,102 @@ func TestNewHTTPSDialerPolicy(t *testing.T) {
// expectType is the string representation of the
// type constructed using these params
expectType string
+
+ // extraChecks is an OPTIONAL function that
+ // will perform extra checks on the policy type
+ extraChecks func(t *testing.T, root httpsDialerPolicy)
}
minimalUserPolicy := []byte(`{"Version":3}`)
- cases := []testcase{{
- name: "when there is a proxy URL and there is a user policy",
- kvStore: func() model.KeyValueStore {
- store := &kvstore.Memory{}
- // this policy is mostly empty but it's enough to load
- runtimex.Try0(store.Set(userPolicyKey, minimalUserPolicy))
- return store
- },
- proxyURL: &url.URL{
- Scheme: "socks5",
- Host: "127.0.0.1:9050",
- Path: "/",
- },
- expectType: "*enginenetx.dnsPolicy",
- }, {
- name: "when there is a proxy URL and there is no user policy",
- kvStore: func() model.KeyValueStore {
- return &kvstore.Memory{}
+ // this function ensures that the part dealing with stats or bridges is correct
+ verifyStatsOrBridgesChain := func(t *testing.T, root *mixPolicyInterleave) {
+ if root.Factor != 3 {
+ t.Fatal("expected .Factory to be 3")
+ }
+ _ = root.Primary.(*statsPolicyV2)
+ _ = root.Fallback.(*bridgesPolicyV2)
+
+ }
+
+ // this function ensures that the DNS ext part of the chain is correct
+ verifyDNSExtChain := func(_ *testing.T, root *testHelpersPolicy) {
+ _ = root.Child.(*dnsPolicy)
+ }
+
+ // this function ensures that the policy used when there's no use policy has
+ // the correct type and anything below it also has the correct type
+ verifyNoUserPolicyChain := func(t *testing.T, root httpsDialerPolicy) {
+ interleavePolicy := root.(*mixPolicyInterleave)
+ if interleavePolicy.Factor != 3 {
+ t.Fatal("expected .Factory to be 3")
+ }
+ verifyDNSExtChain(t, interleavePolicy.Primary.(*testHelpersPolicy))
+ verifyStatsOrBridgesChain(t, interleavePolicy.Fallback.(*mixPolicyInterleave))
+
+ }
+
+ // this function ansures that the policy used when there's an user policy has
+ // the correct type and anything below it also has the correct type
+ verifyUserPolicyChain := func(t *testing.T, root httpsDialerPolicy) {
+ eitherOrPolicy := root.(*mixPolicyEitherOr)
+ _ = eitherOrPolicy.Primary.(*userPolicyV2)
+ verifyNoUserPolicyChain(t, eitherOrPolicy.Fallback)
+ }
+
+ cases := []testcase{
+ {
+ name: "when there is a proxy URL and there is a user policy",
+ kvStore: func() model.KeyValueStore {
+ store := &kvstore.Memory{}
+ // this policy is mostly empty but it's enough to load
+ runtimex.Try0(store.Set(userPolicyKey, minimalUserPolicy))
+ return store
+ },
+ proxyURL: &url.URL{
+ Scheme: "socks5",
+ Host: "127.0.0.1:9050",
+ Path: "/",
+ },
+ expectType: "*enginenetx.dnsPolicy",
},
- proxyURL: &url.URL{
- Scheme: "socks5",
- Host: "127.0.0.1:9050",
- Path: "/",
+
+ {
+ name: "when there is a proxy URL and there is no user policy",
+ kvStore: func() model.KeyValueStore {
+ return &kvstore.Memory{}
+ },
+ proxyURL: &url.URL{
+ Scheme: "socks5",
+ Host: "127.0.0.1:9050",
+ Path: "/",
+ },
+ expectType: "*enginenetx.dnsPolicy",
},
- expectType: "*enginenetx.dnsPolicy",
- }, {
- name: "when there is no proxy URL and there is a user policy",
- kvStore: func() model.KeyValueStore {
- store := &kvstore.Memory{}
- // this policy is mostly empty but it's enough to load
- runtimex.Try0(store.Set(userPolicyKey, minimalUserPolicy))
- return store
+
+ {
+ name: "when there is no proxy URL and there is a user policy",
+ kvStore: func() model.KeyValueStore {
+ store := &kvstore.Memory{}
+ // this policy is mostly empty but it's enough to load
+ runtimex.Try0(store.Set(userPolicyKey, minimalUserPolicy))
+ return store
+ },
+ proxyURL: nil,
+ expectType: "*enginenetx.mixPolicyEitherOr",
+ extraChecks: verifyUserPolicyChain,
},
- proxyURL: nil,
- expectType: "*enginenetx.userPolicy",
- }, {
- name: "when there is no proxy URL and there is no user policy",
- kvStore: func() model.KeyValueStore {
- return &kvstore.Memory{}
+
+ {
+ name: "when there is no proxy URL and there is no user policy",
+ kvStore: func() model.KeyValueStore {
+ return &kvstore.Memory{}
+ },
+ proxyURL: nil,
+ expectType: "*enginenetx.mixPolicyInterleave",
+ extraChecks: verifyNoUserPolicyChain,
},
- proxyURL: nil,
- expectType: "*enginenetx.statsPolicy",
- }}
+ }
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
@@ -292,6 +342,363 @@ func TestNewHTTPSDialerPolicy(t *testing.T) {
if diff := cmp.Diff(tc.expectType, got); diff != "" {
t.Fatal(diff)
}
+
+ if tc.extraChecks != nil {
+ tc.extraChecks(t, p)
+ }
+ })
+ }
+}
+
+// This test ensures that newHTTPSDialerPolicy is functionally working as intended.
+func TestNewHTTPSDialerPolicyFunctional(t *testing.T) {
+ // testcase is a test case implemented by this func
+ type testcase struct {
+ // name is the test case name
+ name string
+
+ // kvStore is the key-value store possibly containing user policies
+ // and previous statistics about TLS endpoints
+ kvStore func() model.KeyValueStore
+
+ // proxyURL is the OPTIONAL proxy URL.
+ proxyURL *url.URL
+
+ // resolver is the DNS resolver.
+ resolver model.Resolver
+
+ // domain is the domain for which to use LookupTactics.
+ domain string
+
+ // totalExpectedEntries is the total number of entries we
+ // expect the code to generate as part of this run
+ totalExpectedEntries int
+
+ // initialExpectedEntries contains the initial entries that
+ // we expect to see when getting results
+ initialExpectedEntries []*httpsDialerTactic
+ }
+
+ cases := []testcase{
+
+ // Let's start with test cases in which there is no proxy and
+ // no state, where we want to see that we're using the DNS, and
+ // that, on top of this, we're getting bridges tactics when
+ // we're using api.ooni.io and we're getting various SNIs when
+ // instead we're using test helper domains.
+
+ {
+ name: "without proxy, with empty key-value store, and NXDOMAIN for www.example.com",
+ kvStore: func() model.KeyValueStore {
+ return &kvstore.Memory{}
+ },
+ proxyURL: nil,
+ resolver: &mocks.Resolver{
+ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
+ return nil, netxlite.ErrOODNSNoSuchHost
+ },
+ },
+ domain: "www.example.com",
+ totalExpectedEntries: 0,
+ initialExpectedEntries: nil,
+ },
+
+ {
+ name: "without proxy, with empty key-value store, and addresses for www.example.com",
+ kvStore: func() model.KeyValueStore {
+ return &kvstore.Memory{}
+ },
+ proxyURL: nil,
+ resolver: &mocks.Resolver{
+ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
+ return []string{"93.184.215.14", "2606:2800:21f:cb07:6820:80da:af6b:8b2c"}, nil
+ },
+ },
+ domain: "www.example.com",
+ totalExpectedEntries: 2,
+ initialExpectedEntries: []*httpsDialerTactic{{
+ Address: "93.184.215.14",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "www.example.com",
+ VerifyHostname: "www.example.com",
+ }, {
+ Address: "2606:2800:21f:cb07:6820:80da:af6b:8b2c",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "www.example.com",
+ VerifyHostname: "www.example.com",
+ }},
+ },
+
+ {
+ name: "without proxy, with empty key-value store, and NXDOMAIN for api.ooni.io",
+ kvStore: func() model.KeyValueStore {
+ return &kvstore.Memory{}
+ },
+ proxyURL: nil,
+ resolver: &mocks.Resolver{
+ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
+ return nil, netxlite.ErrOODNSNoSuchHost
+ },
+ },
+ domain: "api.ooni.io",
+ totalExpectedEntries: 152,
+ initialExpectedEntries: nil,
+ },
+
+ {
+ name: "without proxy, with empty key-value store, and addresses for api.ooni.io",
+ kvStore: func() model.KeyValueStore {
+ return &kvstore.Memory{}
+ },
+ proxyURL: nil,
+ resolver: &mocks.Resolver{
+ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
+ return []string{"130.192.91.211", "130.192.91.231"}, nil
+ },
+ },
+ domain: "api.ooni.io",
+ totalExpectedEntries: 154,
+ initialExpectedEntries: []*httpsDialerTactic{{
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "api.ooni.io",
+ VerifyHostname: "api.ooni.io",
+ }, {
+ Address: "130.192.91.231",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "api.ooni.io",
+ VerifyHostname: "api.ooni.io",
+ }},
+ },
+
+ {
+ name: "without proxy, with empty key-value store, and NXDOMAIN for 0.th.ooni.org",
+ kvStore: func() model.KeyValueStore {
+ return &kvstore.Memory{}
+ },
+ proxyURL: nil,
+ resolver: &mocks.Resolver{
+ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
+ return nil, netxlite.ErrOODNSNoSuchHost
+ },
+ },
+ domain: "0.th.ooni.org",
+ totalExpectedEntries: 0,
+ initialExpectedEntries: nil,
+ },
+
+ {
+ name: "without proxy, with empty key-value store, and addresses for 0.th.ooni.org",
+ kvStore: func() model.KeyValueStore {
+ return &kvstore.Memory{}
+ },
+ proxyURL: nil,
+ resolver: &mocks.Resolver{
+ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
+ return []string{"130.192.91.211", "130.192.91.231"}, nil
+ },
+ },
+ domain: "0.th.ooni.org",
+ totalExpectedEntries: 306,
+ initialExpectedEntries: []*httpsDialerTactic{{
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "0.th.ooni.org",
+ VerifyHostname: "0.th.ooni.org",
+ }, {
+ Address: "130.192.91.231",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "0.th.ooni.org",
+ VerifyHostname: "0.th.ooni.org",
+ }},
+ },
+
+ // Now we repeat the same test cases but with a proxy and we want
+ // to always and only see the results obtained via DNS.
+
+ {
+ name: "with proxy, with empty key-value store, and NXDOMAIN for www.example.com",
+ kvStore: func() model.KeyValueStore {
+ return &kvstore.Memory{}
+ },
+ proxyURL: &url.URL{}, // does not need to be filled
+ resolver: &mocks.Resolver{
+ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
+ return nil, netxlite.ErrOODNSNoSuchHost
+ },
+ },
+ domain: "www.example.com",
+ totalExpectedEntries: 0,
+ initialExpectedEntries: nil,
+ },
+
+ {
+ name: "with proxy, with empty key-value store, and addresses for www.example.com",
+ kvStore: func() model.KeyValueStore {
+ return &kvstore.Memory{}
+ },
+ proxyURL: &url.URL{}, // does not need to be filled
+ resolver: &mocks.Resolver{
+ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
+ return []string{"93.184.215.14", "2606:2800:21f:cb07:6820:80da:af6b:8b2c"}, nil
+ },
+ },
+ domain: "www.example.com",
+ totalExpectedEntries: 2,
+ initialExpectedEntries: []*httpsDialerTactic{{
+ Address: "93.184.215.14",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "www.example.com",
+ VerifyHostname: "www.example.com",
+ }, {
+ Address: "2606:2800:21f:cb07:6820:80da:af6b:8b2c",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "www.example.com",
+ VerifyHostname: "www.example.com",
+ }},
+ },
+
+ {
+ name: "with proxy, with empty key-value store, and NXDOMAIN for api.ooni.io",
+ kvStore: func() model.KeyValueStore {
+ return &kvstore.Memory{}
+ },
+ proxyURL: &url.URL{}, // does not need to be filled
+ resolver: &mocks.Resolver{
+ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
+ return nil, netxlite.ErrOODNSNoSuchHost
+ },
+ },
+ domain: "api.ooni.io",
+ totalExpectedEntries: 0,
+ initialExpectedEntries: nil,
+ },
+
+ {
+ name: "without proxy, with empty key-value store, and addresses for api.ooni.io",
+ kvStore: func() model.KeyValueStore {
+ return &kvstore.Memory{}
+ },
+ proxyURL: &url.URL{}, // does not need to be filled
+ resolver: &mocks.Resolver{
+ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
+ return []string{"130.192.91.211", "130.192.91.231"}, nil
+ },
+ },
+ domain: "api.ooni.io",
+ totalExpectedEntries: 2,
+ initialExpectedEntries: []*httpsDialerTactic{{
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "api.ooni.io",
+ VerifyHostname: "api.ooni.io",
+ }, {
+ Address: "130.192.91.231",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "api.ooni.io",
+ VerifyHostname: "api.ooni.io",
+ }},
+ },
+
+ {
+ name: "with proxy, with empty key-value store, and NXDOMAIN for 0.th.ooni.org",
+ kvStore: func() model.KeyValueStore {
+ return &kvstore.Memory{}
+ },
+ proxyURL: &url.URL{}, // does not need to be filled
+ resolver: &mocks.Resolver{
+ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
+ return nil, netxlite.ErrOODNSNoSuchHost
+ },
+ },
+ domain: "0.th.ooni.org",
+ totalExpectedEntries: 0,
+ initialExpectedEntries: nil,
+ },
+
+ {
+ name: "with proxy, with empty key-value store, and addresses for 0.th.ooni.org",
+ kvStore: func() model.KeyValueStore {
+ return &kvstore.Memory{}
+ },
+ proxyURL: &url.URL{}, // does not need to be filled
+ resolver: &mocks.Resolver{
+ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
+ return []string{"130.192.91.211", "130.192.91.231"}, nil
+ },
+ },
+ domain: "0.th.ooni.org",
+ totalExpectedEntries: 2,
+ initialExpectedEntries: []*httpsDialerTactic{{
+ Address: "130.192.91.211",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "0.th.ooni.org",
+ VerifyHostname: "0.th.ooni.org",
+ }, {
+ Address: "130.192.91.231",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "0.th.ooni.org",
+ VerifyHostname: "0.th.ooni.org",
+ }},
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ // Create manager for keeping track of statistics. This implies creating a background
+ // goroutine that we'll need to close when we're done.
+ stats := newStatsManager(tc.kvStore(), model.DiscardLogger, 24*time.Hour)
+ defer stats.Close()
+
+ // Create a new HTTPS dialer policy.
+ policy := newHTTPSDialerPolicy(
+ tc.kvStore(),
+ model.DiscardLogger,
+ tc.proxyURL, // possibly nil
+ tc.resolver,
+ stats,
+ )
+
+ // Start to generate tactics for the given domain and port.
+ generator := policy.LookupTactics(context.Background(), tc.domain, "443")
+
+ // Collect tactics
+ var tactics []*httpsDialerTactic
+ for entry := range generator {
+ tactics = append(tactics, entry)
+ }
+
+ // To help debugging, log how many tactics we've got
+ t.Log("got", len(tactics), "tactics")
+
+ // make sure the number of expected entries is the actual number
+ if len(tactics) != tc.totalExpectedEntries {
+ t.Fatal("expected", tc.totalExpectedEntries, ", got", len(tactics))
+ }
+
+ // make sure we have at least N initial entries
+ if len(tactics) < len(tc.initialExpectedEntries) {
+ t.Fatal("expected at least", len(tc.initialExpectedEntries), "tactics, got", len(tactics))
+ }
+
+ // if we have expected initial entries, make sure they match
+ if len(tc.initialExpectedEntries) > 0 {
+ if diff := cmp.Diff(tc.initialExpectedEntries, tactics[:len(tc.initialExpectedEntries)]); diff != "" {
+ t.Fatal(diff)
+ }
+ }
})
}
}
diff --git a/pkg/enginenetx/nullpolicy.go b/pkg/enginenetx/nullpolicy.go
new file mode 100644
index 000000000..44424991b
--- /dev/null
+++ b/pkg/enginenetx/nullpolicy.go
@@ -0,0 +1,27 @@
+package enginenetx
+
+//
+// A policy that never returns any tactic.
+//
+
+import "context"
+
+// nullPolicy is a policy that never returns any tactics.
+//
+// You can use this policy to terminate the policy chain and
+// ensure ane existing policy has a "null" fallback.
+//
+// The zero value is ready to use.
+type nullPolicy struct{}
+
+var _ httpsDialerPolicy = &nullPolicy{}
+
+// LookupTactics implements httpsDialerPolicy.
+//
+// This policy returns a closed channel such that it won't
+// be possible to read policies from it.
+func (n *nullPolicy) LookupTactics(ctx context.Context, domain string, port string) <-chan *httpsDialerTactic {
+ output := make(chan *httpsDialerTactic)
+ close(output)
+ return output
+}
diff --git a/pkg/enginenetx/nullpolicy_test.go b/pkg/enginenetx/nullpolicy_test.go
new file mode 100644
index 000000000..0bb40ec0d
--- /dev/null
+++ b/pkg/enginenetx/nullpolicy_test.go
@@ -0,0 +1,17 @@
+package enginenetx
+
+import (
+ "context"
+ "testing"
+)
+
+func TestNullPolicy(t *testing.T) {
+ p := &nullPolicy{}
+ var count int
+ for range p.LookupTactics(context.Background(), "api.ooni.io", "443") {
+ count++
+ }
+ if count != 0 {
+ t.Fatal("should have not returned any policy")
+ }
+}
diff --git a/pkg/enginenetx/statsmanager.go b/pkg/enginenetx/statsmanager.go
index d1e9a802c..b4507cb63 100644
--- a/pkg/enginenetx/statsmanager.go
+++ b/pkg/enginenetx/statsmanager.go
@@ -137,6 +137,8 @@ func statsDefensivelySortTacticsByDescendingSuccessRateWithAcceptPredicate(
input []*statsTactic, acceptfunc func(*statsTactic) bool) []*statsTactic {
// first let's create a working list such that we don't modify
// the input in place thus avoiding any data race
+ //
+ // make sure we explicitly filter out malformed entries
work := []*statsTactic{}
for _, t := range input {
if t != nil && t.Tactic != nil {
@@ -193,8 +195,8 @@ func (st *statsTactic) Clone() *statsTactic {
// a pointer to a location which is typically immutable, so it's perfectly
// fine to copy the LastUpdate field by assignment.
//
- // here we're using a bunch of robustness aware mechanisms to clone
- // considering that the struct may be edited by the user
+ // here we're using safe functions to clone the original struct considering
+ // that a user can edit the content on disk freely introducing nulls.
return &statsTactic{
CountStarted: st.CountStarted,
CountTCPConnectError: st.CountTCPConnectError,
diff --git a/pkg/enginenetx/statspolicy.go b/pkg/enginenetx/statspolicy.go
index 68a708f6e..bd34a3cee 100644
--- a/pkg/enginenetx/statspolicy.go
+++ b/pkg/enginenetx/statspolicy.go
@@ -12,69 +12,28 @@ import (
"github.com/ooni/probe-engine/pkg/runtimex"
)
-// statsPolicy is a policy that schedules tactics already known
-// to work based on statistics and defers to a fallback policy
-// once it has generated all the tactics known to work.
+// statsPolicyV2 is a policy that schedules tactics already known
+// to work based on the previously collected stats.
//
-// The zero value of this struct is invalid; please, make sure you
-// fill all the fields marked as MANDATORY.
-type statsPolicy struct {
- // Fallback is the MANDATORY fallback policy.
- Fallback httpsDialerPolicy
-
+// The zero value of this struct is invalid; please, make sure
+// you fill all the fields marked as MANDATORY.
+//
+// This is v2 of the statsPolicy because the previous implementation
+// incorporated mixing logic, while now the mixing happens outside
+// of this policy, thus giving us much more flexibility.
+type statsPolicyV2 struct {
// Stats is the MANDATORY stats manager.
Stats *statsManager
}
-var _ httpsDialerPolicy = &statsPolicy{}
-
-// LookupTactics implements HTTPSDialerPolicy.
-func (p *statsPolicy) LookupTactics(ctx context.Context, domain string, port string) <-chan *httpsDialerTactic {
- out := make(chan *httpsDialerTactic)
-
- go func() {
- defer close(out) // make sure the parent knows when we're done
- index := 0
-
- // useful to make sure we don't emit two equal policy in a single run
- uniq := make(map[string]int)
-
- // function that emits a given tactic unless we already emitted it
- maybeEmitTactic := func(t *httpsDialerTactic) {
- // as a safety mechanism let's gracefully handle the
- // case in which the tactic is nil
- if t == nil {
- return
- }
-
- // handle the case in which we already emitted a policy
- key := t.tacticSummaryKey()
- if uniq[key] > 0 {
- return
- }
- uniq[key]++
-
- // 🚀!!!
- t.InitialDelay = happyEyeballsDelay(index)
- index += 1
- out <- t
- }
-
- // give priority to what we know from stats
- for _, t := range statsPolicyPostProcessTactics(p.Stats.LookupTactics(domain, port)) {
- maybeEmitTactic(t)
- }
-
- // fallback to the secondary policy
- for t := range p.Fallback.LookupTactics(ctx, domain, port) {
- maybeEmitTactic(t)
- }
- }()
+var _ httpsDialerPolicy = &statsPolicyV2{}
- return out
+// LookupTactics implements httpsDialerPolicy.
+func (p *statsPolicyV2) LookupTactics(ctx context.Context, domain string, port string) <-chan *httpsDialerTactic {
+ return streamTacticsFromSlice(statsPolicyFilterStatsTactics(p.Stats.LookupTactics(domain, port)))
}
-func statsPolicyPostProcessTactics(tactics []*statsTactic, good bool) (out []*httpsDialerTactic) {
+func statsPolicyFilterStatsTactics(tactics []*statsTactic, good bool) (out []*httpsDialerTactic) {
// when good is false, it means p.Stats.LookupTactics failed
if !good {
return
diff --git a/pkg/enginenetx/statspolicy_test.go b/pkg/enginenetx/statspolicy_test.go
index 959152536..1945b5c5d 100644
--- a/pkg/enginenetx/statspolicy_test.go
+++ b/pkg/enginenetx/statspolicy_test.go
@@ -9,13 +9,11 @@ import (
"github.com/apex/log"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-engine/pkg/kvstore"
- "github.com/ooni/probe-engine/pkg/mocks"
"github.com/ooni/probe-engine/pkg/netemx"
- "github.com/ooni/probe-engine/pkg/netxlite"
"github.com/ooni/probe-engine/pkg/runtimex"
)
-func TestStatsPolicyWorkingAsIntended(t *testing.T) {
+func TestStatsPolicyV2(t *testing.T) {
// prepare the content of the stats
twentyMinutesAgo := time.Now().Add(-20 * time.Minute)
@@ -138,26 +136,13 @@ func TestStatsPolicyWorkingAsIntended(t *testing.T) {
return newStatsManager(kvStore, log.Log, trimInterval)
}
- t.Run("when we have unique statistics", func(t *testing.T) {
+ t.Run("when we have relevant stats", func(t *testing.T) {
// create stats manager
stats := createStatsManager("api.ooni.io:443", expectTacticsStats...)
defer stats.Close()
- // create the composed policy
- policy := &statsPolicy{
- Fallback: &dnsPolicy{
- Logger: log.Log,
- Resolver: &mocks.Resolver{
- MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
- switch domain {
- case "api.ooni.io":
- return []string{bridgeAddress}, nil
- default:
- return nil, netxlite.ErrOODNSNoSuchHost
- }
- },
- },
- },
+ // create the policy
+ policy := &statsPolicyV2{
Stats: stats,
}
@@ -169,60 +154,28 @@ func TestStatsPolicyWorkingAsIntended(t *testing.T) {
// compute the list of results we expect to see from the stats data
var expect []*httpsDialerTactic
- idx := 0
for _, entry := range expectTacticsStats {
if entry.CountSuccess <= 0 || entry.Tactic == nil {
continue // we SHOULD NOT include entries that systematically failed
}
t := entry.Tactic.Clone()
- t.InitialDelay = happyEyeballsDelay(idx)
+ t.InitialDelay = 0
expect = append(expect, t)
- idx++
}
- // extend the expected list to include DNS results
- expect = append(expect, &httpsDialerTactic{
- Address: bridgeAddress,
- InitialDelay: 2 * time.Second,
- Port: "443",
- SNI: "api.ooni.io",
- VerifyHostname: "api.ooni.io",
- })
-
// perform the actual comparison
if diff := cmp.Diff(expect, tactics); diff != "" {
t.Fatal(diff)
}
})
- t.Run("when we have duplicates", func(t *testing.T) {
- // add each entry twice to create obvious duplicates
- statsWithDupes := []*statsTactic{}
- for _, entry := range expectTacticsStats {
- statsWithDupes = append(statsWithDupes, entry.Clone())
- statsWithDupes = append(statsWithDupes, entry.Clone())
- }
-
+ t.Run("when there are no relevant stats", func(t *testing.T) {
// create stats manager
- stats := createStatsManager("api.ooni.io:443", statsWithDupes...)
+ stats := createStatsManager("api.ooni.io:443" /*, nothing */)
defer stats.Close()
- // create the composed policy
- policy := &statsPolicy{
- Fallback: &dnsPolicy{
- Logger: log.Log,
- Resolver: &mocks.Resolver{
- MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
- switch domain {
- case "api.ooni.io":
- // Twice so we try to cause duplicate entries also with the DNS policy
- return []string{bridgeAddress, bridgeAddress}, nil
- default:
- return nil, netxlite.ErrOODNSNoSuchHost
- }
- },
- },
- },
+ // create the policy
+ policy := &statsPolicyV2{
Stats: stats,
}
@@ -234,94 +187,17 @@ func TestStatsPolicyWorkingAsIntended(t *testing.T) {
// compute the list of results we expect to see from the stats data
var expect []*httpsDialerTactic
- idx := 0
- for _, entry := range expectTacticsStats {
- if entry.CountSuccess <= 0 || entry.Tactic == nil {
- continue // we SHOULD NOT include entries that systematically failed
- }
- t := entry.Tactic.Clone()
- t.InitialDelay = happyEyeballsDelay(idx)
- expect = append(expect, t)
- idx++
- }
-
- // extend the expected list to include DNS results
- expect = append(expect, &httpsDialerTactic{
- Address: bridgeAddress,
- InitialDelay: 2 * time.Second,
- Port: "443",
- SNI: "api.ooni.io",
- VerifyHostname: "api.ooni.io",
- })
// perform the actual comparison
if diff := cmp.Diff(expect, tactics); diff != "" {
t.Fatal(diff)
}
})
-
- t.Run("we avoid manipulating nil tactics", func(t *testing.T) {
- // create stats manager
- stats := createStatsManager("api.ooni.io:443", expectTacticsStats...)
- defer stats.Close()
-
- // create the composed policy
- policy := &statsPolicy{
- Fallback: &mocksPolicy{
- MockLookupTactics: func(ctx context.Context, domain, port string) <-chan *httpsDialerTactic {
- out := make(chan *httpsDialerTactic)
- go func() {
- defer close(out)
-
- // explicitly send nil on the channel
- out <- nil
- }()
- return out
- },
- },
- Stats: stats,
- }
-
- // obtain the tactics from the saved stats
- var tactics []*httpsDialerTactic
- for entry := range policy.LookupTactics(context.Background(), "api.ooni.io", "443") {
- tactics = append(tactics, entry)
- }
-
- // compute the list of results we expect to see from the stats data
- var expect []*httpsDialerTactic
- idx := 0
- for _, entry := range expectTacticsStats {
- if entry.CountSuccess <= 0 || entry.Tactic == nil {
- continue // we SHOULD NOT include entries that systematically failed
- }
- t := entry.Tactic.Clone()
- t.InitialDelay = happyEyeballsDelay(idx)
- expect = append(expect, t)
- idx++
- }
-
- // perform the actual comparison
- if diff := cmp.Diff(expect, tactics); diff != "" {
- t.Fatal(diff)
- }
- })
-}
-
-type mocksPolicy struct {
- MockLookupTactics func(ctx context.Context, domain string, port string) <-chan *httpsDialerTactic
-}
-
-var _ httpsDialerPolicy = &mocksPolicy{}
-
-// LookupTactics implements httpsDialerPolicy.
-func (p *mocksPolicy) LookupTactics(ctx context.Context, domain string, port string) <-chan *httpsDialerTactic {
- return p.MockLookupTactics(ctx, domain, port)
}
-func TestStatsPolicyPostProcessTactics(t *testing.T) {
+func TestStatsPolicyFilterStatsTactics(t *testing.T) {
t.Run("we do nothing when good is false", func(t *testing.T) {
- tactics := statsPolicyPostProcessTactics(nil, false)
+ tactics := statsPolicyFilterStatsTactics(nil, false)
if len(tactics) != 0 {
t.Fatal("expected zero-lenght return value")
}
@@ -390,7 +266,7 @@ func TestStatsPolicyPostProcessTactics(t *testing.T) {
},
}
- got := statsPolicyPostProcessTactics(input, true)
+ got := statsPolicyFilterStatsTactics(input, true)
if len(got) != 1 {
t.Fatal("expected just one element")
diff --git a/pkg/enginenetx/stream.go b/pkg/enginenetx/stream.go
new file mode 100644
index 000000000..53b848474
--- /dev/null
+++ b/pkg/enginenetx/stream.go
@@ -0,0 +1,16 @@
+package enginenetx
+
+// streamTacticsFromSlice streams tactics from a given slice.
+//
+// This function returns a channel where we emit the edited
+// tactics, and which we clone when we're done.
+func streamTacticsFromSlice(input []*httpsDialerTactic) <-chan *httpsDialerTactic {
+ output := make(chan *httpsDialerTactic)
+ go func() {
+ defer close(output)
+ for _, tx := range input {
+ output <- tx
+ }
+ }()
+ return output
+}
diff --git a/pkg/enginenetx/stream_test.go b/pkg/enginenetx/stream_test.go
new file mode 100644
index 000000000..540ec02f9
--- /dev/null
+++ b/pkg/enginenetx/stream_test.go
@@ -0,0 +1,23 @@
+package enginenetx
+
+import (
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/testingx"
+)
+
+func TestStreamTacticsFromSlice(t *testing.T) {
+ input := []*httpsDialerTactic{}
+ ff := &testingx.FakeFiller{}
+ ff.Fill(&input)
+
+ var output []*httpsDialerTactic
+ for tx := range streamTacticsFromSlice(input) {
+ output = append(output, tx)
+ }
+
+ if diff := cmp.Diff(input, output); diff != "" {
+ t.Fatal(diff)
+ }
+}
diff --git a/pkg/enginenetx/testhelperspolicy.go b/pkg/enginenetx/testhelperspolicy.go
new file mode 100644
index 000000000..482307ead
--- /dev/null
+++ b/pkg/enginenetx/testhelperspolicy.go
@@ -0,0 +1,71 @@
+package enginenetx
+
+import (
+ "context"
+ "slices"
+)
+
+// testHelpersDomains is our understanding of TH domains.
+var testHelpersDomains = []string{
+ "0.th.ooni.org",
+ "1.th.ooni.org",
+ "2.th.ooni.org",
+ "3.th.ooni.org",
+ "d33d1gs9kpq1c5.cloudfront.net",
+}
+
+// testHelpersPolicy is a policy where we extend TH related policies
+// by adding additional SNIs that it makes sense to try.
+//
+// The zero value is invalid; please, init MANDATORY fields.
+type testHelpersPolicy struct {
+ // Child is the MANDATORY child policy.
+ Child httpsDialerPolicy
+}
+
+var _ httpsDialerPolicy = &testHelpersPolicy{}
+
+// LookupTactics implements httpsDialerPolicy.
+func (p *testHelpersPolicy) LookupTactics(ctx context.Context, domain, port string) <-chan *httpsDialerTactic {
+ out := make(chan *httpsDialerTactic)
+
+ go func() {
+ // tell the parent when we're done
+ defer close(out)
+
+ // collect tactics that we may want to modify later
+ var todo []*httpsDialerTactic
+
+ // always emit the original tactic first
+ //
+ // See https://github.com/ooni/probe-cli/pull/1552 review for
+ // a rationale of why we're emitting the original first
+ for tactic := range p.Child.LookupTactics(ctx, domain, port) {
+ out <- tactic
+
+ // When we're not connecting to a TH, our job is done
+ if !slices.Contains(testHelpersDomains, tactic.VerifyHostname) {
+ continue
+ }
+
+ // otherwise, let's rememeber to modify this later
+ todo = append(todo, tactic)
+ }
+
+ // This is the case where we're connecting to a test helper. Let's try
+ // to produce tactics using different SNIs for the domain.
+ for _, tactic := range todo {
+ for _, sni := range bridgesDomainsInRandomOrder() {
+ out <- &httpsDialerTactic{
+ Address: tactic.Address,
+ InitialDelay: 0, // set when dialing
+ Port: tactic.Port,
+ SNI: sni,
+ VerifyHostname: tactic.VerifyHostname,
+ }
+ }
+ }
+ }()
+
+ return out
+}
diff --git a/pkg/enginenetx/testhelperspolicy_test.go b/pkg/enginenetx/testhelperspolicy_test.go
new file mode 100644
index 000000000..964f06e8b
--- /dev/null
+++ b/pkg/enginenetx/testhelperspolicy_test.go
@@ -0,0 +1,144 @@
+package enginenetx
+
+import (
+ "context"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+)
+
+func TestTestHelpersPolicy(t *testing.T) {
+
+ // testHelperTactics contains tactics related to test helpers
+ testHelperTactics := []*httpsDialerTactic{{
+ Address: "18.195.190.71",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "0.th.ooni.org",
+ VerifyHostname: "0.th.ooni.org",
+ }, {
+ Address: "18.198.214.127",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "0.th.ooni.org",
+ VerifyHostname: "0.th.ooni.org",
+ }}
+
+ // wwwExampleComTactics contains tactics related to www.example.com
+ wwwExampleComTactic := []*httpsDialerTactic{{
+ Address: "93.184.215.14",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "www.example.com",
+ VerifyHostname: "www.example.com",
+ }, {
+ Address: "2606:2800:21f:cb07:6820:80da:af6b:8b2c",
+ InitialDelay: 0,
+ Port: "443",
+ SNI: "www.example.com",
+ VerifyHostname: "www.example.com",
+ }}
+
+ // testcase is a test case implemented by this function
+ type testcase struct {
+ // name is the test case name
+ name string
+
+ // childTactics contains the tactics that the child policy
+ // should return when invoked by the policy
+ childTactics []*httpsDialerTactic
+
+ // domain is the domain to attempt to obtain tactics for
+ domain string
+
+ // expectExtra contains the number of expected tactics
+ // we want to see beyond the child tactics above
+ expectExtra int
+ }
+
+ cases := []testcase{{
+ name: "when the children does not return any tactic, duh",
+ childTactics: nil,
+ domain: "www.example.com",
+ expectExtra: 0,
+ }, {
+ name: "when the children returns a non-TH domain",
+ childTactics: wwwExampleComTactic,
+ domain: wwwExampleComTactic[0].VerifyHostname,
+ expectExtra: 0,
+ }, {
+ name: "when the children returns a TH domain",
+ childTactics: testHelperTactics,
+ domain: testHelperTactics[0].VerifyHostname,
+ expectExtra: 304,
+ }}
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+
+ // create the policy that we're testing
+ //
+ // note how the child policy is just returning the expected
+ // set of child tactics in the original order
+ policy := &testHelpersPolicy{
+ Child: &mocksPolicy{
+ MockLookupTactics: func(ctx context.Context, domain, port string) <-chan *httpsDialerTactic {
+ output := make(chan *httpsDialerTactic)
+ go func() {
+ defer close(output)
+ for _, entry := range tc.childTactics {
+ output <- entry
+ }
+ }()
+ return output
+ },
+ },
+ }
+
+ // start to generate tactics for the given domain
+ generator := policy.LookupTactics(context.Background(), tc.domain, "443")
+
+ // obtain all the tactics
+ var tactics []*httpsDialerTactic
+ for entry := range generator {
+ tactics = append(tactics, entry)
+ }
+
+ // make sure we have the expected number of tactics
+ // at the beginning of the list
+ if len(tactics) < len(tc.childTactics) {
+ t.Fatal("expected at least", len(tc.childTactics), "got", len(tactics))
+ }
+
+ // if there are expected tactics make sure they
+ // indeed match our expectations
+ if len(tc.childTactics) > 0 {
+ if diff := cmp.Diff(tc.childTactics, tactics[:len(tc.childTactics)]); diff != "" {
+ t.Fatal(diff)
+ }
+ }
+
+ // make sure we have the expected nymber of extras
+ if diff := len(tactics) - len(tc.childTactics); diff != tc.expectExtra {
+ t.Fatal("expected", tc.expectExtra, "extras but got", diff)
+ return
+ }
+
+ // if the expected number of extras is zero, what are we still
+ // doing here and why don't we return like now?
+ if tc.expectExtra <= 0 {
+ return
+ }
+
+ // make sure we're not going to expose the domain via the SNI
+ for _, entry := range tactics[len(tc.childTactics):] {
+ if entry.SNI == tc.domain {
+ t.Fatal("did not expect to see", tc.domain, "but got", entry.SNI)
+ }
+ if entry.VerifyHostname != tc.domain {
+ t.Fatal("expected to see", tc.domain, "but got", entry.VerifyHostname)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/enginenetx/userpolicy.go b/pkg/enginenetx/userpolicy.go
index 50ce9b1e8..3a7f8a9a0 100644
--- a/pkg/enginenetx/userpolicy.go
+++ b/pkg/enginenetx/userpolicy.go
@@ -18,30 +18,22 @@ import (
"github.com/ooni/probe-engine/pkg/model"
)
-// userPolicy is an [httpsDialerPolicy] incorporating verbatim
+// userPolicyV2 is an [httpsDialerPolicy] incorporating verbatim
// a user policy loaded from the engine's key-value store.
//
// This policy is very useful for exploration and experimentation.
-type userPolicy struct {
- // Fallback is the fallback policy in case the user one does not
- // contain a rule for a specific domain.
- Fallback httpsDialerPolicy
-
+//
+// This is v2 of the userPolicy because the previous implementation
+// incorporated mixing logic, while now the mixing happens outside
+// of this policy, thus giving us much more flexibility.
+type userPolicyV2 struct {
// Root is the root of the user policy loaded from disk.
Root *userPolicyRoot
}
-// userPolicyKey is the kvstore key used to retrieve the user policy.
-const userPolicyKey = "bridges.conf"
-
-// errUserPolicyWrongVersion means that the user policy document has the wrong version number.
-var errUserPolicyWrongVersion = errors.New("wrong user policy version")
-
-// newUserPolicy attempts to constructs a user policy using a given fallback
-// policy and either returns a good policy or an error. The typical error case is the one
+// newUserPolicyV2 attempts to constructs a user policy. The typical error case is the one
// in which there's no httpsDialerUserPolicyKey in the key-value store.
-func newUserPolicy(
- kvStore model.KeyValueStore, fallback httpsDialerPolicy) (*userPolicy, error) {
+func newUserPolicyV2(kvStore model.KeyValueStore) (*userPolicyV2, error) {
// attempt to read the user policy bytes from the kvstore
data, err := kvStore.Get(userPolicyKey)
if err != nil {
@@ -66,13 +58,49 @@ func newUserPolicy(
return nil, err
}
- out := &userPolicy{
- Fallback: fallback,
- Root: &root,
- }
+ out := &userPolicyV2{Root: &root}
return out, nil
}
+var _ httpsDialerPolicy = &userPolicyV2{}
+
+// LookupTactics implements httpsDialerPolicy.
+func (ldp *userPolicyV2) LookupTactics(ctx context.Context, domain string, port string) <-chan *httpsDialerTactic {
+ // create the output channel
+ out := make(chan *httpsDialerTactic)
+
+ go func() {
+ // make sure we close the output channel
+ defer close(out)
+
+ // check whether an entry exists in the user-provided map, which MAY be nil
+ // if/when the user has chosen their policy to be as such
+ tactics, found := ldp.Root.DomainEndpoints[net.JoinHostPort(domain, port)]
+ if !found {
+ return
+ }
+
+ // make sure that there are actionable entries here
+ tactics = userPolicyRemoveNilEntries(tactics)
+ if len(tactics) <= 0 {
+ return
+ }
+
+ // emit all the user-configured tactics
+ for _, tactic := range tactics {
+ out <- tactic
+ }
+ }()
+
+ return out
+}
+
+// userPolicyKey is the kvstore key used to retrieve the user policy.
+const userPolicyKey = "bridges.conf"
+
+// errUserPolicyWrongVersion means that the user policy document has the wrong version number.
+var errUserPolicyWrongVersion = errors.New("wrong user policy version")
+
// userPolicyVersion is the current version of the user policy file.
const userPolicyVersion = 3
@@ -85,36 +113,6 @@ type userPolicyRoot struct {
Version int
}
-var _ httpsDialerPolicy = &userPolicy{}
-
-// LookupTactics implements httpsDialerPolicy.
-func (ldp *userPolicy) LookupTactics(
- ctx context.Context, domain string, port string) <-chan *httpsDialerTactic {
- // check whether an entry exists in the user-provided map, which MAY be nil
- // if/when the user has chosen their policy to be as such
- tactics, found := ldp.Root.DomainEndpoints[net.JoinHostPort(domain, port)]
- if !found {
- return ldp.Fallback.LookupTactics(ctx, domain, port)
- }
-
- // note that we also need to fallback when the tactics contains an empty list
- // or a list that only contains nil entries
- tactics = userPolicyRemoveNilEntries(tactics)
- if len(tactics) <= 0 {
- return ldp.Fallback.LookupTactics(ctx, domain, port)
- }
-
- // emit the resuults, which may possibly be empty
- out := make(chan *httpsDialerTactic)
- go func() {
- defer close(out) // let the caller know we're done
- for _, tactic := range tactics {
- out <- tactic
- }
- }()
- return out
-}
-
func userPolicyRemoveNilEntries(input []*httpsDialerTactic) (output []*httpsDialerTactic) {
for _, entry := range input {
if entry != nil {
diff --git a/pkg/enginenetx/userpolicy_test.go b/pkg/enginenetx/userpolicy_test.go
index 0774efecc..e4b3b8d84 100644
--- a/pkg/enginenetx/userpolicy_test.go
+++ b/pkg/enginenetx/userpolicy_test.go
@@ -6,15 +6,13 @@ import (
"testing"
"time"
- "github.com/apex/log"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-engine/pkg/kvstore"
- "github.com/ooni/probe-engine/pkg/mocks"
"github.com/ooni/probe-engine/pkg/runtimex"
)
-func TestUserPolicy(t *testing.T) {
- t.Run("newUserPolicy", func(t *testing.T) {
+func TestUserPolicyV2(t *testing.T) {
+ t.Run("newUserPolicyV2", func(t *testing.T) {
// testcase is a test case implemented by this function
type testcase struct {
// name is the test case name
@@ -30,11 +28,9 @@ func TestUserPolicy(t *testing.T) {
expectErr string
// expectRoot contains the expected policy we loaded or nil
- expectedPolicy *userPolicy
+ expectedPolicy *userPolicyV2
}
- fallback := &dnsPolicy{}
-
cases := []testcase{{
name: "when there is no key in the kvstore",
key: "",
@@ -106,8 +102,7 @@ func TestUserPolicy(t *testing.T) {
}))
})(),
expectErr: "",
- expectedPolicy: &userPolicy{
- Fallback: fallback,
+ expectedPolicy: &userPolicyV2{
Root: &userPolicyRoot{
DomainEndpoints: map[string][]*httpsDialerTactic{
"api.ooni.io:443": {{
@@ -152,7 +147,7 @@ func TestUserPolicy(t *testing.T) {
kvStore := &kvstore.Memory{}
runtimex.Try0(kvStore.Set(tc.key, tc.input))
- policy, err := newUserPolicy(kvStore, fallback)
+ policy, err := newUserPolicyV2(kvStore)
switch {
case err != nil && tc.expectErr == "":
@@ -178,6 +173,7 @@ func TestUserPolicy(t *testing.T) {
})
t.Run("LookupTactics", func(t *testing.T) {
+ // define the tactic we would expect to see
expectedTactic := &httpsDialerTactic{
Address: "162.55.247.208",
InitialDelay: 0,
@@ -185,6 +181,8 @@ func TestUserPolicy(t *testing.T) {
SNI: "www.example.com",
VerifyHostname: "api.ooni.io",
}
+
+ // define the root of the user policy
userPolicyRoot := &userPolicyRoot{
DomainEndpoints: map[string][]*httpsDialerTactic{
// Note that here we're adding explicitly nil entries
@@ -196,14 +194,16 @@ func TestUserPolicy(t *testing.T) {
},
// We add additional entries to make sure that in those
- // cases we are going to fallback as they're basically empty
- // and so non-actionable for us.
+ // cases we are going to get nil entries as they're basically
+ // empty and so non-actionable for us.
"api.ooni.xyz:443": nil,
"api.ooni.org:443": {},
"api.ooni.com:443": {nil, nil, nil},
},
Version: userPolicyVersion,
}
+
+ // serialize into a key-value store running in memory
kvStore := &kvstore.Memory{}
rawUserPolicyRoot := runtimex.Try1(json.Marshal(userPolicyRoot))
if err := kvStore.Set(userPolicyKey, rawUserPolicyRoot); err != nil {
@@ -213,7 +213,7 @@ func TestUserPolicy(t *testing.T) {
t.Run("with user policy", func(t *testing.T) {
ctx := context.Background()
- policy, err := newUserPolicy(kvStore, nil /* explictly to crash if used */)
+ policy, err := newUserPolicyV2(kvStore)
if err != nil {
t.Fatal(err)
}
@@ -232,19 +232,10 @@ func TestUserPolicy(t *testing.T) {
}
})
- t.Run("we fallback if there is no entry in the user policy", func(t *testing.T) {
+ t.Run("we get nothing if there is no entry in the user policy", func(t *testing.T) {
ctx := context.Background()
- fallback := &dnsPolicy{
- Logger: log.Log,
- Resolver: &mocks.Resolver{
- MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
- return []string{"93.184.216.34"}, nil
- },
- },
- }
-
- policy, err := newUserPolicy(kvStore, fallback)
+ policy, err := newUserPolicyV2(kvStore)
if err != nil {
t.Fatal(err)
}
@@ -256,32 +247,17 @@ func TestUserPolicy(t *testing.T) {
got = append(got, tactic)
}
- expect := []*httpsDialerTactic{{
- Address: "93.184.216.34",
- InitialDelay: 0,
- Port: "443",
- SNI: "www.example.com",
- VerifyHostname: "www.example.com",
- }}
+ expect := []*httpsDialerTactic{}
if diff := cmp.Diff(expect, got); diff != "" {
t.Fatal(diff)
}
})
- t.Run("we fallback if the entry in the user policy is ~empty", func(t *testing.T) {
+ t.Run("we get nothing if the entry in the user policy is ~empty", func(t *testing.T) {
ctx := context.Background()
- fallback := &dnsPolicy{
- Logger: log.Log,
- Resolver: &mocks.Resolver{
- MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
- return []string{"93.184.216.34"}, nil
- },
- },
- }
-
- policy, err := newUserPolicy(kvStore, fallback)
+ policy, err := newUserPolicyV2(kvStore)
if err != nil {
t.Fatal(err)
}
@@ -296,13 +272,7 @@ func TestUserPolicy(t *testing.T) {
got = append(got, tactic)
}
- expect := []*httpsDialerTactic{{
- Address: "93.184.216.34",
- InitialDelay: 0,
- Port: "443",
- SNI: domain,
- VerifyHostname: domain,
- }}
+ expect := []*httpsDialerTactic{}
if diff := cmp.Diff(expect, got); diff != "" {
t.Fatal(diff)
diff --git a/pkg/engineresolver/resolver.go b/pkg/engineresolver/resolver.go
index 60f912bd6..a44a05291 100644
--- a/pkg/engineresolver/resolver.go
+++ b/pkg/engineresolver/resolver.go
@@ -14,9 +14,9 @@ import (
"time"
"github.com/ooni/probe-engine/pkg/bytecounter"
+ "github.com/ooni/probe-engine/pkg/legacy/multierror"
"github.com/ooni/probe-engine/pkg/logx"
"github.com/ooni/probe-engine/pkg/model"
- "github.com/ooni/probe-engine/pkg/multierror"
)
// Resolver is the session resolver. Resolver will try to use
@@ -168,7 +168,7 @@ func (r *Resolver) lookupHost(ctx context.Context, ri *resolverinfo, hostname st
//
// The return value is only meaningful for testing.
func (r *Resolver) maybeConfusion(state []*resolverinfo, seed int64) int {
- rng := rand.New(rand.NewSource(seed))
+ rng := rand.New(rand.NewSource(seed)) // #nosec G404 -- not really important
const confusion = 0.3
if rng.Float64() >= confusion {
return -1
diff --git a/pkg/engineresolver/resolver_test.go b/pkg/engineresolver/resolver_test.go
index 8678ff8b5..b73dabbf4 100644
--- a/pkg/engineresolver/resolver_test.go
+++ b/pkg/engineresolver/resolver_test.go
@@ -14,9 +14,9 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-engine/pkg/bytecounter"
"github.com/ooni/probe-engine/pkg/kvstore"
+ "github.com/ooni/probe-engine/pkg/legacy/multierror"
"github.com/ooni/probe-engine/pkg/mocks"
"github.com/ooni/probe-engine/pkg/model"
- "github.com/ooni/probe-engine/pkg/multierror"
"github.com/ooni/probe-engine/pkg/netxlite"
)
diff --git a/pkg/engineresolver/resolvermaker.go b/pkg/engineresolver/resolvermaker.go
index e23dcb2fb..8946af345 100644
--- a/pkg/engineresolver/resolvermaker.go
+++ b/pkg/engineresolver/resolvermaker.go
@@ -58,7 +58,7 @@ var allbyurl = resolverMakeInitialState()
// see https://github.com/ooni/probe/issues/2544.
func resolverMakeInitialState() map[string]*resolvermaker {
output := make(map[string]*resolvermaker)
- rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+ rng := rand.New(rand.NewSource(time.Now().UnixNano())) // #nosec G404 -- not really important
for _, e := range allmakers {
output[e.url] = e
if e.url != systemResolverURL {
diff --git a/pkg/erroror/erroror.go b/pkg/erroror/erroror.go
new file mode 100644
index 000000000..833272ee8
--- /dev/null
+++ b/pkg/erroror/erroror.go
@@ -0,0 +1,8 @@
+// Package erroror contains code to represent an error or a value.
+package erroror
+
+// Value represents an error or a value.
+type Value[Type any] struct {
+ Err error
+ Value Type
+}
diff --git a/pkg/experiment/echcheck/handshake.go b/pkg/experiment/echcheck/handshake.go
index 7b47a403e..d8d4a3197 100644
--- a/pkg/experiment/echcheck/handshake.go
+++ b/pkg/experiment/echcheck/handshake.go
@@ -68,10 +68,10 @@ var certpool = netxlite.NewMozillaCertPool()
// genTLSConfig generates tls.Config from a given SNI
func genTLSConfig(sni string) *tls.Config {
- return &tls.Config{
+ return &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
RootCAs: certpool,
ServerName: sni,
NextProtos: []string{"h2", "http/1.1"},
- InsecureSkipVerify: true,
+ InsecureSkipVerify: true, // #nosec G402 - it's fine to skip verify in a nettest
}
}
diff --git a/pkg/experiment/echcheck/utls.go b/pkg/experiment/echcheck/utls.go
index 49b50bed3..d585e9a86 100644
--- a/pkg/experiment/echcheck/utls.go
+++ b/pkg/experiment/echcheck/utls.go
@@ -37,7 +37,7 @@ func (t *tlsHandshakerWithExtensions) Handshake(
runtimex.Assert(err == nil, "unexpected error when creating UTLSConn")
if t.extensions != nil && len(t.extensions) != 0 {
- tlsConn.BuildHandshakeState()
+ runtimex.Try0(tlsConn.BuildHandshakeState())
tlsConn.Extensions = append(tlsConn.Extensions, t.extensions...)
}
diff --git a/pkg/experiment/fbmessenger/fbmessenger.go b/pkg/experiment/fbmessenger/fbmessenger.go
index 243525e45..22f95d799 100644
--- a/pkg/experiment/fbmessenger/fbmessenger.go
+++ b/pkg/experiment/fbmessenger/fbmessenger.go
@@ -179,7 +179,7 @@ func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
for _, service := range Services {
inputs = append(inputs, urlgetter.MultiInput{Target: service})
}
- rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
+ rnd := rand.New(rand.NewSource(time.Now().UnixNano())) // #nosec G404 -- not really important
rnd.Shuffle(len(inputs), func(i, j int) {
inputs[i], inputs[j] = inputs[j], inputs[i]
})
diff --git a/pkg/experiment/hhfm/hhfm.go b/pkg/experiment/hhfm/hhfm.go
index 759d5ab2d..1b71fd6d6 100644
--- a/pkg/experiment/hhfm/hhfm.go
+++ b/pkg/experiment/hhfm/hhfm.go
@@ -81,9 +81,6 @@ func (m Measurer) ExperimentVersion() string {
}
var (
- // ErrNoAvailableTestHelpers is emitted when there are no available test helpers.
- ErrNoAvailableTestHelpers = errors.New("no available helpers")
-
// ErrInvalidHelperType is emitted when the helper type is invalid.
ErrInvalidHelperType = errors.New("invalid helper type")
)
@@ -104,7 +101,7 @@ func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
const helperName = "http-return-json-headers"
helpers, ok := sess.GetTestHelpersByName(helperName)
if !ok || len(helpers) < 1 {
- return ErrNoAvailableTestHelpers
+ return model.ErrNoAvailableTestHelpers
}
helper := helpers[0]
if helper.Type != "legacy" {
diff --git a/pkg/experiment/hhfm/hhfm_test.go b/pkg/experiment/hhfm/hhfm_test.go
index fe870d196..87d5a32c2 100644
--- a/pkg/experiment/hhfm/hhfm_test.go
+++ b/pkg/experiment/hhfm/hhfm_test.go
@@ -272,7 +272,7 @@ func TestNoHelpers(t *testing.T) {
Session: sess,
}
err := measurer.Run(ctx, args)
- if !errors.Is(err, hhfm.ErrNoAvailableTestHelpers) {
+ if !errors.Is(err, model.ErrNoAvailableTestHelpers) {
t.Fatal("not the error we expected")
}
tk := measurement.TestKeys.(*hhfm.TestKeys)
@@ -327,7 +327,7 @@ func TestNoActualHelpersInList(t *testing.T) {
Session: sess,
}
err := measurer.Run(ctx, args)
- if !errors.Is(err, hhfm.ErrNoAvailableTestHelpers) {
+ if !errors.Is(err, model.ErrNoAvailableTestHelpers) {
t.Fatal("not the error we expected")
}
tk := measurement.TestKeys.(*hhfm.TestKeys)
diff --git a/pkg/experiment/hirl/hirl.go b/pkg/experiment/hirl/hirl.go
index 1fa90e1ca..14f9e7574 100644
--- a/pkg/experiment/hirl/hirl.go
+++ b/pkg/experiment/hirl/hirl.go
@@ -68,9 +68,6 @@ func (m Measurer) ExperimentVersion() string {
}
var (
- // ErrNoAvailableTestHelpers is emitted when there are no available test helpers.
- ErrNoAvailableTestHelpers = errors.New("no available helpers")
-
// ErrInvalidHelperType is emitted when the helper type is invalid.
ErrInvalidHelperType = errors.New("invalid helper type")
@@ -91,7 +88,7 @@ func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
const helperName = "tcp-echo"
helpers, ok := sess.GetTestHelpersByName(helperName)
if !ok || len(helpers) < 1 {
- return ErrNoAvailableTestHelpers
+ return model.ErrNoAvailableTestHelpers
}
helper := helpers[0]
if helper.Type != "legacy" {
diff --git a/pkg/experiment/hirl/hirl_test.go b/pkg/experiment/hirl/hirl_test.go
index 77efd88f5..19a8fa005 100644
--- a/pkg/experiment/hirl/hirl_test.go
+++ b/pkg/experiment/hirl/hirl_test.go
@@ -302,7 +302,7 @@ func TestNoHelpers(t *testing.T) {
Session: sess,
}
err := measurer.Run(ctx, args)
- if !errors.Is(err, hirl.ErrNoAvailableTestHelpers) {
+ if !errors.Is(err, model.ErrNoAvailableTestHelpers) {
t.Fatal("not the error we expected")
}
tk := measurement.TestKeys.(*hirl.TestKeys)
@@ -340,7 +340,7 @@ func TestNoActualHelperInList(t *testing.T) {
Session: sess,
}
err := measurer.Run(ctx, args)
- if !errors.Is(err, hirl.ErrNoAvailableTestHelpers) {
+ if !errors.Is(err, model.ErrNoAvailableTestHelpers) {
t.Fatal("not the error we expected")
}
tk := measurement.TestKeys.(*hirl.TestKeys)
diff --git a/pkg/experiment/ndt7/dial.go b/pkg/experiment/ndt7/dial.go
index fd329abc9..e62aa0c2c 100644
--- a/pkg/experiment/ndt7/dial.go
+++ b/pkg/experiment/ndt7/dial.go
@@ -29,7 +29,7 @@ func newDialManager(ndt7URL string, logger model.Logger, userAgent string) dialM
}
}
-func (mgr dialManager) dialWithTestName(ctx context.Context, testName string) (*websocket.Conn, error) {
+func (mgr dialManager) dialWithTestName(ctx context.Context, _ string) (*websocket.Conn, error) {
netx := &netxlite.Netx{}
reso := netx.NewStdlibResolver(mgr.logger)
dlr := netx.NewDialerWithResolver(mgr.logger, reso)
@@ -40,7 +40,7 @@ func (mgr dialManager) dialWithTestName(ctx context.Context, testName string) (*
// See https://github.com/ooni/probe/issues/2413 to understand
// why we're using nil to force netxlite to use the cached
// default Mozilla cert pool.
- tlsConfig := &tls.Config{
+ tlsConfig := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
RootCAs: nil,
}
dialer := websocket.Dialer{
diff --git a/pkg/experiment/portfiltering/tcpconnect.go b/pkg/experiment/portfiltering/tcpconnect.go
index 28cc4b1cd..70aba4a8b 100644
--- a/pkg/experiment/portfiltering/tcpconnect.go
+++ b/pkg/experiment/portfiltering/tcpconnect.go
@@ -44,6 +44,6 @@ func (m *Measurer) tcpConnect(ctx context.Context, index int64,
dialer := trace.NewDialerWithoutResolver(logger)
conn, err := dialer.DialContext(ctx, "tcp", address)
ol.Stop(err)
- measurexlite.MaybeClose(conn)
+ _ = measurexlite.MaybeClose(conn)
return trace.FirstTCPConnectOrNil()
}
diff --git a/pkg/experiment/quicping/quic.go b/pkg/experiment/quicping/quic.go
index f69d07869..74188cd00 100644
--- a/pkg/experiment/quicping/quic.go
+++ b/pkg/experiment/quicping/quic.go
@@ -49,7 +49,7 @@ func buildPacket() ([]byte, connectionID, connectionID) {
// generate random payload
minPayloadSize := 1200 - 14 - (len(destConnID) + len(srcConnID))
randomPayload := make([]byte, minPayloadSize)
- rand.Read(randomPayload)
+ _ = runtimex.Try1(rand.Read(randomPayload))
clientSecret, _ := computeSecrets(destConnID)
encrypted := encryptPayload(randomPayload, destConnID, clientSecret)
@@ -128,5 +128,5 @@ type errUnexpectedResponse struct {
// Error implements error.Error()
func (e *errUnexpectedResponse) Error() string {
- return fmt.Sprintf("unexptected response: %s", e.msg)
+ return fmt.Sprintf("unexpected response: %s", e.msg)
}
diff --git a/pkg/experiment/quicping/quicping.go b/pkg/experiment/quicping/quicping.go
index bfe4557db..0ce50385a 100644
--- a/pkg/experiment/quicping/quicping.go
+++ b/pkg/experiment/quicping/quicping.go
@@ -259,7 +259,7 @@ func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
// set context and read timeouts
deadline := time.Duration(rep*2) * time.Second
- pconn.SetDeadline(time.Now().Add(deadline))
+ _ = pconn.SetDeadline(time.Now().Add(deadline))
ctx, cancel := context.WithTimeout(ctx, deadline)
defer cancel()
diff --git a/pkg/experiment/simplequicping/simplequicping.go b/pkg/experiment/simplequicping/simplequicping.go
index 863992bc3..8bcc36591 100644
--- a/pkg/experiment/simplequicping/simplequicping.go
+++ b/pkg/experiment/simplequicping/simplequicping.go
@@ -178,7 +178,7 @@ func (m *Measurer) quicHandshake(ctx context.Context, index int64,
// See https://github.com/ooni/probe/issues/2413 to understand
// why we're using nil to force netxlite to use the cached
// default Mozilla cert pool.
- tlsConfig := &tls.Config{
+ tlsConfig := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
NextProtos: alpn,
RootCAs: nil,
ServerName: sni,
diff --git a/pkg/experiment/sniblocking/sniblocking.go b/pkg/experiment/sniblocking/sniblocking.go
index 10d1b9892..b5217b93b 100644
--- a/pkg/experiment/sniblocking/sniblocking.go
+++ b/pkg/experiment/sniblocking/sniblocking.go
@@ -112,7 +112,7 @@ func (m *Measurer) measureone(
thaddr string,
) Subresult {
// slightly delay the measurement
- gen := rand.New(rand.NewSource(time.Now().UnixNano()))
+ gen := rand.New(rand.NewSource(time.Now().UnixNano())) // #nosec G404 -- not really important
sleeptime := time.Duration(gen.Intn(250)) * time.Millisecond
select {
case <-time.After(sleeptime):
diff --git a/pkg/experiment/tcpping/tcpping.go b/pkg/experiment/tcpping/tcpping.go
index 8201cb2ab..dbb4ed75a 100644
--- a/pkg/experiment/tcpping/tcpping.go
+++ b/pkg/experiment/tcpping/tcpping.go
@@ -138,7 +138,7 @@ func (m *Measurer) tcpConnect(ctx context.Context, index int64,
ol := logx.NewOperationLogger(logger, "TCPPing #%d %s", index, address)
conn, err := dialer.DialContext(ctx, "tcp", address)
ol.Stop(err)
- measurexlite.MaybeClose(conn)
+ _ = measurexlite.MaybeClose(conn)
sp := &SinglePing{
TCPConnect: trace.FirstTCPConnectOrNil(), // record the first connect from the buffer
}
diff --git a/pkg/experiment/tlsmiddlebox/connect.go b/pkg/experiment/tlsmiddlebox/connect.go
index f505b14bb..29a312ad6 100644
--- a/pkg/experiment/tlsmiddlebox/connect.go
+++ b/pkg/experiment/tlsmiddlebox/connect.go
@@ -21,7 +21,7 @@ func (m *Measurer) TCPConnect(ctx context.Context, index int64, zeroTime time.Ti
ol := logx.NewOperationLogger(logger, "TCPConnect #%d %s", index, address)
conn, err := dialer.DialContext(ctx, "tcp", address)
ol.Stop(err)
- measurexlite.MaybeClose(conn)
+ _ = measurexlite.MaybeClose(conn)
tcpEvents := trace.TCPConnects()
tk.addTCPConnect(tcpEvents)
return err
diff --git a/pkg/experiment/tlsmiddlebox/syscall_unix.go b/pkg/experiment/tlsmiddlebox/syscall_unix.go
index 1562f5380..5a54bcd4d 100644
--- a/pkg/experiment/tlsmiddlebox/syscall_unix.go
+++ b/pkg/experiment/tlsmiddlebox/syscall_unix.go
@@ -8,8 +8,8 @@ package tlsmiddlebox
import (
"net"
- "syscall"
"strings"
+ "syscall"
)
// SetTTL sets the IP TTL field for the underlying net.TCPConn
diff --git a/pkg/experiment/tlsmiddlebox/syscall_windows.go b/pkg/experiment/tlsmiddlebox/syscall_windows.go
index ba00cdc81..d9597e5c5 100644
--- a/pkg/experiment/tlsmiddlebox/syscall_windows.go
+++ b/pkg/experiment/tlsmiddlebox/syscall_windows.go
@@ -8,8 +8,8 @@ package tlsmiddlebox
import (
"net"
- "syscall"
"strings"
+ "syscall"
)
// SetTTL sets the IP TTL field for the underlying net.TCPConn
diff --git a/pkg/experiment/tlsmiddlebox/tracing.go b/pkg/experiment/tlsmiddlebox/tracing.go
index 093ba14c1..c91c5212e 100644
--- a/pkg/experiment/tlsmiddlebox/tracing.go
+++ b/pkg/experiment/tlsmiddlebox/tracing.go
@@ -124,11 +124,11 @@ func genTLSConfig(sni string) *tls.Config {
// See https://github.com/ooni/probe/issues/2413 to understand
// why we're using nil to force netxlite to use the cached
// default Mozilla cert pool.
- return &tls.Config{
+ return &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
RootCAs: nil,
ServerName: sni,
NextProtos: []string{"h2", "http/1.1"},
- InsecureSkipVerify: true,
+ InsecureSkipVerify: true, // #nosec G402 - it's fine to skip verify in a nettest
}
}
diff --git a/pkg/experiment/tlsping/tlsping.go b/pkg/experiment/tlsping/tlsping.go
index a1edffb3d..1134754b5 100644
--- a/pkg/experiment/tlsping/tlsping.go
+++ b/pkg/experiment/tlsping/tlsping.go
@@ -184,7 +184,7 @@ func (m *Measurer) tlsConnectAndHandshake(ctx context.Context, index int64,
// See https://github.com/ooni/probe/issues/2413 to understand
// why we're using nil to force netxlite to use the cached
// default Mozilla cert pool.
- config := &tls.Config{
+ config := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
NextProtos: alpn,
RootCAs: nil,
ServerName: sni,
diff --git a/pkg/experiment/tlstool/internal/splitter.go b/pkg/experiment/tlstool/internal/splitter.go
index 0aecfb817..88c7f773e 100644
--- a/pkg/experiment/tlstool/internal/splitter.go
+++ b/pkg/experiment/tlstool/internal/splitter.go
@@ -58,7 +58,7 @@ func Splitter3264rand(input []byte) (output [][]byte) {
output = append(output, input)
return
}
- rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
+ rnd := rand.New(rand.NewSource(time.Now().UnixNano())) // #nosec G404 -- not really important
offset := rnd.Intn(32) + 32
output = append(output, input[:offset])
output = append(output, input[offset:])
diff --git a/pkg/experiment/tlstool/tlstool.go b/pkg/experiment/tlstool/tlstool.go
index 8038ac149..1013e2e46 100644
--- a/pkg/experiment/tlstool/tlstool.go
+++ b/pkg/experiment/tlstool/tlstool.go
@@ -136,13 +136,13 @@ func (m Measurer) run(ctx context.Context, config runConfig) error {
if err != nil {
return err
}
- conn.Close()
+ _ = conn.Close()
return nil
}
func (m Measurer) tlsConfig() *tls.Config {
if m.config.SNI != "" {
- return &tls.Config{ServerName: m.config.SNI}
+ return &tls.Config{ServerName: m.config.SNI} // #nosec G402 - we need to use a large TLS versions range for measuring
}
return nil
}
diff --git a/pkg/experiment/urlgetter/configurer.go b/pkg/experiment/urlgetter/configurer.go
index c41fd3644..8f4af8a0f 100644
--- a/pkg/experiment/urlgetter/configurer.go
+++ b/pkg/experiment/urlgetter/configurer.go
@@ -80,7 +80,7 @@ func (c Configurer) NewConfiguration() (Configuration, error) {
configuration.DNSClient = dnsclient
configuration.HTTPConfig.BaseResolver = dnsclient
// configure TLS
- configuration.HTTPConfig.TLSConfig = &tls.Config{
+ configuration.HTTPConfig.TLSConfig = &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
NextProtos: []string{"h2", "http/1.1"},
}
if c.Config.TLSServerName != "" {
diff --git a/pkg/experiment/urlgetter/runner.go b/pkg/experiment/urlgetter/runner.go
index 36b8e054c..d88dbb4b1 100644
--- a/pkg/experiment/urlgetter/runner.go
+++ b/pkg/experiment/urlgetter/runner.go
@@ -113,7 +113,7 @@ func (r Runner) tlsHandshake(ctx context.Context, address string) error {
tlsDialer := netx.NewTLSDialer(r.HTTPConfig)
conn, err := tlsDialer.DialTLSContext(ctx, "tcp", address)
if conn != nil {
- conn.Close()
+ _ = conn.Close()
}
return err
}
@@ -122,7 +122,7 @@ func (r Runner) tcpConnect(ctx context.Context, address string) error {
dialer := netx.NewDialer(r.HTTPConfig)
conn, err := dialer.DialContext(ctx, "tcp", address)
if conn != nil {
- conn.Close()
+ _ = conn.Close()
}
return err
}
diff --git a/pkg/experiment/webconnectivity/control.go b/pkg/experiment/webconnectivity/control.go
index 70222422c..e7d6120e2 100644
--- a/pkg/experiment/webconnectivity/control.go
+++ b/pkg/experiment/webconnectivity/control.go
@@ -4,11 +4,10 @@ import (
"context"
"github.com/ooni/probe-engine/pkg/geoipx"
- "github.com/ooni/probe-engine/pkg/httpapi"
"github.com/ooni/probe-engine/pkg/model"
"github.com/ooni/probe-engine/pkg/netxlite"
- "github.com/ooni/probe-engine/pkg/ooapi"
"github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/webconnectivityalgo"
)
// Redirect to types defined inside the model package
@@ -24,12 +23,8 @@ type (
func Control(
ctx context.Context, sess model.ExperimentSession,
testhelpers []model.OOAPIService, creq ControlRequest) (ControlResponse, *model.OOAPIService, error) {
- seqCaller := httpapi.NewSequenceCaller(
- ooapi.NewDescriptorTH(&creq),
- httpapi.NewEndpointList(sess.DefaultHTTPClient(), sess.Logger(), sess.UserAgent(), testhelpers...)...,
- )
sess.Logger().Infof("control for %s...", creq.HTTPRequest)
- out, idx, err := seqCaller.Call(ctx)
+ out, idx, err := webconnectivityalgo.CallWebConnectivityTestHelper(ctx, &creq, testhelpers, sess)
sess.Logger().Infof("control for %s... %+v", creq.HTTPRequest, model.ErrorToStringOrOK(err))
if err != nil {
// make sure error is wrapped
diff --git a/pkg/experiment/webconnectivity/webconnectivity.go b/pkg/experiment/webconnectivity/webconnectivity.go
index 3eb4ec58a..3ef7fb923 100644
--- a/pkg/experiment/webconnectivity/webconnectivity.go
+++ b/pkg/experiment/webconnectivity/webconnectivity.go
@@ -94,9 +94,6 @@ func (m Measurer) ExperimentVersion() string {
}
var (
- // ErrNoAvailableTestHelpers is emitted when there are no available test helpers.
- ErrNoAvailableTestHelpers = errors.New("no available helpers")
-
// ErrNoInput indicates that no input was provided
ErrNoInput = errors.New("no input provided")
@@ -145,7 +142,7 @@ func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
// 1. find test helper
testhelpers, _ := sess.GetTestHelpersByName("web-connectivity")
if len(testhelpers) < 1 {
- return ErrNoAvailableTestHelpers
+ return model.ErrNoAvailableTestHelpers
}
// 2. perform the DNS lookup step
dnsBegin := time.Now()
diff --git a/pkg/experiment/webconnectivity/webconnectivity_test.go b/pkg/experiment/webconnectivity/webconnectivity_test.go
index d29e17288..3694b75e4 100644
--- a/pkg/experiment/webconnectivity/webconnectivity_test.go
+++ b/pkg/experiment/webconnectivity/webconnectivity_test.go
@@ -80,7 +80,7 @@ func TestMeasureWithCancelledContext(t *testing.T) {
}
tk := measurement.TestKeys.(*webconnectivity.TestKeys)
if *tk.ControlFailure != netxlite.FailureInterrupted {
- t.Fatal("unexpected control_failure")
+ t.Fatal("unexpected control_failure", *tk.ControlFailure)
}
if *tk.DNSExperimentFailure != netxlite.FailureInterrupted {
t.Fatal("unexpected dns_experiment_failure")
@@ -206,7 +206,7 @@ func TestMeasureWithNoAvailableTestHelpers(t *testing.T) {
Session: sess,
}
err := measurer.Run(ctx, args)
- if !errors.Is(err, webconnectivity.ErrNoAvailableTestHelpers) {
+ if !errors.Is(err, model.ErrNoAvailableTestHelpers) {
t.Fatal(err)
}
tk := measurement.TestKeys.(*webconnectivity.TestKeys)
@@ -236,11 +236,11 @@ func newsession(t *testing.T, lookupBackends bool) model.ExperimentSession {
t.Fatal(err)
}
if lookupBackends {
- if err := sess.MaybeLookupBackends(); err != nil {
+ if err := sess.MaybeLookupBackendsContext(context.Background()); err != nil {
t.Fatal(err)
}
}
- if err := sess.MaybeLookupLocation(); err != nil {
+ if err := sess.MaybeLookupLocationContext(context.Background()); err != nil {
t.Fatal(err)
}
return sess
diff --git a/pkg/experiment/webconnectivitylte/analysisclassic.go b/pkg/experiment/webconnectivitylte/analysisclassic.go
index 798e478e3..21ec80c00 100644
--- a/pkg/experiment/webconnectivitylte/analysisclassic.go
+++ b/pkg/experiment/webconnectivitylte/analysisclassic.go
@@ -25,7 +25,7 @@ func analysisEngineClassic(tk *TestKeys, logger model.Logger) {
tk.analysisClassic(model.GeoIPASNLookupperFunc(geoipx.LookupASN), logger)
}
-func (tk *TestKeys) analysisClassic(lookupper model.GeoIPASNLookupper, logger model.Logger) {
+func (tk *TestKeys) analysisClassic(lookupper model.GeoIPASNLookupper, _ model.Logger) {
// Since we run after all tasks have completed (or so we assume) we're
// not going to use any form of locking here.
diff --git a/pkg/experiment/webconnectivitylte/cleartextflow.go b/pkg/experiment/webconnectivitylte/cleartextflow.go
index e238a8238..c3ef0cd78 100644
--- a/pkg/experiment/webconnectivitylte/cleartextflow.go
+++ b/pkg/experiment/webconnectivitylte/cleartextflow.go
@@ -96,7 +96,7 @@ func (t *CleartextFlow) Start(ctx context.Context) {
index := t.IDGenerator.NewIDForEndpointCleartext()
go func() {
defer t.WaitGroup.Done() // synchronize with the parent
- t.Run(ctx, index)
+ _ = t.Run(ctx, index)
}()
}
@@ -114,7 +114,7 @@ func (t *CleartextFlow) Run(parentCtx context.Context, index int64) error {
sampler := throttling.NewSampler(trace)
defer func() {
t.TestKeys.AppendNetworkEvents(sampler.ExtractSamples()...)
- sampler.Close()
+ _ = sampler.Close()
}()
// start the operation logger
@@ -282,6 +282,9 @@ func (t *CleartextFlow) httpTransaction(ctx context.Context, network, address, a
}
if err == nil && httpRedirectIsRedirect(resp) {
err = httpValidateRedirect(resp)
+ if err == nil && t.FollowRedirects && !t.NumRedirects.CanFollowOneMoreRedirect() {
+ err = ErrTooManyRedirects
+ }
}
finished := trace.TimeSince(trace.ZeroTime())
@@ -319,10 +322,7 @@ func (t *CleartextFlow) httpTransaction(ctx context.Context, network, address, a
// maybeFollowRedirects follows redirects if configured and needed
func (t *CleartextFlow) maybeFollowRedirects(ctx context.Context, resp *http.Response) {
- if !t.FollowRedirects || !t.NumRedirects.CanFollowOneMoreRedirect() {
- return // not configured or too many redirects
- }
- if httpRedirectIsRedirect(resp) {
+ if t.FollowRedirects && httpRedirectIsRedirect(resp) {
location, err := resp.Location()
if err != nil {
return // broken response from server
diff --git a/pkg/experiment/webconnectivitylte/control.go b/pkg/experiment/webconnectivitylte/control.go
index c9351c5d8..15a14a31a 100644
--- a/pkg/experiment/webconnectivitylte/control.go
+++ b/pkg/experiment/webconnectivitylte/control.go
@@ -8,12 +8,11 @@ import (
"time"
"github.com/ooni/probe-engine/pkg/experiment/webconnectivity"
- "github.com/ooni/probe-engine/pkg/httpapi"
"github.com/ooni/probe-engine/pkg/logx"
"github.com/ooni/probe-engine/pkg/model"
"github.com/ooni/probe-engine/pkg/netxlite"
- "github.com/ooni/probe-engine/pkg/ooapi"
"github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/webconnectivityalgo"
)
// EndpointMeasurementsStarter is used by Control to start extra
@@ -109,14 +108,8 @@ func (c *Control) Run(parentCtx context.Context) {
c.TestHelpers,
)
- // create an httpapi sequence caller
- seqCaller := httpapi.NewSequenceCaller(
- ooapi.NewDescriptorTH(creq),
- httpapi.NewEndpointList(c.Session.DefaultHTTPClient(), c.Logger, c.Session.UserAgent(), c.TestHelpers...)...,
- )
-
// issue the control request and wait for the response
- cresp, idx, err := seqCaller.Call(opCtx)
+ cresp, idx, err := webconnectivityalgo.CallWebConnectivityTestHelper(opCtx, creq, c.TestHelpers, c.Session)
if err != nil {
// make sure error is wrapped
err = netxlite.NewTopLevelGenericErrWrapper(err)
diff --git a/pkg/experiment/webconnectivitylte/measurer.go b/pkg/experiment/webconnectivitylte/measurer.go
index 9bc6e4f62..d000c01f7 100644
--- a/pkg/experiment/webconnectivitylte/measurer.go
+++ b/pkg/experiment/webconnectivitylte/measurer.go
@@ -10,7 +10,6 @@ import (
"net/http/cookiejar"
"sync"
- "github.com/ooni/probe-engine/pkg/experiment/webconnectivity"
"github.com/ooni/probe-engine/pkg/inputparser"
"github.com/ooni/probe-engine/pkg/model"
"github.com/ooni/probe-engine/pkg/webconnectivityalgo"
@@ -108,7 +107,7 @@ func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
testhelpers, _ := sess.GetTestHelpersByName("web-connectivity")
if len(testhelpers) < 1 {
sess.Logger().Warnf("continuing without a valid TH address")
- tk.SetControlFailure(webconnectivity.ErrNoAvailableTestHelpers)
+ tk.SetControlFailure(model.ErrNoAvailableTestHelpers)
}
registerExtensions(measurement)
@@ -121,7 +120,7 @@ func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
Domain: URL.Hostname(),
IDGenerator: NewIDGenerator(),
Logger: sess.Logger(),
- NumRedirects: NewNumRedirects(5),
+ NumRedirects: NewNumRedirects(10),
TestKeys: tk,
URL: URL,
ZeroTime: measurement.MeasurementStartTimeSaved,
diff --git a/pkg/experiment/webconnectivitylte/redirects.go b/pkg/experiment/webconnectivitylte/redirects.go
index 882a05d0b..5a2e238ee 100644
--- a/pkg/experiment/webconnectivitylte/redirects.go
+++ b/pkg/experiment/webconnectivitylte/redirects.go
@@ -19,5 +19,5 @@ func NewNumRedirects(n int64) *NumRedirects {
// CanFollowOneMoreRedirect returns true if we are
// allowed to follow one more redirect.
func (nr *NumRedirects) CanFollowOneMoreRedirect() bool {
- return nr.count.Add(-1) > 0
+ return nr.count.Add(-1) >= 0
}
diff --git a/pkg/experiment/webconnectivitylte/redirects_test.go b/pkg/experiment/webconnectivitylte/redirects_test.go
new file mode 100644
index 000000000..03c9b56d2
--- /dev/null
+++ b/pkg/experiment/webconnectivitylte/redirects_test.go
@@ -0,0 +1,16 @@
+package webconnectivitylte
+
+import "testing"
+
+func TestNumRedirects(t *testing.T) {
+ const count = 10
+ nr := NewNumRedirects(count)
+ for idx := 0; idx < count; idx++ {
+ if !nr.CanFollowOneMoreRedirect() {
+ t.Fatal("got false with idx=", idx)
+ }
+ }
+ if nr.CanFollowOneMoreRedirect() {
+ t.Fatal("got true after the loop")
+ }
+}
diff --git a/pkg/experiment/webconnectivitylte/secureflow.go b/pkg/experiment/webconnectivitylte/secureflow.go
index 1f9b7ab32..b01d82d6b 100644
--- a/pkg/experiment/webconnectivitylte/secureflow.go
+++ b/pkg/experiment/webconnectivitylte/secureflow.go
@@ -9,6 +9,7 @@ package webconnectivitylte
import (
"context"
"crypto/tls"
+ "errors"
"io"
"net"
"net/http"
@@ -103,7 +104,7 @@ func (t *SecureFlow) Start(ctx context.Context) {
index := t.IDGenerator.NewIDForEndpointSecure()
go func() {
defer t.WaitGroup.Done() // synchronize with the parent
- t.Run(ctx, index)
+ _ = t.Run(ctx, index)
}()
}
@@ -121,7 +122,7 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) error {
sampler := throttling.NewSampler(trace)
defer func() {
t.TestKeys.AppendNetworkEvents(sampler.ExtractSamples()...)
- sampler.Close()
+ _ = sampler.Close()
}()
// start the operation logger
@@ -161,7 +162,7 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) error {
// See https://github.com/ooni/probe/issues/2413 to understand
// why we're using nil to force netxlite to use the cached
// default Mozilla cert pool.
- tlsConfig := &tls.Config{
+ tlsConfig := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
NextProtos: t.alpn(),
RootCAs: nil,
ServerName: tlsSNI,
@@ -305,6 +306,9 @@ func (t *SecureFlow) newHTTPRequest(ctx context.Context) (*http.Request, error)
return httpReq, nil
}
+// ErrTooManyRedirects indicates we have seen too many HTTP redirects.
+var ErrTooManyRedirects = errors.New("stopped after too many redirects")
+
// httpTransaction runs the HTTP transaction and saves the results.
func (t *SecureFlow) httpTransaction(ctx context.Context, network, address, alpn string,
txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) {
@@ -337,6 +341,9 @@ func (t *SecureFlow) httpTransaction(ctx context.Context, network, address, alpn
}
if err == nil && httpRedirectIsRedirect(resp) {
err = httpValidateRedirect(resp)
+ if err == nil && t.FollowRedirects && !t.NumRedirects.CanFollowOneMoreRedirect() {
+ err = ErrTooManyRedirects
+ }
}
finished := trace.TimeSince(trace.ZeroTime())
@@ -374,10 +381,7 @@ func (t *SecureFlow) httpTransaction(ctx context.Context, network, address, alpn
// maybeFollowRedirects follows redirects if configured and needed
func (t *SecureFlow) maybeFollowRedirects(ctx context.Context, resp *http.Response) {
- if !t.FollowRedirects || !t.NumRedirects.CanFollowOneMoreRedirect() {
- return // not configured or too many redirects
- }
- if httpRedirectIsRedirect(resp) {
+ if t.FollowRedirects && httpRedirectIsRedirect(resp) {
location, err := resp.Location()
if err != nil {
return // broken response from server
diff --git a/pkg/experiment/whatsapp/whatsapp.go b/pkg/experiment/whatsapp/whatsapp.go
index 20c6f3d88..a66091788 100644
--- a/pkg/experiment/whatsapp/whatsapp.go
+++ b/pkg/experiment/whatsapp/whatsapp.go
@@ -162,7 +162,7 @@ func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
// don't care about the HTTP response code.
Target: WebHTTPSURL,
})
- rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
+ rnd := rand.New(rand.NewSource(time.Now().UnixNano())) // #nosec G404 -- not really important
rnd.Shuffle(len(inputs), func(i, j int) {
inputs[i], inputs[j] = inputs[j], inputs[i]
})
diff --git a/pkg/feature/psiphonfeat/psiphon_enabled.go b/pkg/feature/psiphonfeat/psiphon_enabled.go
index 6adc0222a..95f11ee95 100644
--- a/pkg/feature/psiphonfeat/psiphon_enabled.go
+++ b/pkg/feature/psiphonfeat/psiphon_enabled.go
@@ -1,4 +1,4 @@
-//go:build !go1.21 && !ooni_feature_disable_psiphon
+//go:build !go1.22 && !ooni_feature_disable_psiphon
package psiphonfeat
diff --git a/pkg/feature/psiphonfeat/psiphon_otherwise.go b/pkg/feature/psiphonfeat/psiphon_otherwise.go
index 979312171..4ed78fa05 100644
--- a/pkg/feature/psiphonfeat/psiphon_otherwise.go
+++ b/pkg/feature/psiphonfeat/psiphon_otherwise.go
@@ -1,4 +1,4 @@
-//go:build go1.21 || ooni_feature_disable_psiphon
+//go:build go1.22 || ooni_feature_disable_psiphon
package psiphonfeat
diff --git a/pkg/fsx/fsx.go b/pkg/fsx/fsx.go
index 6ee374dfa..ff8c4d50a 100644
--- a/pkg/fsx/fsx.go
+++ b/pkg/fsx/fsx.go
@@ -29,11 +29,11 @@ func openWithFS(fs fs.FS, pathname string) (fs.File, error) {
}
info, err := file.Stat()
if err != nil {
- file.Close()
+ _ = file.Close()
return nil, err
}
if !IsRegular(info) {
- file.Close()
+ _ = file.Close()
return nil, fmt.Errorf("%w: %s", ErrNotRegularFile, pathname)
}
return file, nil
@@ -44,7 +44,7 @@ type filesystem struct{}
// Open implements fs.FS.Open.
func (filesystem) Open(pathname string) (fs.File, error) {
- return os.Open(pathname)
+ return os.Open(pathname) // #nosec G304 - this is working as intended
}
// IsRegular returns whether a file is a regular file.
diff --git a/pkg/httpapi/call.go b/pkg/httpapi/call.go
deleted file mode 100644
index 4c35fe07a..000000000
--- a/pkg/httpapi/call.go
+++ /dev/null
@@ -1,210 +0,0 @@
-package httpapi
-
-//
-// Calling HTTP APIs.
-//
-
-import (
- "bytes"
- "compress/gzip"
- "context"
- "errors"
- "fmt"
- "io"
- "net/http"
- "net/url"
- "strings"
-
- "github.com/ooni/probe-engine/pkg/netxlite"
- "github.com/ooni/probe-engine/pkg/runtimex"
-)
-
-// joinURLPath appends resourcePath to urlPath.
-func joinURLPath(urlPath, resourcePath string) string {
- if resourcePath == "" {
- if urlPath == "" {
- return "/"
- }
- return urlPath
- }
- if !strings.HasSuffix(urlPath, "/") {
- urlPath += "/"
- }
- resourcePath = strings.TrimPrefix(resourcePath, "/")
- return urlPath + resourcePath
-}
-
-// newRequest creates a new http.Request from the given ctx, endpoint, and desc.
-func newRequest[RequestType, ResponseType any](
- ctx context.Context,
- endpoint *Endpoint,
- desc *Descriptor[RequestType, ResponseType],
-) (*http.Request, error) {
- URL, err := url.Parse(endpoint.BaseURL)
- if err != nil {
- return nil, err
- }
- // BaseURL and resource URL are joined if they have a path
- URL.Path = joinURLPath(URL.Path, desc.URLPath)
- if len(desc.URLQuery) > 0 {
- URL.RawQuery = desc.URLQuery.Encode()
- } else {
- URL.RawQuery = "" // as documented we only honour desc.URLQuery
- }
- var reqBody io.Reader
- if desc.Request != nil && len(desc.Request.Body) > 0 {
- reqBody = bytes.NewReader(desc.Request.Body)
- endpoint.Logger.Debugf("httpapi: request body length: %d", len(desc.Request.Body))
- if desc.LogBody {
- endpoint.Logger.Debugf("httpapi: request body: %s", string(desc.Request.Body))
- }
- }
- request, err := http.NewRequestWithContext(ctx, desc.Method, URL.String(), reqBody)
- if err != nil {
- return nil, err
- }
- request.Host = endpoint.Host // allow cloudfronting
- if desc.Authorization != "" {
- request.Header.Set("Authorization", desc.Authorization)
- }
- if desc.ContentType != "" {
- request.Header.Set("Content-Type", desc.ContentType)
- }
- if desc.Accept != "" {
- request.Header.Set("Accept", desc.Accept)
- }
- if endpoint.UserAgent != "" {
- request.Header.Set("User-Agent", endpoint.UserAgent)
- }
- if desc.AcceptEncodingGzip {
- request.Header.Set("Accept-Encoding", "gzip")
- }
- return request, nil
-}
-
-// ErrHTTPRequestFailed indicates that the server returned >= 400.
-type ErrHTTPRequestFailed struct {
- // StatusCode is the status code that failed.
- StatusCode int
-}
-
-// Error implements error.
-func (err *ErrHTTPRequestFailed) Error() string {
- return fmt.Sprintf("httpapi: http request failed: %d", err.StatusCode)
-}
-
-// errMaybeCensorship indicates that there was an error at the networking layer
-// including, e.g., DNS, TCP connect, TLS. When we see this kind of error, we
-// will consider retrying with another endpoint under the assumption that it
-// may be that the current endpoint is censored.
-type errMaybeCensorship struct {
- // Err is the underlying error
- Err error
-}
-
-// Error implements error
-func (err *errMaybeCensorship) Error() string {
- return err.Err.Error()
-}
-
-// Unwrap allows to get the underlying error
-func (err *errMaybeCensorship) Unwrap() error {
- return err.Err
-}
-
-// ErrTruncated indicates we truncated the response body.
-var ErrTruncated = errors.New("httpapi: truncated response body")
-
-// docall calls the API represented by the given request req on the given endpoint
-// and returns the response and its body or an error.
-func docall[RequestType, ResponseType any](
- endpoint *Endpoint,
- desc *Descriptor[RequestType, ResponseType],
- request *http.Request,
-) (*http.Response, []byte, error) {
- // Implementation note: remember to mark errors for which you want
- // to retry with another endpoint using errMaybeCensorship.
-
- response, err := endpoint.HTTPClient.Do(request)
- if err != nil {
- return nil, nil, &errMaybeCensorship{err}
- }
- defer response.Body.Close()
-
- var reader io.Reader = response.Body
- if response.Header.Get("Content-Encoding") == "gzip" {
- reader, err = gzip.NewReader(reader)
- if err != nil {
- // This case happens when we cannot read the gzip header
- // hence it can be "triggered" remotely and we cannot just
- // panic on error to handle this error condition.
- return response, nil, err
- }
- }
- maxBodySize := desc.MaxBodySize
- if maxBodySize <= 0 {
- maxBodySize = DefaultMaxBodySize
- }
- // Implementation note: when there's decompression we must (obviously?)
- // enforce the maximum body size on the _decompressed_ body.
- reader = io.LimitReader(reader, maxBodySize)
-
- // Implementation note: always read and log the response body _before_
- // checking the status code, since it's quite useful to log the JSON
- // returned by the OONI API in case of errors. Obviously, the flip side
- // of this choice is that we read potentially very large error pages.
- data, err := netxlite.ReadAllContext(request.Context(), reader)
- if err != nil {
- return response, nil, &errMaybeCensorship{err}
- }
- if int64(len(data)) >= maxBodySize {
- return response, nil, ErrTruncated
- }
- endpoint.Logger.Debugf("httpapi: response body length: %d bytes", len(data))
- if desc.LogBody {
- endpoint.Logger.Debugf("httpapi: response body: %s", string(data))
- }
-
- if response.StatusCode >= 400 {
- return response, nil, &ErrHTTPRequestFailed{response.StatusCode}
- }
- return response, data, nil
-}
-
-// call is like Call but also returns the response.
-func call[RequestType, ResponseType any](
- ctx context.Context,
- desc *Descriptor[RequestType, ResponseType],
- endpoint *Endpoint,
-) (*http.Response, []byte, error) {
- timeout := desc.Timeout
- if timeout <= 0 {
- timeout = DefaultCallTimeout // as documented
- }
- ctx, cancel := context.WithTimeout(ctx, timeout)
- defer cancel()
- request, err := newRequest(ctx, endpoint, desc)
- if err != nil {
- return nil, nil, err
- }
- return docall(endpoint, desc, request)
-}
-
-// Call invokes the API described by desc on the given HTTP endpoint and
-// returns the response body (as a ResponseType instance) or an error.
-//
-// Note: this function returns ErrHTTPRequestFailed if the HTTP status code is
-// greater or equal than 400. You could use errors.As to obtain a copy of the
-// error that was returned and see for yourself the actual status code.
-func Call[RequestType, ResponseType any](
- ctx context.Context,
- desc *Descriptor[RequestType, ResponseType],
- endpoint *Endpoint,
-) (ResponseType, error) {
- runtimex.Assert(desc.Response != nil, "desc.Response is nil")
- resp, rawResponseBody, err := call(ctx, desc, endpoint)
- if err != nil {
- return *new(ResponseType), err
- }
- return desc.Response.Unmarshal(resp, rawResponseBody)
-}
diff --git a/pkg/httpapi/call_test.go b/pkg/httpapi/call_test.go
deleted file mode 100644
index 48de34b95..000000000
--- a/pkg/httpapi/call_test.go
+++ /dev/null
@@ -1,1131 +0,0 @@
-package httpapi
-
-import (
- "bytes"
- "compress/gzip"
- "context"
- "errors"
- "io"
- "net/http"
- "net/http/httptest"
- "reflect"
- "strings"
- "syscall"
- "testing"
-
- "github.com/google/go-cmp/cmp"
- "github.com/ooni/probe-engine/pkg/mocks"
- "github.com/ooni/probe-engine/pkg/model"
- "github.com/ooni/probe-engine/pkg/netxlite"
-)
-
-func Test_joinURLPath(t *testing.T) {
- tests := []struct {
- name string
- urlPath string
- resourcePath string
- want string
- }{{
- name: "whole path inside urlPath and empty resourcePath",
- urlPath: "/robots.txt",
- resourcePath: "",
- want: "/robots.txt",
- }, {
- name: "empty urlPath and slash-prefixed resourcePath",
- urlPath: "",
- resourcePath: "/foo",
- want: "/foo",
- }, {
- name: "slash urlPath and slash-prefixed resourcePath",
- urlPath: "/",
- resourcePath: "/foo",
- want: "/foo",
- }, {
- name: "empty urlPath and empty resourcePath",
- urlPath: "",
- resourcePath: "",
- want: "/",
- }, {
- name: "non-slash-terminated urlPath and slash-prefixed resourcePath",
- urlPath: "/foo",
- resourcePath: "/bar",
- want: "/foo/bar",
- }, {
- name: "slash-terminated urlPath and slash-prefixed resourcePath",
- urlPath: "/foo/",
- resourcePath: "/bar",
- want: "/foo/bar",
- }, {
- name: "slash-terminated urlPath and non-slash-prefixed resourcePath",
- urlPath: "/foo",
- resourcePath: "bar",
- want: "/foo/bar",
- }}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got := joinURLPath(tt.urlPath, tt.resourcePath)
- if diff := cmp.Diff(tt.want, got); diff != "" {
- t.Fatal(diff)
- }
- })
- }
-}
-
-func Test_newRequest(t *testing.T) {
- type args struct {
- ctx context.Context
- endpoint *Endpoint
- desc *Descriptor[RawRequest, []byte]
- }
- tests := []struct {
- name string
- args args
- wantFn func(*testing.T, *http.Request)
- wantErr error
- }{{
- name: "url.Parse fails",
- args: args{
- ctx: nil,
- endpoint: &Endpoint{
- BaseURL: "\t\t\t", // does not parse!
- HTTPClient: nil,
- Host: "",
- Logger: nil,
- UserAgent: "",
- },
- desc: &Descriptor[RawRequest, []byte]{
- Accept: "",
- Authorization: "",
- ContentType: "",
- LogBody: false,
- MaxBodySize: 0,
- Method: "",
- Request: nil,
- Response: &RawResponseDescriptor{},
- Timeout: 0,
- URLPath: "",
- URLQuery: nil,
- },
- },
- wantFn: nil,
- wantErr: errors.New(`parse "\t\t\t": net/url: invalid control character in URL`),
- }, {
- name: "http.NewRequestWithContext fails",
- args: args{
- ctx: nil, // causes http.NewRequestWithContext to fail
- endpoint: &Endpoint{
- BaseURL: "https://example.com/",
- HTTPClient: nil,
- Host: "",
- Logger: nil,
- UserAgent: "",
- },
- desc: &Descriptor[RawRequest, []byte]{
- Accept: "",
- Authorization: "",
- ContentType: "",
- LogBody: false,
- MaxBodySize: 0,
- Method: "",
- Request: nil,
- Response: &RawResponseDescriptor{},
- Timeout: 0,
- URLPath: "",
- URLQuery: nil,
- },
- },
- wantFn: nil,
- wantErr: errors.New("net/http: nil Context"),
- }, {
- name: "successful case with GET method, no body, and no extra headers",
- args: args{
- ctx: context.Background(),
- endpoint: &Endpoint{
- BaseURL: "https://example.com/",
- HTTPClient: nil,
- Host: "",
- Logger: nil,
- UserAgent: "",
- },
- desc: &Descriptor[RawRequest, []byte]{
- Accept: "",
- Authorization: "",
- ContentType: "",
- LogBody: false,
- MaxBodySize: 0,
- Method: http.MethodGet,
- Request: nil,
- Response: &RawResponseDescriptor{},
- Timeout: 0,
- URLPath: "",
- URLQuery: nil,
- },
- },
- wantFn: func(t *testing.T, req *http.Request) {
- if req == nil {
- t.Fatal("expected non-nil request")
- }
- if req.Method != http.MethodGet {
- t.Fatal("invalid method")
- }
- if req.URL.String() != "https://example.com/" {
- t.Fatal("invalid URL")
- }
- if req.Body != nil {
- t.Fatal("invalid body", req.Body)
- }
- },
- wantErr: nil,
- }, {
- name: "successful case with POST method and body",
- args: args{
- ctx: context.Background(),
- endpoint: &Endpoint{
- BaseURL: "https://example.com/",
- HTTPClient: nil,
- Host: "",
- Logger: model.DiscardLogger,
- UserAgent: "",
- },
- desc: &Descriptor[RawRequest, []byte]{
- Accept: "",
- Authorization: "",
- ContentType: "",
- LogBody: true, // just to exercise the code path
- MaxBodySize: 0,
- Method: http.MethodPost,
- Request: &RequestDescriptor[RawRequest]{Body: []byte("deadbeef")},
- Response: &RawResponseDescriptor{},
- Timeout: 0,
- URLPath: "",
- URLQuery: nil,
- },
- },
- wantFn: func(t *testing.T, req *http.Request) {
- if req == nil {
- t.Fatal("expected non-nil request")
- }
- if req.Method != http.MethodPost {
- t.Fatal("invalid method")
- }
- if req.URL.String() != "https://example.com/" {
- t.Fatal("invalid URL")
- }
- data, err := netxlite.ReadAllContext(context.Background(), req.Body)
- if err != nil {
- t.Fatal(err)
- }
- if diff := cmp.Diff([]byte("deadbeef"), data); diff != "" {
- t.Fatal(diff)
- }
- },
- wantErr: nil,
- }, {
- name: "with GET method and custom headers",
- args: args{
- ctx: context.Background(),
- endpoint: &Endpoint{
- BaseURL: "https://example.com/",
- HTTPClient: nil,
- Host: "antani.org",
- Logger: nil,
- UserAgent: "httpclient/1.0.1",
- },
- desc: &Descriptor[RawRequest, []byte]{
- Accept: "application/json",
- Authorization: "deafbeef",
- ContentType: "text/plain",
- LogBody: false,
- MaxBodySize: 0,
- Method: http.MethodPut,
- Request: nil,
- Response: &RawResponseDescriptor{},
- Timeout: 0,
- URLPath: "",
- URLQuery: nil,
- },
- },
- wantFn: func(t *testing.T, req *http.Request) {
- if req == nil {
- t.Fatal("expected non-nil request")
- }
- if req.Method != http.MethodPut {
- t.Fatal("invalid method")
- }
- if req.Host != "antani.org" {
- t.Fatal("invalid request host")
- }
- if req.URL.String() != "https://example.com/" {
- t.Fatal("invalid URL")
- }
- if req.Header.Get("Authorization") != "deafbeef" {
- t.Fatal("invalid authorization")
- }
- if req.Header.Get("Content-Type") != "text/plain" {
- t.Fatal("invalid content-type")
- }
- if req.Header.Get("Accept") != "application/json" {
- t.Fatal("invalid accept")
- }
- if req.Header.Get("User-Agent") != "httpclient/1.0.1" {
- t.Fatal("invalid user-agent")
- }
- },
- wantErr: nil,
- }, {
- name: "we join the urlPath with the resourcePath",
- args: args{
- ctx: context.Background(),
- endpoint: &Endpoint{
- BaseURL: "https://www.example.com/api/v1",
- HTTPClient: nil,
- Host: "",
- Logger: nil,
- UserAgent: "",
- },
- desc: &Descriptor[RawRequest, []byte]{
- Accept: "",
- Authorization: "",
- ContentType: "",
- LogBody: false,
- MaxBodySize: 0,
- Method: http.MethodGet,
- Request: nil,
- Response: &RawResponseDescriptor{},
- Timeout: 0,
- URLPath: "/test-list/urls",
- URLQuery: nil,
- },
- },
- wantFn: func(t *testing.T, req *http.Request) {
- if req == nil {
- t.Fatal("expected non-nil request")
- }
- if req.Method != http.MethodGet {
- t.Fatal("invalid method")
- }
- if req.URL.String() != "https://www.example.com/api/v1/test-list/urls" {
- t.Fatal("invalid URL")
- }
- },
- wantErr: nil,
- }, {
- name: "we discard any query element inside the Endpoint.BaseURL",
- args: args{
- ctx: context.Background(),
- endpoint: &Endpoint{
- BaseURL: "https://example.org/api/v1/?probe_cc=IT",
- HTTPClient: nil,
- Host: "",
- Logger: nil,
- UserAgent: "",
- },
- desc: &Descriptor[RawRequest, []byte]{
- Accept: "",
- Authorization: "",
- ContentType: "",
- LogBody: false,
- MaxBodySize: 0,
- Method: http.MethodGet,
- Request: nil,
- Response: &RawResponseDescriptor{},
- Timeout: 0,
- URLPath: "",
- URLQuery: nil,
- },
- },
- wantFn: func(t *testing.T, req *http.Request) {
- if req == nil {
- t.Fatal("expected non-nil request")
- }
- if req.Method != http.MethodGet {
- t.Fatal("invalid method")
- }
- if req.URL.String() != "https://example.org/api/v1/" {
- t.Fatal("invalid URL")
- }
- },
- wantErr: nil,
- }, {
- name: "we include query elements from Descriptor.URLQuery",
- args: args{
- ctx: context.Background(),
- endpoint: &Endpoint{
- BaseURL: "https://www.example.com/api/v1/",
- HTTPClient: nil,
- Host: "",
- Logger: nil,
- UserAgent: "",
- },
- desc: &Descriptor[RawRequest, []byte]{
- Accept: "",
- Authorization: "",
- ContentType: "",
- LogBody: false,
- MaxBodySize: 0,
- Method: http.MethodGet,
- Request: nil,
- Response: &RawResponseDescriptor{},
- Timeout: 0,
- URLPath: "test-list/urls",
- URLQuery: map[string][]string{
- "probe_cc": {"IT"},
- },
- },
- },
- wantFn: func(t *testing.T, req *http.Request) {
- if req == nil {
- t.Fatal("expected non-nil request")
- }
- if req.Method != http.MethodGet {
- t.Fatal("invalid method")
- }
- if req.URL.String() != "https://www.example.com/api/v1/test-list/urls?probe_cc=IT" {
- t.Fatal("invalid URL")
- }
- },
- wantErr: nil,
- }, {
- name: "with as many implicitly-initialized fields as possible",
- args: args{
- ctx: context.Background(),
- endpoint: &Endpoint{
- BaseURL: "https://example.com/",
- },
- desc: &Descriptor[RawRequest, []byte]{},
- },
- wantFn: func(t *testing.T, req *http.Request) {
- if req == nil {
- t.Fatal("expected non-nil request")
- }
- if req.Method != http.MethodGet {
- t.Fatal("invalid method")
- }
- if req.URL.String() != "https://example.com/" {
- t.Fatal("invalid URL")
- }
- },
- wantErr: nil,
- }, {
- name: "we honour the AcceptEncodingGzip flag",
- args: args{
- ctx: context.Background(),
- endpoint: &Endpoint{
- BaseURL: "https://example.com/",
- },
- desc: &Descriptor[RawRequest, []byte]{
- AcceptEncodingGzip: true,
- },
- },
- wantFn: func(t *testing.T, req *http.Request) {
- if req.Header.Get("Accept-Encoding") != "gzip" {
- t.Fatal("did not set the Accept-Encoding header")
- }
- },
- wantErr: nil,
- }}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := newRequest(tt.args.ctx, tt.args.endpoint, tt.args.desc)
- switch {
- case err == nil && tt.wantErr == nil:
- // nothing
- case err != nil && tt.wantErr == nil:
- t.Fatalf("expected error but got %s", err.Error())
- case err == nil && tt.wantErr != nil:
- t.Fatalf("expected %s but got ", tt.wantErr.Error())
- case err.Error() == tt.wantErr.Error():
- // nothing
- default:
- t.Fatalf("expected %s but got %s", err.Error(), tt.wantErr.Error())
- }
- if tt.wantFn != nil {
- tt.wantFn(t, got)
- return
- }
- if got != nil {
- t.Fatal("got response with nil tt.wantFn")
- }
- })
- }
-}
-
-// gzipBombForCall contains one megabyte of zeroes compressed using gzip
-var gzipBombForCall = []byte{
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0xff, 0xec, 0xc0, 0x31, 0x01, 0x00, 0x00,
- 0x00, 0xc2, 0x20, 0xfb, 0xa7, 0x36, 0xc4, 0x5e,
- 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x40, 0xf4, 0x00, 0x00, 0x00, 0xff, 0xff, 0x1c,
- 0xea, 0x38, 0xa7, 0x00, 0x00, 0x10, 0x00,
-}
-
-func Test_docall(t *testing.T) {
- type args struct {
- endpoint *Endpoint
- desc *Descriptor[RawRequest, []byte]
- request *http.Request
- }
- tests := []struct {
- name string
- args args
- wantResp *http.Response
- wantBody []byte
- wantErr error
- }{{
- name: "we honour the configured max body size",
- args: args{
- endpoint: &Endpoint{
- BaseURL: "http://127.0.0.2/", // actually unused
- HTTPClient: &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- resp := &http.Response{
- StatusCode: 200,
- Body: io.NopCloser(strings.NewReader("AAAAAAAAAAAAAAAAA")),
- }
- return resp, nil
- },
- },
- Host: "",
- Logger: model.DiscardLogger,
- UserAgent: "",
- },
- desc: &Descriptor[RawRequest, []byte]{
- MaxBodySize: 7,
- Method: http.MethodGet,
- Response: &RawResponseDescriptor{},
- URLPath: "/",
- },
- request: &http.Request{},
- },
- wantResp: &http.Response{
- // Implementation note: the test will ONLY match
- // the status code and the response headers.
- StatusCode: 200,
- },
- wantBody: nil,
- wantErr: ErrTruncated,
- }, {
- name: "we have a default max body size",
- args: args{
- endpoint: &Endpoint{
- BaseURL: "http://127.0.0.2/", // actually unused
- HTTPClient: &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- resp := &http.Response{
- StatusCode: 200,
- Body: io.NopCloser(strings.NewReader("AAAAAAAAAAAAAAAAA")),
- }
- return resp, nil
- },
- },
- Host: "",
- Logger: model.DiscardLogger,
- UserAgent: "",
- },
- desc: &Descriptor[RawRequest, []byte]{
- MaxBodySize: 0, // we're testing that putting zero here implies default
- Method: http.MethodGet,
- Response: &RawResponseDescriptor{},
- URLPath: "/",
- },
- request: &http.Request{},
- },
- wantResp: &http.Response{
- // Implementation note: the test will ONLY match
- // the status code and the response headers.
- StatusCode: 200,
- },
- wantBody: []byte("AAAAAAAAAAAAAAAAA"),
- wantErr: nil,
- }, {
- name: "we decompress gzip encoded bodies",
- args: args{
- endpoint: &Endpoint{
- BaseURL: "http://127.0.0.2/", // actually unused
- HTTPClient: &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- resp := &http.Response{
- StatusCode: 200,
- Body: io.NopCloser(bytes.NewReader([]byte{
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0xff, 0x72, 0x74, 0x74, 0x74, 0xd4, 0x45,
- 0x25, 0x00, 0x01, 0x00, 0x00, 0xff, 0xff, 0xc2,
- 0x43, 0xb0, 0x08, 0x13, 0x00, 0x00, 0x00,
- })),
- Header: http.Header{
- "Content-Encoding": {"gzip"},
- },
- }
- return resp, nil
- },
- },
- Host: "",
- Logger: model.DiscardLogger,
- UserAgent: "",
- },
- desc: &Descriptor[RawRequest, []byte]{
- Method: http.MethodGet,
- Response: &RawResponseDescriptor{},
- URLPath: "/",
- },
- request: &http.Request{},
- },
- wantResp: &http.Response{
- // Implementation note: the test will ONLY match
- // the status code and the response headers.
- StatusCode: 200,
- Header: http.Header{
- "Content-Encoding": {"gzip"},
- },
- },
- wantBody: []byte("AAAA-AAAA-AAAA-AAAA"),
- wantErr: nil,
- }, {
- name: "we handle issues with the gzip header",
- args: args{
- endpoint: &Endpoint{
- BaseURL: "http://127.0.0.2/", // actually unused
- HTTPClient: &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- resp := &http.Response{
- StatusCode: 200,
- Body: io.NopCloser(bytes.NewReader([]byte{
- 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, // <- changed this line
- 0x00, 0xff, 0x72, 0x74, 0x74, 0x74, 0xd4, 0x45,
- 0x25, 0x00, 0x01, 0x00, 0x00, 0xff, 0xff, 0xc2,
- 0x43, 0xb0, 0x08, 0x13, 0x00, 0x00, 0x00,
- })),
- Header: http.Header{
- "Content-Encoding": {"gzip"},
- },
- }
- return resp, nil
- },
- },
- Host: "",
- Logger: model.DiscardLogger,
- UserAgent: "",
- },
- desc: &Descriptor[RawRequest, []byte]{
- Method: http.MethodGet,
- Response: &RawResponseDescriptor{},
- URLPath: "/",
- },
- request: &http.Request{},
- },
- wantResp: &http.Response{
- // Implementation note: the test will ONLY match
- // the status code and the response headers.
- StatusCode: 200,
- Header: http.Header{
- "Content-Encoding": {"gzip"},
- },
- },
- wantBody: nil,
- wantErr: gzip.ErrHeader,
- }, {
- name: "we protect against a gzip bomb",
- args: args{
- endpoint: &Endpoint{
- BaseURL: "http://127.0.0.2/", // actually unused
- HTTPClient: &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- resp := &http.Response{
- StatusCode: 200,
- Body: io.NopCloser(bytes.NewReader(gzipBombForCall)),
- Header: http.Header{
- "Content-Encoding": {"gzip"},
- },
- }
- return resp, nil
- },
- },
- Host: "",
- Logger: model.DiscardLogger,
- UserAgent: "",
- },
- desc: &Descriptor[RawRequest, []byte]{
- MaxBodySize: 2048, // very small value
- Method: http.MethodGet,
- Response: &RawResponseDescriptor{},
- URLPath: "/",
- },
- request: &http.Request{},
- },
- wantResp: &http.Response{
- // Implementation note: the test will ONLY match
- // the status code and the response headers.
- StatusCode: 200,
- Header: http.Header{
- "Content-Encoding": {"gzip"},
- },
- },
- wantBody: nil,
- wantErr: ErrTruncated,
- }}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- resp, body, err := docall(tt.args.endpoint, tt.args.desc, tt.args.request)
- if err != tt.wantErr {
- t.Errorf("docall() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
-
- // as documented we match ONLY status code and response headers
- if !reflect.DeepEqual(resp.StatusCode, tt.wantResp.StatusCode) {
- t.Errorf("docall() got = %v, want %v", resp.StatusCode, tt.wantResp.StatusCode)
- }
- if !reflect.DeepEqual(resp.Header, tt.wantResp.Header) {
- t.Errorf("docall() got = %v, want %v", resp.Header, tt.wantResp.Header)
- }
-
- if !reflect.DeepEqual(body, tt.wantBody) {
- t.Errorf("docall() got1 = %v, want %v", body, tt.wantBody)
- }
- })
- }
-}
-
-func TestCall(t *testing.T) {
- type args struct {
- ctx context.Context
- desc *Descriptor[RawRequest, []byte]
- endpoint *Endpoint
- }
- tests := []struct {
- name string
- args args
- want []byte
- wantErr error
- errfn func(t *testing.T, err error)
- }{{
- name: "newRequest fails",
- args: args{
- ctx: context.Background(),
- desc: &Descriptor[RawRequest, []byte]{
- Accept: "",
- Authorization: "",
- ContentType: "",
- LogBody: false,
- MaxBodySize: 0,
- Method: "",
- Request: nil,
- Response: &RawResponseDescriptor{},
- Timeout: 0,
- URLPath: "",
- URLQuery: nil,
- },
- endpoint: &Endpoint{
- BaseURL: "\t\t\t", // causes newRequest to fail
- HTTPClient: nil,
- Host: "",
- Logger: nil,
- UserAgent: "",
- },
- },
- want: nil,
- wantErr: errors.New(`parse "\t\t\t": net/url: invalid control character in URL`),
- errfn: nil,
- }, {
- name: "endpoint.HTTPClient.Do fails",
- args: args{
- ctx: context.Background(),
- desc: &Descriptor[RawRequest, []byte]{
- Method: http.MethodGet,
- Response: &RawResponseDescriptor{},
- },
- endpoint: &Endpoint{
- BaseURL: "https://example.com/",
- HTTPClient: &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- return nil, io.EOF
- },
- },
- Logger: model.DiscardLogger,
- },
- },
- want: nil,
- wantErr: io.EOF,
- errfn: func(t *testing.T, err error) {
- var expect *errMaybeCensorship
- if !errors.As(err, &expect) {
- t.Fatal("unexpected error type")
- }
- },
- }, {
- name: "reading body fails",
- args: args{
- ctx: context.Background(),
- desc: &Descriptor[RawRequest, []byte]{
- Method: http.MethodGet,
- Response: &RawResponseDescriptor{},
- },
- endpoint: &Endpoint{
- BaseURL: "https://www.example.com/",
- HTTPClient: &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- resp := &http.Response{
- Body: io.NopCloser(&mocks.Reader{
- MockRead: func(b []byte) (int, error) {
- return 0, netxlite.ECONNRESET
- },
- }),
- }
- return resp, nil
- },
- },
- Logger: model.DiscardLogger,
- },
- },
- want: nil,
- wantErr: errors.New(netxlite.FailureConnectionReset),
- errfn: func(t *testing.T, err error) {
- var expect *errMaybeCensorship
- if !errors.As(err, &expect) {
- t.Fatal("unexpected error type")
- }
- },
- }, {
- name: "status code indicates failure",
- args: args{
- ctx: context.Background(),
- desc: &Descriptor[RawRequest, []byte]{
- Method: http.MethodGet,
- Response: &RawResponseDescriptor{},
- },
- endpoint: &Endpoint{
- BaseURL: "https://example.com/",
- HTTPClient: &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- resp := &http.Response{
- Body: io.NopCloser(strings.NewReader("deadbeef")),
- StatusCode: 403,
- }
- return resp, nil
- },
- },
- Logger: model.DiscardLogger,
- },
- },
- want: nil,
- wantErr: errors.New("httpapi: http request failed: 403"),
- errfn: func(t *testing.T, err error) {
- var expect *ErrHTTPRequestFailed
- if !errors.As(err, &expect) {
- t.Fatal("invalid error type")
- }
- },
- }, {
- name: "success with log body flag",
- args: args{
- ctx: context.Background(),
- desc: &Descriptor[RawRequest, []byte]{
- LogBody: true, // as documented by this test's name
- Method: http.MethodGet,
- Response: &RawResponseDescriptor{},
- },
- endpoint: &Endpoint{
- BaseURL: "https://example.com/",
- HTTPClient: &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- resp := &http.Response{
- Body: io.NopCloser(strings.NewReader("deadbeef")),
- StatusCode: 200,
- }
- return resp, nil
- },
- },
- Logger: model.DiscardLogger,
- },
- },
- want: []byte("deadbeef"),
- wantErr: nil,
- errfn: nil,
- }}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := Call(tt.args.ctx, tt.args.desc, tt.args.endpoint)
- switch {
- case err == nil && tt.wantErr == nil:
- // nothing
- case err != nil && tt.wantErr == nil:
- t.Fatalf("expected error but got %s", err.Error())
- case err == nil && tt.wantErr != nil:
- t.Fatalf("expected %s but got ", tt.wantErr.Error())
- case err.Error() == tt.wantErr.Error():
- // nothing
- default:
- t.Fatalf("expected %s but got %s", err.Error(), tt.wantErr.Error())
- }
- if diff := cmp.Diff(tt.want, got); diff != "" {
- t.Fatal(diff)
- }
- })
- }
-}
-
-// CallStructRequest is the request used by TestCallWithJSON
-type CallStructRequest struct {
- Name string
- Age int
-}
-
-// CallStructResponse is the response used by TestCallWithJSON
-type CallStructResponse struct {
- Name string
- AgeSquared int
-}
-
-func TestCallWithJSON(t *testing.T) {
- tests := []struct {
- name string
- bodyToReturn []byte
- want *CallStructResponse
- wantErr error
- }{{
- name: "with JSON parser failure",
- bodyToReturn: []byte(`{`),
- want: nil,
- wantErr: errors.New("unexpected end of JSON input"),
- }, {
- name: "with literal null response",
- bodyToReturn: []byte(`null`),
- want: &CallStructResponse{},
- wantErr: nil,
- }, {
- name: "with good response",
- bodyToReturn: []byte(`{"Name": "sbs", "AgeSquared": 1156}`),
- want: &CallStructResponse{
- Name: "sbs",
- AgeSquared: 1156,
- },
- wantErr: nil,
- }}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
-
- // start a server that will return the configured body
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Write(tt.bodyToReturn)
- }))
- defer server.Close()
-
- // prepare endpoint for the call
- epnt := &Endpoint{
- BaseURL: server.URL,
- HTTPClient: http.DefaultClient,
- Host: "",
- Logger: model.DiscardLogger,
- UserAgent: "",
- }
-
- // prepare descriptor for the call
- desc := &Descriptor[*CallStructRequest, *CallStructResponse]{
- Accept: ApplicationJSON,
- Authorization: "",
- AcceptEncodingGzip: false,
- ContentType: ApplicationJSON,
- LogBody: true,
- MaxBodySize: 0,
- Method: http.MethodPost,
- Request: &RequestDescriptor[*CallStructRequest]{
- Body: []byte(`{"Name": "sbs", "Age": 34}`),
- },
- Response: &JSONResponseDescriptor[CallStructResponse]{},
- Timeout: 0,
- URLPath: "/",
- URLQuery: nil,
- }
-
- // call the API
- resp, err := Call(context.Background(), desc, epnt)
-
- // check the error
- switch {
- case err == nil && tt.wantErr == nil:
- // nothing
- case err != nil && tt.wantErr == nil:
- t.Fatalf("expected error but got %s", err.Error())
- case err == nil && tt.wantErr != nil:
- t.Fatalf("expected %s but got ", tt.wantErr.Error())
- case err.Error() == tt.wantErr.Error():
- // nothing
- default:
- t.Fatalf("expected %s but got %s", err.Error(), tt.wantErr.Error())
- }
-
- // check the response
- if diff := cmp.Diff(tt.want, resp); diff != "" {
- t.Fatal(diff)
- }
- })
- }
-}
-
-func TestCallHonoursContext(t *testing.T) {
- ctx, cancel := context.WithCancel(context.Background())
- cancel() // should fail HTTP request immediately
- desc := &Descriptor[RawRequest, []byte]{
- LogBody: false,
- Method: http.MethodGet,
- Response: &RawResponseDescriptor{},
- URLPath: "/robots.txt",
- }
- endpoint := &Endpoint{
- BaseURL: "https://www.example.com/",
- HTTPClient: http.DefaultClient,
- Logger: model.DiscardLogger,
- UserAgent: model.HTTPHeaderUserAgent,
- }
- body, err := Call(ctx, desc, endpoint)
- if !errors.Is(err, context.Canceled) {
- t.Fatal("unexpected err", err)
- }
- if len(body) > 0 {
- t.Fatal("expected zero-length body")
- }
-}
-
-func Test_errMaybeCensorship_Unwrap(t *testing.T) {
- t.Run("for errors.Is", func(t *testing.T) {
- var err error = &errMaybeCensorship{io.EOF}
- if !errors.Is(err, io.EOF) {
- t.Fatal("cannot unwrap")
- }
- })
-
- t.Run("for errors.As", func(t *testing.T) {
- var err error = &errMaybeCensorship{netxlite.ECONNRESET}
- var syserr syscall.Errno
- if !errors.As(err, &syserr) || syserr != netxlite.ECONNRESET {
- t.Fatal("cannot unwrap")
- }
- })
-}
diff --git a/pkg/httpapi/descriptor.go b/pkg/httpapi/descriptor.go
deleted file mode 100644
index 66ef04b77..000000000
--- a/pkg/httpapi/descriptor.go
+++ /dev/null
@@ -1,124 +0,0 @@
-package httpapi
-
-//
-// HTTP API descriptor (e.g., GET /api/v1/test-list/urls)
-//
-
-import (
- "encoding/json"
- "net/http"
- "net/url"
- "time"
-)
-
-// RawRequest is the type to use with [RequestDescriptor] and
-// [Descriptor] when the request body is just raw bytes.
-type RawRequest struct{}
-
-// RequestDescriptor describes the request.
-type RequestDescriptor[T any] struct {
- // Body is the raw request body.
- Body []byte
-}
-
-// ResponseDescriptor describes the response.
-type ResponseDescriptor[T any] interface {
- // Unmarshal unmarshals the raw response into a T.
- Unmarshal(resp *http.Response, data []byte) (T, error)
-}
-
-// RawResponseDescriptor is the type to use with [Descriptor]
-// when the response's body is just raw bytes.
-type RawResponseDescriptor struct{}
-
-var _ ResponseDescriptor[[]byte] = &RawResponseDescriptor{}
-
-// Unmarshal implements ResponseDescriptor
-func (r *RawResponseDescriptor) Unmarshal(resp *http.Response, data []byte) ([]byte, error) {
- return data, nil
-}
-
-// JSONResponseDescriptor is the type to use with [Descriptor]
-// when the response's body is encoded using JSON.
-type JSONResponseDescriptor[T any] struct{}
-
-// Unmarshal implements ResponseDescriptor
-func (r *JSONResponseDescriptor[T]) Unmarshal(resp *http.Response, data []byte) (*T, error) {
- // Important safety note: this implementation is tailored so that, when
- // the raw JSON body is `null`, we DO NOT return `nil`, `nil`. Because
- // we create a T on the stack and then let it escape, in such a case the
- // code will instead return an empty T and nil. Returning an empty T is
- // slightly better because the caller does not need to worry about the
- // returned pointer also being nil, but they just need to worry about
- // whether any field inside the returned struct is the zero value.
- //
- // (Of course, the above reasoning breaks if the caller asks for a T
- // equal to `*Foo`, which causes the return value to be `**Foo`. That
- // said, in all cases in OONI we have T equal to `Foo` and we return
- // a `*Foo` type. This scenario is, in fact, the only one making sense
- // when you're reading a JSON from a server. So, while the problem is
- // only solved for a sub-problem, this sub-problem is the one that matters.)
- //
- // Because this safety property is important, there is also a test that
- // makes sure we don't return `nil`, `nil` with `null` input.
- var value T
- if err := json.Unmarshal(data, &value); err != nil {
- return nil, err
- }
- return &value, nil
-}
-
-// Descriptor contains the parameters for calling a given HTTP
-// API (e.g., GET /api/v1/test-list/urls).
-//
-// The zero value of this struct is invalid. Please, fill all the
-// fields marked as MANDATORY for correct initialization.
-type Descriptor[RequestType, ResponseType any] struct {
- // Accept contains the OPTIONAL accept header.
- Accept string
-
- // Authorization is the OPTIONAL authorization.
- Authorization string
-
- // AcceptEncodingGzip OPTIONALLY accepts gzip-encoding bodies.
- AcceptEncodingGzip bool
-
- // ContentType is the OPTIONAL content-type header.
- ContentType string
-
- // LogBody OPTIONALLY enables logging bodies.
- LogBody bool
-
- // MaxBodySize is the OPTIONAL maximum response body size. If
- // not set, we use the [DefaultMaxBodySize] constant.
- MaxBodySize int64
-
- // Method is the MANDATORY request method.
- Method string
-
- // Request is the OPTIONAL request descriptor.
- Request *RequestDescriptor[RequestType]
-
- // Response is the MANDATORY response descriptor.
- Response ResponseDescriptor[ResponseType]
-
- // Timeout is the OPTIONAL timeout for this call. If no timeout
- // is specified we will use the [DefaultCallTimeout] const.
- Timeout time.Duration
-
- // URLPath is the MANDATORY URL path.
- URLPath string
-
- // URLQuery is the OPTIONAL query.
- URLQuery url.Values
-}
-
-// DefaultMaxBodySize is the default value for the maximum
-// body size you can fetch using the httpapi package.
-const DefaultMaxBodySize = 1 << 24
-
-// DefaultCallTimeout is the default timeout for an httpapi call.
-const DefaultCallTimeout = 60 * time.Second
-
-// ApplicationJSON is the content-type for JSON
-const ApplicationJSON = "application/json"
diff --git a/pkg/httpapi/doc.go b/pkg/httpapi/doc.go
deleted file mode 100644
index c90932274..000000000
--- a/pkg/httpapi/doc.go
+++ /dev/null
@@ -1,15 +0,0 @@
-// Package httpapi contains code for calling HTTP APIs.
-//
-// We model HTTP APIs as follows:
-//
-// 1. [Endpoint] is an API endpoint (e.g., https://api.ooni.io);
-//
-// 2. [Descriptor] describes the specific API you want to use (e.g.,
-// GET /api/v1/test-list/urls with JSON response body).
-//
-// Generally, you use [Call] to call the API identified by a [Descriptor]
-// on the specified [Endpoint]. However, there are cases where you
-// need more complex calling patterns. For example, with [SequenceCaller]
-// you can invoke the same API [Descriptor] with multiple equivalent
-// API [Endpoint]s until one of them succeeds or all fail.
-package httpapi
diff --git a/pkg/httpapi/endpoint.go b/pkg/httpapi/endpoint.go
deleted file mode 100644
index 4490ddc63..000000000
--- a/pkg/httpapi/endpoint.go
+++ /dev/null
@@ -1,89 +0,0 @@
-package httpapi
-
-//
-// HTTP API Endpoint (e.g., https://api.ooni.io)
-//
-
-import "github.com/ooni/probe-engine/pkg/model"
-
-// Endpoint models an HTTP endpoint on which you can call
-// several HTTP APIs (e.g., https://api.ooni.io) using a
-// given HTTP client potentially using a circumvention tunnel
-// mechanism such as psiphon or torsf.
-//
-// The zero value of this struct is invalid. Please, fill all the
-// fields marked as MANDATORY for correct initialization.
-type Endpoint struct {
- // BaseURL is the MANDATORY endpoint base URL. We will honour the
- // path of this URL and prepend it to the actual path specified inside
- // a [Descriptor] URLPath. However, we will always discard any query
- // that may have been set inside the BaseURL. The only query string
- // will be composed from the [Descriptor] URLQuery values.
- //
- // For example, https://api.ooni.io.
- BaseURL string
-
- // HTTPClient is the MANDATORY HTTP client to use.
- //
- // For example, http.DefaultClient. You can introduce circumvention
- // here by using an HTTPClient bound to a specific tunnel.
- HTTPClient model.HTTPClient
-
- // Host is the OPTIONAL host header to use.
- //
- // If this field is empty we use the BaseURL's hostname. A specific
- // host header may be needed when using cloudfronting.
- Host string
-
- // Logger is the MANDATORY logger to use.
- //
- // For example, model.DiscardLogger.
- Logger model.Logger
-
- // User-Agent is the OPTIONAL user-agent to use. If empty,
- // we'll use the stdlib's default user-agent string.
- UserAgent string
-}
-
-// NewEndpointList constructs a list of API endpoints from services
-// returned by the OONI backend (or known in advance).
-//
-// Arguments:
-//
-// - httpClient is the HTTP client to use for accessing the endpoints;
-//
-// - logger is the logger to use;
-//
-// - userAgent is the user agent you would like to use;
-//
-// - service is the list of services gathered from the backend.
-func NewEndpointList(
- httpClient model.HTTPClient,
- logger model.Logger,
- userAgent string,
- services ...model.OOAPIService,
-) (out []*Endpoint) {
- for _, svc := range services {
- switch svc.Type {
- case "https":
- out = append(out, &Endpoint{
- BaseURL: svc.Address,
- HTTPClient: httpClient,
- Host: "",
- Logger: logger,
- UserAgent: userAgent,
- })
- case "cloudfront":
- out = append(out, &Endpoint{
- BaseURL: svc.Address,
- HTTPClient: httpClient,
- Host: svc.Front,
- Logger: logger,
- UserAgent: userAgent,
- })
- default:
- // nothing!
- }
- }
- return
-}
diff --git a/pkg/httpapi/endpoint_test.go b/pkg/httpapi/endpoint_test.go
deleted file mode 100644
index 7aa6671d2..000000000
--- a/pkg/httpapi/endpoint_test.go
+++ /dev/null
@@ -1,76 +0,0 @@
-package httpapi
-
-import (
- "testing"
-
- "github.com/google/go-cmp/cmp"
- "github.com/ooni/probe-engine/pkg/mocks"
- "github.com/ooni/probe-engine/pkg/model"
-)
-
-func TestNewEndpointList(t *testing.T) {
- type args struct {
- httpClient model.HTTPClient
- userAgent string
- services []model.OOAPIService
- }
- defaultHTTPClient := &mocks.HTTPClient{}
- tests := []struct {
- name string
- args args
- wantOut []*Endpoint
- }{{
- name: "with no services",
- args: args{
- httpClient: defaultHTTPClient,
- userAgent: model.HTTPHeaderUserAgent,
- services: nil,
- },
- wantOut: nil,
- }, {
- name: "common cases",
- args: args{
- httpClient: defaultHTTPClient,
- userAgent: model.HTTPHeaderUserAgent,
- services: []model.OOAPIService{{
- Address: "https://www.example.com/",
- Type: "https",
- Front: "",
- }, {
- Address: "https://www.example.org/",
- Type: "cloudfront",
- Front: "example.org.it",
- }, {
- Address: "https://nonexistent.onion/",
- Type: "onion",
- Front: "",
- }},
- },
- wantOut: []*Endpoint{{
- BaseURL: "https://www.example.com/",
- HTTPClient: defaultHTTPClient,
- Host: "",
- Logger: model.DiscardLogger,
- UserAgent: model.HTTPHeaderUserAgent,
- }, {
- BaseURL: "https://www.example.org/",
- HTTPClient: defaultHTTPClient,
- Host: "example.org.it",
- Logger: model.DiscardLogger,
- UserAgent: model.HTTPHeaderUserAgent,
- }},
- }}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- gotOut := NewEndpointList(
- tt.args.httpClient,
- model.DiscardLogger,
- tt.args.userAgent,
- tt.args.services...,
- )
- if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
- t.Fatal(diff)
- }
- })
- }
-}
diff --git a/pkg/httpapi/sequence.go b/pkg/httpapi/sequence.go
deleted file mode 100644
index 44ece4c3b..000000000
--- a/pkg/httpapi/sequence.go
+++ /dev/null
@@ -1,75 +0,0 @@
-package httpapi
-
-//
-// Sequentially call available API endpoints until one succeed
-// or all of them fail. A future implementation of this code may
-// (probably should?) take into account knowledge of what is
-// working and what is not working to optimize the order with
-// which to try different alternatives.
-//
-
-import (
- "context"
- "errors"
-
- "github.com/ooni/probe-engine/pkg/multierror"
- "github.com/ooni/probe-engine/pkg/runtimex"
-)
-
-// SequenceCaller calls the API specified by [Descriptor] once for each of
-// the available [Endpoint]s until one of them succeeds.
-//
-// CAVEAT: this code will ONLY retry API calls with subsequent endpoints when
-// the error originates in the HTTP round trip or while reading the body.
-type SequenceCaller[RequestType, ResponseType any] struct {
- // Descriptor is the API [Descriptor].
- Descriptor *Descriptor[RequestType, ResponseType]
-
- // Endpoints is the list of [Endpoint] to use.
- Endpoints []*Endpoint
-}
-
-// NewSequenceCaller is a factory for creating a [SequenceCaller].
-func NewSequenceCaller[RequestType, ResponseType any](
- desc *Descriptor[RequestType, ResponseType],
- endpoints ...*Endpoint,
-) *SequenceCaller[RequestType, ResponseType] {
- return &SequenceCaller[RequestType, ResponseType]{
- Descriptor: desc,
- Endpoints: endpoints,
- }
-}
-
-// ErrAllEndpointsFailed indicates that all endpoints failed.
-var ErrAllEndpointsFailed = errors.New("httpapi: all endpoints failed")
-
-// sequenceCallershouldRetry returns true when we should try with another endpoint given the
-// value of err which could (obviously) be nil in case of success.
-func sequenceCallerShouldRetry(err error) bool {
- var kind *errMaybeCensorship
- belongs := errors.As(err, &kind)
- return belongs
-}
-
-// Call calls [Call] for each [Endpoint] and [Descriptor] until one endpoint succeeds. The
-// return value is the response body and the selected endpoint index or the error.
-//
-// CAVEAT: this code will ONLY retry API calls with subsequent endpoints when
-// the error originates in the HTTP round trip or while reading the body.
-func (sc *SequenceCaller[RequestType, ResponseType]) Call(ctx context.Context) (ResponseType, int, error) {
- runtimex.Assert(sc.Descriptor.Response != nil, "sc.Descriptor.Response is nil")
- var selected int
- merr := multierror.New(ErrAllEndpointsFailed)
- for _, epnt := range sc.Endpoints {
- respBody, err := Call(ctx, sc.Descriptor, epnt)
- if sequenceCallerShouldRetry(err) {
- merr.Add(err)
- selected++
- continue
- }
- // Note: some errors will lead us to return
- // early as documented for this method
- return respBody, selected, err
- }
- return *new(ResponseType), -1, merr
-}
diff --git a/pkg/httpapi/sequence_test.go b/pkg/httpapi/sequence_test.go
deleted file mode 100644
index 8fdbff0b3..000000000
--- a/pkg/httpapi/sequence_test.go
+++ /dev/null
@@ -1,183 +0,0 @@
-package httpapi
-
-import (
- "context"
- "errors"
- "io"
- "net/http"
- "strings"
- "testing"
-
- "github.com/google/go-cmp/cmp"
- "github.com/ooni/probe-engine/pkg/mocks"
- "github.com/ooni/probe-engine/pkg/model"
-)
-
-func TestSequenceCaller(t *testing.T) {
- t.Run("Call", func(t *testing.T) {
- t.Run("first success", func(t *testing.T) {
- sc := NewSequenceCaller(
- &Descriptor[RawRequest, []byte]{
- Method: http.MethodGet,
- Response: &RawResponseDescriptor{},
- URLPath: "/",
- },
- &Endpoint{
- BaseURL: "https://a.example.com/",
- HTTPClient: &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- resp := &http.Response{
- StatusCode: 200,
- Body: io.NopCloser(strings.NewReader("deadbeef")),
- }
- return resp, nil
- },
- },
- Logger: model.DiscardLogger,
- },
- &Endpoint{
- BaseURL: "https://b.example.com/",
- HTTPClient: &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- return nil, io.EOF
- },
- },
- Logger: model.DiscardLogger,
- },
- )
- data, idx, err := sc.Call(context.Background())
- if err != nil {
- t.Fatal(err)
- }
- if idx != 0 {
- t.Fatal("invalid idx")
- }
- if diff := cmp.Diff([]byte("deadbeef"), data); diff != "" {
- t.Fatal(diff)
- }
- })
-
- t.Run("first HTTP failure and we immediately stop", func(t *testing.T) {
- sc := NewSequenceCaller(
- &Descriptor[RawRequest, []byte]{
- Method: http.MethodGet,
- Response: &RawResponseDescriptor{},
- URLPath: "/",
- },
- &Endpoint{
- BaseURL: "https://a.example.com/",
- HTTPClient: &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- resp := &http.Response{
- StatusCode: 403, // should cause us to return early
- Body: io.NopCloser(strings.NewReader("deadbeef")),
- }
- return resp, nil
- },
- },
- Logger: model.DiscardLogger,
- },
- &Endpoint{
- BaseURL: "https://b.example.com/",
- HTTPClient: &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- return nil, io.EOF
- },
- },
- Logger: model.DiscardLogger,
- },
- )
- data, idx, err := sc.Call(context.Background())
- var failure *ErrHTTPRequestFailed
- if !errors.As(err, &failure) || failure.StatusCode != 403 {
- t.Fatal("unexpected err", err)
- }
- if idx != 0 {
- t.Fatal("invalid idx")
- }
- if len(data) > 0 {
- t.Fatal("expected to see no response body")
- }
- })
-
- t.Run("first network failure, second success", func(t *testing.T) {
- sc := NewSequenceCaller(
- &Descriptor[RawRequest, []byte]{
- Method: http.MethodGet,
- Response: &RawResponseDescriptor{},
- URLPath: "/",
- },
- &Endpoint{
- BaseURL: "https://a.example.com/",
- HTTPClient: &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- return nil, io.EOF // should cause us to cycle to the second entry
- },
- },
- Logger: model.DiscardLogger,
- },
- &Endpoint{
- BaseURL: "https://b.example.com/",
- HTTPClient: &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- resp := &http.Response{
- StatusCode: 200,
- Body: io.NopCloser(strings.NewReader("abad1dea")),
- }
- return resp, nil
- },
- },
- Logger: model.DiscardLogger,
- },
- )
- data, idx, err := sc.Call(context.Background())
- if err != nil {
- t.Fatal(err)
- }
- if idx != 1 {
- t.Fatal("invalid idx")
- }
- if diff := cmp.Diff([]byte("abad1dea"), data); diff != "" {
- t.Fatal(diff)
- }
- })
-
- t.Run("all network failure", func(t *testing.T) {
- sc := NewSequenceCaller(
- &Descriptor[RawRequest, []byte]{
- Method: http.MethodGet,
- Response: &RawResponseDescriptor{},
- URLPath: "/",
- },
- &Endpoint{
- BaseURL: "https://a.example.com/",
- HTTPClient: &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- return nil, io.EOF // should cause us to cycle to the next entry
- },
- },
- Logger: model.DiscardLogger,
- },
- &Endpoint{
- BaseURL: "https://b.example.com/",
- HTTPClient: &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- return nil, io.EOF // should cause us to cycle to the next entry
- },
- },
- Logger: model.DiscardLogger,
- },
- )
- data, idx, err := sc.Call(context.Background())
- if !errors.Is(err, ErrAllEndpointsFailed) {
- t.Fatal("unexpected err", err)
- }
- if idx != -1 {
- t.Fatal("invalid idx")
- }
- if len(data) > 0 {
- t.Fatal("expected zero-length data")
- }
- })
- })
-}
diff --git a/pkg/httpclientx/DESIGN.md b/pkg/httpclientx/DESIGN.md
new file mode 100644
index 000000000..dd3c396f4
--- /dev/null
+++ b/pkg/httpclientx/DESIGN.md
@@ -0,0 +1,417 @@
+# ./internal/httpclientx
+
+This package aims to replace previously existing packages for interact with
+backend services controlled either by us or by third parties.
+
+As of 2024-04-22, these packages are:
+
+* `./internal/httpx`, which is currently deprecated;
+
+* `./internal/httpapi`, which aims to automatically generate swaggers from input
+and output messages and implements support for falling back.
+
+The rest of this document explains the requirements and describes the design.
+
+## Table of Contents
+
+- [Requirements](#requirements)
+- [Design](#design)
+ - [Overlapped Operations](#overlapped-operations)
+ - [Extensibility](#extensibility)
+ - [Functionality Comparison](#functionality-comparison)
+ - [GetJSON](#getjson)
+ - [GetRaw](#getraw)
+ - [GetXML](#getxml)
+ - [PostJSON](#postjson)
+- [Nil Safety](#nil-safety)
+- [Refactoring Plan](#refactoring-plan)
+- [Limitations and Future Work](#limitations-and-future-work)
+
+## Requirements
+
+We want this new package to:
+
+1. Implement common access patterns (GET with JSON body, GET with XML body, GET with
+raw body, and POST with JSON request and response bodies).
+
+2. Optionally log request and response bodies.
+
+3. Factor common code for accessing such services.
+
+4. Take advantage of Go generics to automatically marshal and unmarshal without
+having to write specific functions for each request/response pair.
+
+5. Support some kind of fallback policy like `httpapi` because we used this
+for test helpers and, while, as of today, we're mapping serveral TH domain names
+to a single IP address, it might still be useful to fallback.
+
+6. Provide an easy way for the caller to know whether an HTTP request failed
+and, if so, with which status code, which is needed to intercept `401` responses
+to take the appropriate logging-in actions.
+
+7. Make the design extensible such that re-adding unused functionality
+does not require us to refactor the code much.
+
+8. Functional equivalent with existing packages (modulo the existing
+functionality that is not relevant anymore).
+
+Non goals:
+
+1. automatically generate swaggers from the Go representation of API invocation
+for these reasons: (1) the OONI backend swagger is only for documentational
+purposes and it is not always in sync with reality; (2) by doing that, we obtained
+overly complex code, which hampers maintenance.
+
+2. implementing algorithms such as logging in to the OONI backend and requesting
+tokens, which should be the responsibility of another package.
+
+## Design
+
+This package supports the following operations:
+
+```Go
+type Config struct {
+ Client model.HTTPClient
+ Logger model.Logger
+ UserAgent string
+}
+
+type Endpoint struct {
+ URL string
+ Host string // optional for cloudfronting
+}
+
+func GetJSON[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error)
+
+func GetRaw(ctx context.Context, epnt *Endpoint, config *Config) ([]byte, error)
+
+func GetXML[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error)
+
+func PostJSON[Input, Output any](ctx context.Context, epnt *Endpoint, input Input, config *Config) (Output, error)
+```
+
+(The `*Config` is the last argument because it is handy to create it inline when calling
+and having it last reduces readability the least.)
+
+These operations implement all the actions listed in the first requirement.
+
+The `Config` struct allows to add new optional fields to implement new functionality without
+changing the API and with minimal code refactoring efforts (seventh requirement).
+
+We're using generics to automate marshaling and umarshaling (requirement four).
+
+The internal implementation is such that we reuse code to avoid code duplication,
+thus addressing also requirement three.
+
+Additionally, whenever a call fails with a non-200 status code, the return
+value can be converted to the following type using `errors.As`:
+
+```Go
+type ErrRequestFailed struct {
+ StatusCode int
+}
+
+func (err *ErrRequestFailed) Error() string
+```
+
+Therefore, we have a way to know why a request failed (requirement six).
+
+To avoid logging bodies, one just needs to pass `model.DiscardLogger` as the
+`logger` (thus fulfilling requirement two).
+
+### Overlapped Operations
+
+The code at `./internal/httpapi` performs sequential function calls. This design
+does not interact well with the `enginenetx` package and its dial tactics. A better
+strategy is to allow calls to be overlapped. This means that, if the `enginenetx`
+is busy trying tactics for a given API endpoint, we eventually try to use the
+subsequent (semantically-equivalent) endpoint after a given time, without waiting
+for the first endpoint to complete.
+
+We allow for overlapped operations by defining these constructors:
+
+```Go
+func NewOverlappedGetJSON[Output any](config *Config) *Overlapped[Output]
+
+func NewOverlappedGetRaw(config *Config) *Overlapped[[]byte]
+
+func NewOverlappedGetXML[Output any](config *Config) *Overlapped[Output]
+
+func NewOverlappedPostJSON[Input, Output any](input Input, config *Config) *Overlapped[Output]
+```
+
+They all construct the same `*Overlapped` struct, which looks like this:
+
+```Go
+type Overlapped[Output any] struct {
+ RunFunc func(ctx context.Context, epnt *Endpoint) (Output, error)
+
+ ScheduleInterval time.Duration
+}
+```
+
+The constructor configures `RunFunc` to invoke the call corresponding to the construct
+name (i.e., `NewOverlappedGetXML` configures `RunFunc` to run `GetXML`).
+
+Then, we define the following method:
+
+```Go
+func (ovx *Overlapped[Output]) Run(ctx context.Context, epnts ...*Endpoint) (Output, error)
+```
+
+This method starts N goroutines to issue the API calls with each endpoint URL. (A classic example
+is for the URLs to be `https://0.th.ooni.org/`, `https://1.th.ooni.org/` and so on.)
+
+By default, `ScheduleInterval` is 15 seconds. If the first endpoint URL does not provide a result
+within 15 seconds, we try the second one. That is, every 15 seconds, we will attempt using
+another endpoint URL, until there's a successful response or we run out of URLs.
+
+As soon as we have a successful response, we cancel all the other pending operations
+that may exist. Once all operations have terminated, we return to the caller.
+
+### Extensibility
+
+We use the `Config` object to package common settings. Thus adding a new field, only means
+the following:
+
+1. Adding a new OPTIONAL field to `Config`.
+
+2. Honoring this field inside the internal implementation.
+
+_Et voilà_, this should allow for minimal efforts API upgrades.
+
+In fact, we used this strategy to easily add support for cloudfront in
+[probe-cli#1577](https://github.com/ooni/probe-cli/pull/1577).
+
+### Functionality Comparison
+
+This section compares side-by-side the operations performed by each implementation
+as of [probe-cli@7dab5a29812](https://github.com/ooni/probe-cli/tree/7dab5a29812) to
+show that they implement ~equivalent functionality. This should be the case, since
+the `httpxclientx` package is a refactoring of `httpapi`, which in turn contains code
+originally derived from `httpx`. Anyways, better to double check.
+
+#### GetJSON
+
+We compare to `httpapi.Call` and `httpx.GetJSONWithQuery`.
+
+| Operation | GetJSON | httpapi | httpx |
+| ------------------------- | ------- | ------- | ----- |
+| enforce a call timeout | NO | yes | NO |
+| parse base URL | NO | yes | yes |
+| join path and base URL | NO | yes | yes |
+| append query to URL | NO | yes | yes |
+| NewRequestWithContext | yes️ | yes | yes |
+| handle cloud front | yes | yes | yes |
+| set Authorization | yes | yes | yes |
+| set Accept | NO | yes | yes |
+| set User-Agent | yes ️ | yes | yes |
+| set Accept-Encoding gzip | yes️ | yes | NO |
+| (HTTPClient).Do() | yes | yes | yes |
+| defer resp.Body.Close() | yes | yes | yes |
+| handle gzip encoding | yes | yes | NO |
+| limit io.Reader | yes | yes | yes |
+| netxlite.ReadAllContext() | yes | yes | yes |
+| handle truncated body | yes | yes | NO |
+| log response body | yes | yes | yes |
+| handle non-200 response | ️ yes | yes* | yes* |
+| unmarshal JSON | yes | yes | yes |
+
+The `yes*` means that `httpapi` rejects responses with status codes `>= 400` (like cURL)
+while the new package only accepts status codes `== 200`. This difference should be of little
+practical significance for all the APIs we invoke and the new behavior is stricter.
+
+Regarding all the other cases for which `GetJSON` is marked as "NO":
+
+1. Enforcing a call timeout is better done just through the context like `httpx` does.
+
+2. `GetJSON` lets the caller completely manage the construction of the URL, so we do not need
+code to join together a base URL, possibly including a base path, a path, and a query (and we're
+introducing the new `./internal/urlx` package to handle this situation).
+
+3. Setting the `Accept` header does not seem to matter in out context because we mostly
+call API for which there's no need for content negotiation.
+
+#### GetRaw
+
+Here we're comparing to `httpapi.Call` and `httpx.FetchResource`.
+
+| Operation | GetRaw | httpapi | httpx |
+| ------------------------- | ------- | ------- | ----- |
+| enforce a call timeout | NO | yes | NO |
+| parse base URL | NO | yes | yes |
+| join path and base URL | NO | yes | yes |
+| append query to URL | NO | yes | yes |
+| NewRequestWithContext | yes️ | yes | yes |
+| handle cloud front | yes | yes | yes |
+| set Authorization | yes | yes | yes |
+| set Accept | NO | yes | yes |
+| set User-Agent | yes ️ | yes | yes |
+| set Accept-Encoding gzip | yes️ | yes | NO |
+| (HTTPClient).Do() | yes | yes | yes |
+| defer resp.Body.Close() | yes | yes | yes |
+| handle gzip encoding | yes | yes | NO |
+| limit io.Reader | yes | yes | yes |
+| netxlite.ReadAllContext() | yes | yes | yes |
+| handle truncated body | yes | yes | NO |
+| log response body | yes | yes | yes |
+| handle non-200 response | ️ yes | yes* | yes |
+
+Here we can basically make equivalent remarks as those of the previous section.
+
+#### GetXML
+
+There's no direct equivalent of `GetXML` in `httpapi` and `httpx`. Therefore, when using these
+two APIs, the caller would need to fetch a raw body and then manually parse XML.
+
+| Operation | GetXML | httpapi | httpx |
+| ------------------------- | ------- | ------- | ----- |
+| enforce a call timeout | NO | N/A | N/A |
+| parse base URL | NO | N/A | N/A |
+| join path and base URL | NO | N/A | N/A |
+| append query to URL | NO | N/A | N/A |
+| NewRequestWithContext | yes️ | N/A | N/A |
+| handle cloud front | yes | N/A | N/A |
+| set Authorization | yes | N/A | N/A |
+| set Accept | NO | N/A | N/A |
+| set User-Agent | yes ️ | N/A | N/A |
+| set Accept-Encoding gzip | yes️ | N/A | N/A |
+| (HTTPClient).Do() | yes | N/A | N/A |
+| defer resp.Body.Close() | yes | N/A | N/A |
+| handle gzip encoding | yes | N/A | N/A |
+| limit io.Reader | yes | N/A | N/A |
+| netxlite.ReadAllContext() | yes | N/A | N/A |
+| handle truncated body | yes | N/A | N/A |
+| log response body | yes | N/A | N/A |
+| handle non-200 response | ️ yes | N/A | N/A |
+| unmarshal XML | yes | N/A | N/A |
+
+Because comparison is not possible, there is not much else to say.
+
+#### PostJSON
+
+Here we're comparing to `httpapi.Call` and `httpx.PostJSON`.
+
+| Operation | PostJSON | httpapi | httpx |
+| ------------------------- | -------- | ------- | ----- |
+| marshal JSON | yes | yes~ | yes |
+| log request body | yes | yes | yes |
+| enforce a call timeout | NO | yes | NO |
+| parse base URL | NO | yes | yes |
+| join path and base URL | NO | yes | yes |
+| append query to URL | NO | yes | yes |
+| NewRequestWithContext | yes️ | yes | yes |
+| handle cloud front | yes | yes | yes |
+| set Authorization | yes | yes | yes |
+| set Accept | NO | yes | yes |
+| set Content-Type | yes | yes | yes |
+| set User-Agent | yes ️ | yes | yes |
+| set Accept-Encoding gzip | yes️ | yes | NO |
+| (HTTPClient).Do() | yes | yes | yes |
+| defer resp.Body.Close() | yes | yes | yes |
+| handle gzip encoding | yes | yes | NO |
+| limit io.Reader | yes | yes | yes |
+| netxlite.ReadAllContext() | yes | yes | yes |
+| handle truncated body | yes | yes | NO |
+| log response body | yes | yes | yes |
+| handle non-200 response | ️ yes | yes* | yes* |
+| unmarshal JSON | yes | yes | yes |
+
+The `yes*` means that `httpapi` rejects responses with status codes `>= 400` (like cURL)
+while the new package only accepts status codes `== 200`. This difference should be of little
+practical significance for all the APIs we invoke and the new behavior is stricter.
+
+The `yes~` means that `httpapi` already receives a marshaled body from a higher-level API
+that is part of the same package, while in this package we marshal in `PostJSON`.
+
+## Nil Safety
+
+Consider the following code snippet:
+
+```Go
+resp, err := httpclientx.GetJSON[*APIResponse](ctx, epnt, config)
+runtimex.Assert((resp == nil && err != nil) || (resp != nil && err == nil), "ouch")
+```
+
+Now, consider the case where `URL` refers to a server that returns `null` as the JSON
+answer, rather than returning a JSON object. The `encoding/json` package will accept the
+`null` value and unmarshal it into a `nil` pointer. So, `GetJSON` will return `nil` and
+`nil`, and the `runtimex.Assert` will fail.
+
+The `httpx` package did not have this issue because the usage pattern was:
+
+```Go
+var resp APIResponse
+err := apiClient.GetJSON(ctx, "/foobar", &resp) // where apiClient implements httpx.APIClient
+```
+
+In such a case, the `null` would have no effect and `resp` would be an empty response.
+
+However, it is still handy to return a value and an error, and it is the most commonly used
+pattern in Go and, as a result, in OONI Probe. So, what do we do?
+
+Well, here's the strategy:
+
+1. When sending pointers, slices, or maps in `PostJSON`, we return `ErrIsNil` if the pointer,
+slice, or map is `nil`, to avoid sending literal `null` to servers.
+
+2. `GetJSON`, `GetXML`, and `PostJSON` include checks after unmarshaling so that, if the API response
+type is a slice, pointer, or map, and it is `nil`, we also return `ErrIsNil`.
+
+Strictly speaking, it is still unclear to us whether this could happen with `GetXML` but we have
+decided to implements these checks for `GetXML` as well, just in case.
+
+## Refactoring Plan
+
+The overall goal is to replace usages of `httpapi` and `httpx` with usages of `httpclient`.
+
+The following packages use `httpapi`:
+
+1. `internal/experiment/webconnectivity`: uses `httpapi.SeqCaller` to chain calls to all
+the available test helpers, which we can replace with using `*Overlapped`;
+
+2. `internal/experiment/webconnectivitylte`: same as above;
+
+3. `internal/ooapi`: uses `httpapi` to define the OONI backend APIs for the purpose of
+generating and cross-validating swaggers, which is something we defined as a non-goal given
+that we never really managed to do it reliably, and it has only led to code complexity;
+
+4. `internal/probeservices`: uses `httpapi` to implement check-in and the main reason why
+this is the case is because it supports `"gzip"` encoding;
+
+The following packages use `httpx`:
+
+1. `internal/cmd/apitool`: this is just a byproduct of `probeservices.Client` embedding
+the `httpx.APIClientTemplate`, so this should really be easy to get rid of;
+
+2. `internal/enginelocate`: we're using `httpx` convenience functions to figure out
+the probe IP and we can easily replace these calls with `httpclientx`;
+
+3. `internal/oonirun`: uses `httpx` to fetch descriptors and can be easily replaced;
+
+4. `internal/probeservices`: uses `httpx` for most other calls.
+
+Based on the above information, it seems the easiest way to proceed is this:
+
+1. `internal/enginelocate`: replace `httpx` with `httpclientx`;
+
+2. `internal/oonirun`: replace `httpx` with `httpclientx`;
+
+3. `internal/probeservices`: replace the check-in implementation to use `httpclientx`
+instead of using the `httpapi` package;
+
+4. `internal/experiment/webconnectivity{,lte}`: replace the `httpapi.SeqCaller` usage
+with invocations of `*Overlapped`;
+
+5. remove the `httpapi` and `ooapi` packages, now unused;
+
+6. finish replacing `httpx` with `httpclientx` in `internal/probeservices`
+
+7. remove the `httpx` package.
+
+## Limitations and Future Work
+
+The current implementation of `*Overlapped` may cause us to do more work than needed in
+case the network is really slow and an attempt is slowly fetching the body. In such a case,
+starting a new attempt duplicates work. Handling this case does not seem straightforward
+currently, therefore, we will focus on this as part of future work.
diff --git a/pkg/httpclientx/config.go b/pkg/httpclientx/config.go
new file mode 100644
index 000000000..2fc33cff4
--- /dev/null
+++ b/pkg/httpclientx/config.go
@@ -0,0 +1,35 @@
+package httpclientx
+
+import "github.com/ooni/probe-engine/pkg/model"
+
+// Config contains configuration shared by [GetJSON], [GetXML], [GetRaw], and [PostJSON].
+//
+// The zero value is invalid; initialize the MANDATORY fields.
+type Config struct {
+ // Authorization contains the OPTIONAL Authorization header value to use.
+ Authorization string
+
+ // Client is the MANDATORY [model.HTTPClient] to use.
+ Client model.HTTPClient
+
+ // Logger is the MANDATORY [model.Logger] to use.
+ Logger model.Logger
+
+ // MaxResponseBodySize OPTIONALLY limits the maximum body size. If not set, we
+ // use the [DefaultMaxResponseBodySize] value.
+ MaxResponseBodySize int64
+
+ // UserAgent is the MANDATORY User-Agent header value to use.
+ UserAgent string
+}
+
+// DefaultMaxResponseBodySize is the default maximum response body size.
+const DefaultMaxResponseBodySize = 1 << 24
+
+func (c *Config) maxResponseBodySize() (value int64) {
+ value = c.MaxResponseBodySize
+ if value <= 0 {
+ value = DefaultMaxResponseBodySize
+ }
+ return
+}
diff --git a/pkg/httpclientx/config_test.go b/pkg/httpclientx/config_test.go
new file mode 100644
index 000000000..2c2674317
--- /dev/null
+++ b/pkg/httpclientx/config_test.go
@@ -0,0 +1,21 @@
+package httpclientx
+
+import "testing"
+
+func TestConfigMaxResponseBodySize(t *testing.T) {
+ t.Run("the default returned value corresponds to the constant default", func(t *testing.T) {
+ config := &Config{}
+ if value := config.maxResponseBodySize(); value != DefaultMaxResponseBodySize {
+ t.Fatal("unexpected maxResponseBodySize()", value)
+ }
+ })
+
+ t.Run("we can override the default", func(t *testing.T) {
+ config := &Config{}
+ const expectedValue = DefaultMaxResponseBodySize / 2
+ config.MaxResponseBodySize = expectedValue
+ if value := config.maxResponseBodySize(); value != expectedValue {
+ t.Fatal("unexpected maxResponseBodySize()", value)
+ }
+ })
+}
diff --git a/pkg/httpclientx/endpoint.go b/pkg/httpclientx/endpoint.go
new file mode 100644
index 000000000..fa34a6633
--- /dev/null
+++ b/pkg/httpclientx/endpoint.go
@@ -0,0 +1,49 @@
+package httpclientx
+
+import "github.com/ooni/probe-engine/pkg/model"
+
+// Endpoint is an HTTP endpoint.
+//
+// The zero value is invalid; construct using [NewEndpoint].
+type Endpoint struct {
+ // URL is the MANDATORY endpoint URL.
+ URL string
+
+ // Host is the OPTIONAL host header to use for cloudfronting.
+ Host string
+}
+
+// NewEndpoint constructs a new [*Endpoint] instance using the given URL.
+func NewEndpoint(URL string) *Endpoint {
+ return &Endpoint{
+ URL: URL,
+ Host: "",
+ }
+}
+
+// WithHostOverride returns a copy of the [*Endpoint] using the given host header override.
+func (e *Endpoint) WithHostOverride(host string) *Endpoint {
+ return &Endpoint{
+ URL: e.URL,
+ Host: host,
+ }
+}
+
+// NewEndpointFromModelOOAPIServices constructs new [*Endpoint] instances from the
+// given [model.OOAPIService] instances, assigning the host header if "cloudfront", and
+// skipping all the entries that are neither "https" not "cloudfront".
+func NewEndpointFromModelOOAPIServices(svcs ...model.OOAPIService) (epnts []*Endpoint) {
+ for _, svc := range svcs {
+ epnt := NewEndpoint(svc.Address)
+ switch svc.Type {
+ case "cloudfront":
+ epnt = epnt.WithHostOverride(svc.Front)
+ fallthrough
+ case "https":
+ epnts = append(epnts, epnt)
+ default:
+ // skip entry
+ }
+ }
+ return
+}
diff --git a/pkg/httpclientx/endpoint_test.go b/pkg/httpclientx/endpoint_test.go
new file mode 100644
index 000000000..fee3ec8ef
--- /dev/null
+++ b/pkg/httpclientx/endpoint_test.go
@@ -0,0 +1,65 @@
+package httpclientx
+
+import (
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/model"
+)
+
+func TestEndpoint(t *testing.T) {
+ t.Run("the constructor only assigns the URL", func(t *testing.T) {
+ epnt := NewEndpoint("https://www.example.com/")
+ if epnt.URL != "https://www.example.com/" {
+ t.Fatal("unexpected URL")
+ }
+ if epnt.Host != "" {
+ t.Fatal("unexpected host")
+ }
+ })
+
+ t.Run("we can optionally get a copy with an assigned host header", func(t *testing.T) {
+ epnt := NewEndpoint("https://www.example.com/").WithHostOverride("www.cloudfront.com")
+ if epnt.URL != "https://www.example.com/" {
+ t.Fatal("unexpected URL")
+ }
+ if epnt.Host != "www.cloudfront.com" {
+ t.Fatal("unexpected host")
+ }
+ })
+
+ t.Run("we can convert from model.OOAPIService", func(t *testing.T) {
+ services := []model.OOAPIService{{
+ Address: "",
+ Type: "onion",
+ Front: "",
+ }, {
+ Address: "https://www.example.com/",
+ Type: "https",
+ }, {
+ Address: "",
+ Type: "onion",
+ Front: "",
+ }, {
+ Address: "https://www.example.com/",
+ Type: "cloudfront",
+ Front: "www.cloudfront.com",
+ }, {
+ Address: "",
+ Type: "onion",
+ Front: "",
+ }}
+
+ expect := []*Endpoint{{
+ URL: "https://www.example.com/",
+ }, {
+ URL: "https://www.example.com/",
+ Host: "www.cloudfront.com",
+ }}
+
+ got := NewEndpointFromModelOOAPIServices(services...)
+ if diff := cmp.Diff(expect, got); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+}
diff --git a/pkg/httpclientx/getjson.go b/pkg/httpclientx/getjson.go
new file mode 100644
index 000000000..346c52423
--- /dev/null
+++ b/pkg/httpclientx/getjson.go
@@ -0,0 +1,44 @@
+package httpclientx
+
+//
+// getjson.go - GET a JSON response.
+//
+
+import (
+ "context"
+ "encoding/json"
+)
+
+// GetJSON sends a GET request and reads a JSON response.
+//
+// Arguments:
+//
+// - ctx is the cancellable context;
+//
+// - epnt is the HTTP [*Endpoint] to use;
+//
+// - config contains the config.
+//
+// This function either returns an error or a valid Output.
+func GetJSON[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error) {
+ return OverlappedIgnoreIndex(NewOverlappedGetJSON[Output](config).Run(ctx, epnt))
+}
+
+func getJSON[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error) {
+ // read the raw body
+ rawrespbody, err := GetRaw(ctx, epnt, config)
+
+ // handle the case of error
+ if err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // parse the response body as JSON
+ var output Output
+ if err := json.Unmarshal(rawrespbody, &output); err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // avoid returning nil pointers, maps, slices
+ return NilSafetyErrorIfNil(output)
+}
diff --git a/pkg/httpclientx/getjson_test.go b/pkg/httpclientx/getjson_test.go
new file mode 100644
index 000000000..d5f03c677
--- /dev/null
+++ b/pkg/httpclientx/getjson_test.go
@@ -0,0 +1,321 @@
+package httpclientx
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/netxlite"
+ "github.com/ooni/probe-engine/pkg/testingx"
+)
+
+type apiResponse struct {
+ Age int
+ Name string
+}
+
+func TestGetJSON(t *testing.T) {
+ t.Run("when GetRaw fails", func(t *testing.T) {
+ // create a server that RST connections
+ server := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetJSON[*apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is nil.
+ if resp != nil {
+ t.Fatal("expected nil response")
+ }
+ })
+
+ t.Run("when JSON parsing fails", func(t *testing.T) {
+ // create a server that returns an invalid JSON type
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte("[]"))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetJSON[*apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if err.Error() != "json: cannot unmarshal array into Go value of type httpclientx.apiResponse" {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is nil.
+ if resp != nil {
+ t.Fatal("expected nil response")
+ }
+ })
+
+ t.Run("on success", func(t *testing.T) {
+ // create a server that returns a legit response
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"Name": "simone", "Age": 41}`))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetJSON[*apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is OK
+ expect := &apiResponse{Name: "simone", Age: 41}
+ if diff := cmp.Diff(expect, resp); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+}
+
+// This test ensures that GetJSON sets correct HTTP headers
+func TestGetJSONHeadersOkay(t *testing.T) {
+ var (
+ gothost string
+ gotheaders http.Header
+ gotmu sync.Mutex
+ )
+
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // save the headers
+ gotmu.Lock()
+ gothost = r.Host
+ gotheaders = r.Header
+ gotmu.Unlock()
+
+ // send a minimal 200 Ok response
+ w.WriteHeader(200)
+ w.Write([]byte(`{}`))
+ }))
+ defer server.Close()
+
+ // send the request and receive the response
+ apiresp, err := GetJSON[*apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL).WithHostOverride("www.cloudfront.com"),
+ &Config{
+ Authorization: "scribai",
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ // we do not expect to see an error here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // given the handler, we expect to see an empty structure here
+ if apiresp.Age != 0 || apiresp.Name != "" {
+ t.Fatal("expected empty response")
+ }
+
+ // make sure there are no data races
+ defer gotmu.Unlock()
+ gotmu.Lock()
+
+ // make sure we have sent the authorization header
+ if value := gotheaders.Get("Authorization"); value != "scribai" {
+ t.Fatal("unexpected Authorization value", value)
+ }
+
+ // now make sure we have sent user-agent
+ if value := gotheaders.Get("User-Agent"); value != model.HTTPHeaderUserAgent {
+ t.Fatal("unexpected User-Agent value", value)
+ }
+
+ // now make sure we have sent accept-encoding
+ if value := gotheaders.Get("Accept-Encoding"); value != "gzip" {
+ t.Fatal("unexpected Accept-Encoding value", value)
+ }
+
+ // now make sure we could use cloudfronting
+ if gothost != "www.cloudfront.com" {
+ t.Fatal("unexpected Host value", gothost)
+ }
+}
+
+// This test ensures GetJSON logs the response body at Debug level.
+func TestGetJSONLoggingOkay(t *testing.T) {
+ // create a server that returns a legit response
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"Name": "simone", "Age": 41}`))
+ }))
+ defer server.Close()
+
+ // instantiate a logger that collects logs
+ logger := &testingx.Logger{}
+
+ // invoke the API
+ resp, err := GetJSON[*apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: logger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is OK
+ expect := &apiResponse{Name: "simone", Age: 41}
+ if diff := cmp.Diff(expect, resp); diff != "" {
+ t.Fatal(diff)
+ }
+
+ // collect and verify the debug lines
+ debuglines := logger.DebugLines()
+ t.Log(debuglines)
+ if len(debuglines) != 1 {
+ t.Fatal("expected to see a single debug line")
+ }
+ if !strings.Contains(debuglines[0], "raw response body:") {
+ t.Fatal("did not see raw response body log line")
+ }
+}
+
+// TestGetJSONCorrectlyRejectsNilValues ensures we correctly reject nil values.
+func TestGetJSONCorrectlyRejectsNilValues(t *testing.T) {
+
+ t.Run("when unmarshaling into a map", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`null`))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetJSON[map[string]string](
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when unmarshaling into a struct pointer", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`null`))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetJSON[*apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when unmarshaling into a slice", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`null`))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetJSON[[]string](
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+}
diff --git a/pkg/httpclientx/getraw.go b/pkg/httpclientx/getraw.go
new file mode 100644
index 000000000..da155215a
--- /dev/null
+++ b/pkg/httpclientx/getraw.go
@@ -0,0 +1,36 @@
+package httpclientx
+
+//
+// getraw.go - GET a raw response.
+//
+
+import (
+ "context"
+ "net/http"
+)
+
+// GetRaw sends a GET request and reads a raw response.
+//
+// Arguments:
+//
+// - ctx is the cancellable context;
+//
+// - epnt is the HTTP [*Endpoint] to use;
+//
+// - config is the config to use.
+//
+// This function either returns an error or a valid Output.
+func GetRaw(ctx context.Context, epnt *Endpoint, config *Config) ([]byte, error) {
+ return OverlappedIgnoreIndex(NewOverlappedGetRaw(config).Run(ctx, epnt))
+}
+
+func getRaw(ctx context.Context, epnt *Endpoint, config *Config) ([]byte, error) {
+ // construct the request to use
+ req, err := http.NewRequestWithContext(ctx, "GET", epnt.URL, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ // get raw response body
+ return do(ctx, req, epnt, config)
+}
diff --git a/pkg/httpclientx/getraw_test.go b/pkg/httpclientx/getraw_test.go
new file mode 100644
index 000000000..6eb1b1ef5
--- /dev/null
+++ b/pkg/httpclientx/getraw_test.go
@@ -0,0 +1,181 @@
+package httpclientx
+
+import (
+ "context"
+ "net/http"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/testingx"
+)
+
+func TestGetRaw(t *testing.T) {
+ t.Run("when we cannot create a request", func(t *testing.T) {
+ // create API call config
+
+ rawrespbody, err := GetRaw(
+ context.Background(),
+ NewEndpoint("\t"), // <- invalid URL that we cannot parse
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ },
+ )
+
+ t.Log(rawrespbody)
+ t.Log(err)
+
+ if err.Error() != `parse "\t": net/url: invalid control character in URL` {
+ t.Fatal("unexpected error", err)
+ }
+
+ if len(rawrespbody) != 0 {
+ t.Fatal("expected zero-length body")
+ }
+ })
+
+ t.Run("on success", func(t *testing.T) {
+ expected := []byte(`Bonsoir, Elliot`)
+
+ // create a server that returns a legit response
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write(expected)
+ }))
+ defer server.Close()
+
+ rawrespbody, err := GetRaw(
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(rawrespbody)
+ t.Log(err)
+
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ if diff := cmp.Diff(expected, rawrespbody); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+}
+
+// This test ensures that GetRaw sets correct HTTP headers
+func TestGetRawHeadersOkay(t *testing.T) {
+ var (
+ gothost string
+ gotheaders http.Header
+ gotmu sync.Mutex
+ )
+
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // save the headers
+ gotmu.Lock()
+ gothost = r.Host
+ gotheaders = r.Header
+ gotmu.Unlock()
+
+ // send a minimal 200 Ok response
+ w.WriteHeader(200)
+ w.Write([]byte(``))
+ }))
+ defer server.Close()
+
+ // send the request and receive the response
+ rawresp, err := GetRaw(
+ context.Background(),
+ NewEndpoint(server.URL).WithHostOverride("www.cloudfront.com"),
+ &Config{
+ Authorization: "scribai",
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ // we do not expect to see an error here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure the raw response is exactly what we expect to receive
+ if diff := cmp.Diff([]byte(``), rawresp); diff != "" {
+ t.Fatal("unexpected raw response")
+ }
+
+ // make sure there are no data races
+ defer gotmu.Unlock()
+ gotmu.Lock()
+
+ // make sure we have sent the authorization header
+ if value := gotheaders.Get("Authorization"); value != "scribai" {
+ t.Fatal("unexpected Authorization value", value)
+ }
+
+ // now make sure we have sent user-agent
+ if value := gotheaders.Get("User-Agent"); value != model.HTTPHeaderUserAgent {
+ t.Fatal("unexpected User-Agent value", value)
+ }
+
+ // now make sure we have sent accept-encoding
+ if value := gotheaders.Get("Accept-Encoding"); value != "gzip" {
+ t.Fatal("unexpected Accept-Encoding value", value)
+ }
+
+ // now make sure we could use cloudfronting
+ if gothost != "www.cloudfront.com" {
+ t.Fatal("unexpected Host value", gothost)
+ }
+}
+
+// This test ensures GetRaw logs the response body at Debug level.
+func TestGetRawLoggingOkay(t *testing.T) {
+ expected := []byte(`Bonsoir, Elliot`)
+
+ // create a server that returns a legit response
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write(expected)
+ }))
+ defer server.Close()
+
+ // instantiate a logger that collects logs
+ logger := &testingx.Logger{}
+
+ rawrespbody, err := GetRaw(
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: logger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(rawrespbody)
+ t.Log(err)
+
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ if diff := cmp.Diff(expected, rawrespbody); diff != "" {
+ t.Fatal(diff)
+ }
+
+ // collect and verify the debug lines
+ debuglines := logger.DebugLines()
+ t.Log(debuglines)
+ if len(debuglines) != 1 {
+ t.Fatal("expected to see a single debug line")
+ }
+ if !strings.Contains(debuglines[0], "raw response body:") {
+ t.Fatal("did not see raw response body log line")
+ }
+}
diff --git a/pkg/httpclientx/getxml.go b/pkg/httpclientx/getxml.go
new file mode 100644
index 000000000..8163f3df3
--- /dev/null
+++ b/pkg/httpclientx/getxml.go
@@ -0,0 +1,48 @@
+package httpclientx
+
+//
+// getxml.go - GET an XML response.
+//
+
+import (
+ "context"
+ "encoding/xml"
+)
+
+// GetXML sends a GET request and reads an XML response.
+//
+// Arguments:
+//
+// - ctx is the cancellable context;
+//
+// - epnt is the HTTP [*Endpoint] to use;
+//
+// - config is the config to use.
+//
+// This function either returns an error or a valid Output.
+func GetXML[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error) {
+ return OverlappedIgnoreIndex(NewOverlappedGetXML[Output](config).Run(ctx, epnt))
+}
+
+func getXML[Output any](ctx context.Context, epnt *Endpoint, config *Config) (Output, error) {
+ // read the raw body
+ rawrespbody, err := GetRaw(ctx, epnt, config)
+
+ // handle the case of error
+ if err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // parse the response body as JSON
+ var output Output
+ if err := xml.Unmarshal(rawrespbody, &output); err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // TODO(bassosimone): it's unclear to me whether output can be nil when unmarshaling
+ // XML input, since there is no "null" in XML. In any case, the code below checks for
+ // and avoids emitting nil, so I guess we should be fine here.
+
+ // avoid returning nil pointers, maps, slices
+ return NilSafetyErrorIfNil(output)
+}
diff --git a/pkg/httpclientx/getxml_test.go b/pkg/httpclientx/getxml_test.go
new file mode 100644
index 000000000..fa207f64d
--- /dev/null
+++ b/pkg/httpclientx/getxml_test.go
@@ -0,0 +1,223 @@
+package httpclientx
+
+import (
+ "context"
+ "errors"
+ "io"
+ "net/http"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/netxlite"
+ "github.com/ooni/probe-engine/pkg/testingx"
+)
+
+func TestGetXML(t *testing.T) {
+ t.Run("when GetRaw fails", func(t *testing.T) {
+ // create a server that RST connections
+ server := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetXML[*apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is nil.
+ if resp != nil {
+ t.Fatal("expected nil response")
+ }
+ })
+
+ t.Run("when XML parsing fails", func(t *testing.T) {
+ // create a server that returns an invalid XML file
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte("[]"))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetXML[*apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, io.EOF) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is nil.
+ if resp != nil {
+ t.Fatal("expected nil response")
+ }
+ })
+
+ t.Run("on success", func(t *testing.T) {
+ // create a server that returns a legit response
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`simone41`))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetXML[*apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is OK
+ expect := &apiResponse{Name: "simone", Age: 41}
+ if diff := cmp.Diff(expect, resp); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+}
+
+// This test ensures that GetXML sets correct HTTP headers
+func TestGetXMLHeadersOkay(t *testing.T) {
+ var (
+ gothost string
+ gotheaders http.Header
+ gotmu sync.Mutex
+ )
+
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // save the headers
+ gotmu.Lock()
+ gothost = r.Host
+ gotheaders = r.Header
+ gotmu.Unlock()
+
+ // send a minimal 200 Ok response
+ w.WriteHeader(200)
+ w.Write([]byte(``))
+ }))
+ defer server.Close()
+
+ // send the request and receive the response
+ apiresp, err := GetXML[*apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL).WithHostOverride("www.cloudfront.com"),
+ &Config{
+ Authorization: "scribai",
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ // we do not expect to see an error here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // given the handler, we expect to see an empty structure here
+ if apiresp.Age != 0 || apiresp.Name != "" {
+ t.Fatal("expected empty response")
+ }
+
+ // make sure there are no data races
+ defer gotmu.Unlock()
+ gotmu.Lock()
+
+ // make sure we have sent the authorization header
+ if value := gotheaders.Get("Authorization"); value != "scribai" {
+ t.Fatal("unexpected Authorization value", value)
+ }
+
+ // now make sure we have sent user-agent
+ if value := gotheaders.Get("User-Agent"); value != model.HTTPHeaderUserAgent {
+ t.Fatal("unexpected User-Agent value", value)
+ }
+
+ // now make sure we have sent accept-encoding
+ if value := gotheaders.Get("Accept-Encoding"); value != "gzip" {
+ t.Fatal("unexpected Accept-Encoding value", value)
+ }
+
+ // now make sure we could use cloudfronting
+ if gothost != "www.cloudfront.com" {
+ t.Fatal("unexpected Host value", gothost)
+ }
+}
+
+// This test ensures GetXML logs the response body at Debug level.
+func TestGetXMLLoggingOkay(t *testing.T) {
+ // create a server that returns a legit response
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`simone41`))
+ }))
+ defer server.Close()
+
+ // instantiate a logger that collects logs
+ logger := &testingx.Logger{}
+
+ // invoke the API
+ resp, err := GetXML[*apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: logger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is OK
+ expect := &apiResponse{Name: "simone", Age: 41}
+ if diff := cmp.Diff(expect, resp); diff != "" {
+ t.Fatal(diff)
+ }
+
+ // collect and verify the debug lines
+ debuglines := logger.DebugLines()
+ t.Log(debuglines)
+ if len(debuglines) != 1 {
+ t.Fatal("expected to see a single debug line")
+ }
+ if !strings.Contains(debuglines[0], "raw response body:") {
+ t.Fatal("did not see raw response body log line")
+ }
+}
diff --git a/pkg/httpclientx/httpclientx.go b/pkg/httpclientx/httpclientx.go
new file mode 100644
index 000000000..79e2cede4
--- /dev/null
+++ b/pkg/httpclientx/httpclientx.go
@@ -0,0 +1,116 @@
+// Package httpclientx contains extensions to more easily invoke HTTP APIs.
+package httpclientx
+
+//
+// httpclientx.go - common code
+//
+
+import (
+ "compress/gzip"
+ "context"
+ "errors"
+ "io"
+ "net/http"
+
+ "github.com/ooni/probe-engine/pkg/netxlite"
+)
+
+// ErrRequestFailed indicates that an HTTP request status indicates failure.
+type ErrRequestFailed struct {
+ StatusCode int
+}
+
+var _ error = &ErrRequestFailed{}
+
+// Error returns the error as a string.
+//
+// The string returned by this error starts with the httpx prefix for backwards
+// compatibility with the legacy httpx package.
+func (err *ErrRequestFailed) Error() string {
+ return "httpx: request failed"
+}
+
+// zeroValue is a convenience function to return the zero value.
+func zeroValue[T any]() T {
+ return *new(T)
+}
+
+// ErrTruncated indicates we truncated the response body.
+//
+// Note: we SHOULD NOT change the error string because this error string was previously
+// used by the httpapi package and it's better to keep the same strings.
+var ErrTruncated = errors.New("httpapi: truncated response body")
+
+// do is the internal function to finish preparing the request and getting a raw response.
+func do(ctx context.Context, req *http.Request, epnt *Endpoint, config *Config) ([]byte, error) {
+ // optionally assign authorization
+ if value := config.Authorization; value != "" {
+ req.Header.Set("Authorization", value)
+ }
+
+ // assign the user agent
+ req.Header.Set("User-Agent", config.UserAgent)
+
+ // say that we're accepting gzip encoded bodies
+ req.Header.Set("Accept-Encoding", "gzip")
+
+ // OPTIONALLY allow for cloudfronting (the default in net/http is for
+ // the req.Host to be empty and to use req.URL.Host)
+ req.Host = epnt.Host
+
+ // get the response
+ resp, err := config.Client.Do(req)
+
+ // handle the case of error
+ if err != nil {
+ return nil, err
+ }
+
+ // eventually close the response body
+ defer resp.Body.Close()
+
+ // Implementation note: here we choose to always read the response
+ // body before checking the status code because it helps a lot to log
+ // the response body received on failure when testing a backend
+
+ var baseReader io.Reader = resp.Body
+
+ // handle the case of gzip encoded body
+ if resp.Header.Get("Content-Encoding") == "gzip" {
+ gzreader, err := gzip.NewReader(baseReader)
+ if err != nil {
+ return nil, err
+ }
+ baseReader = gzreader
+ }
+
+ // protect against unreasonably large response bodies
+ //
+ // read one more byte than the maximum allowed size so we can
+ // always tell whether it was truncated here
+ limitReader := io.LimitReader(baseReader, config.maxResponseBodySize()+1)
+
+ // read the raw body
+ rawrespbody, err := netxlite.ReadAllContext(ctx, limitReader)
+
+ // handle the case of failure
+ if err != nil {
+ return nil, err
+ }
+
+ // handle the case of truncated body
+ if int64(len(rawrespbody)) > config.maxResponseBodySize() {
+ return nil, ErrTruncated
+ }
+
+ // log the response body for debugging purposes
+ config.Logger.Debugf("%s %s: raw response body: %s", req.Method, req.URL.String(), string(rawrespbody))
+
+ // handle the case of HTTP error
+ if resp.StatusCode != 200 {
+ return nil, &ErrRequestFailed{resp.StatusCode}
+ }
+
+ // make sure we replace a nil slice with an empty slice
+ return NilSafetyAvoidNilBytesSlice(rawrespbody), nil
+}
diff --git a/pkg/httpclientx/httpclientx_test.go b/pkg/httpclientx/httpclientx_test.go
new file mode 100644
index 000000000..ac1409bf0
--- /dev/null
+++ b/pkg/httpclientx/httpclientx_test.go
@@ -0,0 +1,255 @@
+package httpclientx
+
+import (
+ "bytes"
+ "compress/gzip"
+ "context"
+ "errors"
+ "net/http"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/netxlite"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/testingx"
+)
+
+// createGzipBomb creates a gzip bomb with the given size.
+func createGzipBomb(size int) []byte {
+ input := make([]byte, size)
+ runtimex.Assert(len(input) == size, "unexpected input length")
+ var buf bytes.Buffer
+ gz := runtimex.Try1(gzip.NewWriterLevel(&buf, gzip.BestCompression))
+ _ = runtimex.Try1(gz.Write(input))
+ runtimex.Try0(gz.Close())
+ return buf.Bytes()
+}
+
+// gzipBomb is a gzip bomb containing 1 megabyte of zeroes
+var gzipBomb = createGzipBomb(1 << 20)
+
+func TestGzipDecompression(t *testing.T) {
+ t.Run("we correctly handle gzip encoding", func(t *testing.T) {
+ expected := []byte(`Bonsoir, Elliot!!!`)
+
+ // create a server returning compressed content
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var buffer bytes.Buffer
+ writer := gzip.NewWriter(&buffer)
+ _ = runtimex.Try1(writer.Write(expected))
+ runtimex.Try0(writer.Close())
+ w.Header().Add("Content-Encoding", "gzip")
+ w.Write(buffer.Bytes())
+ }))
+ defer server.Close()
+
+ // make sure we can read it
+ respbody, err := GetRaw(
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(respbody)
+ t.Log(err)
+
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if diff := cmp.Diff(expected, respbody); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+
+ t.Run("we correctly handle the case where we cannot decode gzip", func(t *testing.T) {
+ expected := []byte(`Bonsoir, Elliot!!!`)
+
+ // create a server pretending to return compressed content
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Add("Content-Encoding", "gzip")
+ w.Write(expected)
+ }))
+ defer server.Close()
+
+ // attempt to get a response body
+ respbody, err := GetRaw(
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(respbody)
+ t.Log(err)
+
+ if err.Error() != "gzip: invalid header" {
+ t.Fatal(err)
+ }
+
+ if respbody != nil {
+ t.Fatal("expected nil response body")
+ }
+ })
+
+ t.Run("we can correctly decode a large body", func(t *testing.T) {
+ // create a server returning compressed content
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Add("Content-Encoding", "gzip")
+ w.Write(gzipBomb)
+ }))
+ defer server.Close()
+
+ // make sure we can read it
+ respbody, err := GetRaw(
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ //t.Log(respbody) // maybe this operation is a bit expensive to be the default
+ t.Log(err)
+
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if length := len(respbody); length != 1<<20 {
+ t.Fatal("unexpected response body length", length)
+ }
+ })
+}
+
+func TestHTTPStatusCodeHandling(t *testing.T) {
+ server := testingx.MustNewHTTPServer(testingx.HTTPHandlerBlockpage451())
+ defer server.Close()
+
+ respbody, err := GetRaw(
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(respbody)
+ t.Log(err)
+
+ if err.Error() != "httpx: request failed" {
+ t.Fatal(err)
+ }
+
+ if respbody != nil {
+ t.Fatal("expected nil response body")
+ }
+
+ var orig *ErrRequestFailed
+ if !errors.As(err, &orig) {
+ t.Fatal("not an *ErrRequestFailed instance")
+ }
+ if orig.StatusCode != 451 {
+ t.Fatal("unexpected status code", orig.StatusCode)
+ }
+}
+
+func TestHTTPReadBodyErrorsHandling(t *testing.T) {
+ server := testingx.MustNewHTTPServer(testingx.HTTPHandlerResetWhileReadingBody())
+ defer server.Close()
+
+ respbody, err := GetRaw(
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(respbody)
+ t.Log(err)
+
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("expected ECONNRESET, got", err)
+ }
+
+ if respbody != nil {
+ t.Fatal("expected nil response body")
+ }
+}
+
+func TestLimitMaximumBodySize(t *testing.T) {
+ t.Run("we can correctly avoid receiving a large body when uncompressed", func(t *testing.T) {
+ // create a server returning uncompressed content
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write(make([]byte, 1<<20))
+ }))
+ defer server.Close()
+
+ // make sure we can read it
+ //
+ // note: here we're using a small max body size, definitely smaller than what we send
+ respbody, err := GetRaw(
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ MaxResponseBodySize: 1 << 10,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(respbody)
+ t.Log(err)
+
+ if !errors.Is(err, ErrTruncated) {
+ t.Fatal("unexpected error", err)
+ }
+
+ if len(respbody) != 0 {
+ t.Fatal("expected zero length response body length")
+ }
+ })
+
+ t.Run("we can correctly avoid receiving a large body when compressed", func(t *testing.T) {
+ // create a server returning compressed content
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Add("Content-Encoding", "gzip")
+ w.Write(gzipBomb)
+ }))
+ defer server.Close()
+
+ // make sure we can read it
+ //
+ // note: here we're using a small max body size, definitely smaller than the gzip bomb
+ respbody, err := GetRaw(
+ context.Background(),
+ NewEndpoint(server.URL),
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ MaxResponseBodySize: 1 << 10,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(respbody)
+ t.Log(err)
+
+ if !errors.Is(err, ErrTruncated) {
+ t.Fatal("unexpected error", err)
+ }
+
+ if len(respbody) != 0 {
+ t.Fatal("expected zero length response body length")
+ }
+ })
+}
diff --git a/pkg/httpclientx/nilsafety.go b/pkg/httpclientx/nilsafety.go
new file mode 100644
index 000000000..c73323cb1
--- /dev/null
+++ b/pkg/httpclientx/nilsafety.go
@@ -0,0 +1,31 @@
+package httpclientx
+
+import (
+ "errors"
+ "reflect"
+)
+
+// ErrIsNil indicates that [NilSafetyErrorIfNil] was passed a nil value.
+var ErrIsNil = errors.New("nil map, pointer, or slice")
+
+// NilSafetyErrorIfNil returns [ErrIsNil] iff input is a nil map, struct, or slice.
+//
+// This mechanism prevents us from mistakenly sending to a server a literal JSON "null" and
+// protects us from attempting to process a literal JSON "null" from a server.
+func NilSafetyErrorIfNil[Type any](value Type) (Type, error) {
+ switch rv := reflect.ValueOf(value); rv.Kind() {
+ case reflect.Map, reflect.Pointer, reflect.Slice:
+ if rv.IsNil() {
+ return zeroValue[Type](), ErrIsNil
+ }
+ }
+ return value, nil
+}
+
+// NilSafetyAvoidNilBytesSlice replaces a nil bytes slice with an empty slice.
+func NilSafetyAvoidNilBytesSlice(input []byte) []byte {
+ if input == nil {
+ input = []byte{}
+ }
+ return input
+}
diff --git a/pkg/httpclientx/nilsafety_test.go b/pkg/httpclientx/nilsafety_test.go
new file mode 100644
index 000000000..85551266e
--- /dev/null
+++ b/pkg/httpclientx/nilsafety_test.go
@@ -0,0 +1,122 @@
+package httpclientx
+
+import (
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+)
+
+func TestNilSafetyErrorIfNil(t *testing.T) {
+
+ // testcase is a test case implemented by this function.
+ type testcase struct {
+ name string
+ input any
+ err error
+ output any
+ }
+
+ cases := []testcase{{
+ name: "with a nil map",
+ input: func() any {
+ var v map[string]string
+ return v
+ }(),
+ err: ErrIsNil,
+ output: nil,
+ }, {
+ name: "with a non-nil but empty map",
+ input: make(map[string]string),
+ err: nil,
+ output: make(map[string]string),
+ }, {
+ name: "with a non-nil non-empty map",
+ input: map[string]string{"a": "b"},
+ err: nil,
+ output: map[string]string{"a": "b"},
+ }, {
+ name: "with a nil pointer",
+ input: func() any {
+ var v *apiRequest
+ return v
+ }(),
+ err: ErrIsNil,
+ output: nil,
+ }, {
+ name: "with a non-nil empty pointer",
+ input: &apiRequest{},
+ err: nil,
+ output: &apiRequest{},
+ }, {
+ name: "with a non-nil non-empty pointer",
+ input: &apiRequest{UserID: 11},
+ err: nil,
+ output: &apiRequest{UserID: 11},
+ }, {
+ name: "with a nil slice",
+ input: func() any {
+ var v []int
+ return v
+ }(),
+ err: ErrIsNil,
+ output: nil,
+ }, {
+ name: "with a non-nil empty slice",
+ input: []int{},
+ err: nil,
+ output: []int{},
+ }, {
+ name: "with a non-nil non-empty slice",
+ input: []int{44},
+ err: nil,
+ output: []int{44},
+ }}
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ output, err := NilSafetyErrorIfNil(tc.input)
+
+ switch {
+ case err == nil && tc.err == nil:
+ if diff := cmp.Diff(tc.output, output); diff != "" {
+ t.Fatal(diff)
+ }
+ return
+
+ case err != nil && tc.err == nil:
+ t.Fatal("expected", tc.err.Error(), "got", err.Error())
+ return
+
+ case err == nil && tc.err != nil:
+ t.Fatal("expected", tc.err.Error(), "got", err.Error())
+ return
+
+ case err != nil && tc.err != nil:
+ if err.Error() != tc.err.Error() {
+ t.Fatal("expected", tc.err.Error(), "got", err.Error())
+ }
+ return
+ }
+ })
+ }
+}
+
+func TestNilSafetyAvoidNilByteSlice(t *testing.T) {
+ t.Run("for nil byte slice", func(t *testing.T) {
+ output := NilSafetyAvoidNilBytesSlice(nil)
+ if output == nil {
+ t.Fatal("expected non-nil")
+ }
+ if len(output) != 0 {
+ t.Fatal("expected zero length")
+ }
+ })
+
+ t.Run("for non-nil byte slice", func(t *testing.T) {
+ expected := []byte{44}
+ output := NilSafetyAvoidNilBytesSlice(expected)
+ if diff := cmp.Diff(expected, output); diff != "" {
+ t.Fatal("not the same pointer")
+ }
+ })
+}
diff --git a/pkg/httpclientx/overlapped.go b/pkg/httpclientx/overlapped.go
new file mode 100644
index 000000000..8577e119f
--- /dev/null
+++ b/pkg/httpclientx/overlapped.go
@@ -0,0 +1,243 @@
+package httpclientx
+
+//
+// overlapped.go - overlapped operations.
+//
+
+import (
+ "context"
+ "errors"
+ "time"
+)
+
+// OverlappedDefaultScheduleInterval is the default schedule interval. After this interval
+// has elapsed for a URL without seeing a success, we will schedule the next URL.
+const OverlappedDefaultScheduleInterval = 15 * time.Second
+
+// OverlappedDefaultWatchdogTimeout is the timeout after which we assume all the API calls
+// have gone rogue and forcibly interrupt all of them.
+const OverlappedDefaultWatchdogTimeout = 5 * time.Minute
+
+// Overlapped represents the possibility of overlapping HTTP calls for a set of
+// functionally equivalent URLs, such that we start a new call if the previous one
+// has failed to produce a result within the configured ScheduleInterval.
+//
+// # Limitations
+//
+// Under very bad networking conditions, [*Overlapped] would cause a new network
+// call to start while the previous one is still in progress and very slowly downloading
+// a response. A future implementation MIGHT want to account for this possibility.
+type Overlapped[Output any] struct {
+ // RunFunc is the MANDATORY function that fetches the given [*Endpoint].
+ //
+ // This field is typically initialized by [NewOverlappedGetJSON], [NewOverlappedGetRaw],
+ // [NewOverlappedGetXML], or [NewOverlappedPostJSON] to be the proper function that
+ // makes sense for the operation that you requested with the constructor.
+ //
+ // If you set it manually, you MUST modify it before calling [*Overlapped.Run].
+ RunFunc func(ctx context.Context, epnt *Endpoint) (Output, error)
+
+ // ScheduleInterval is the MANDATORY scheduling interval.
+ //
+ // This field is typically initialized by [NewOverlappedGetJSON], [NewOverlappedGetRaw],
+ // [NewOverlappedGetXML], or [NewOverlappedPostJSON] to be [OverlappedDefaultScheduleInterval].
+ //
+ // If you set it manually, you MUST modify it before calling [*Overlapped.Run].
+ ScheduleInterval time.Duration
+
+ // WatchdogTimeout is the MANDATORY timeout after which the code assumes
+ // that all API calls must be aborted and give up.
+ //
+ // This field is typically initialized by [NewOverlappedGetJSON], [NewOverlappedGetRaw],
+ // [NewOverlappedGetXML], or [NewOverlappedPostJSON] to be [OverlappedDefaultWatchdogTimeout].
+ //
+ // If you set it manually, you MUST modify it before calling [*Overlapped.Run].
+ WatchdogTimeout time.Duration
+}
+
+func newOverlappedWithFunc[Output any](fx func(context.Context, *Endpoint) (Output, error)) *Overlapped[Output] {
+ return &Overlapped[Output]{
+ RunFunc: fx,
+ ScheduleInterval: OverlappedDefaultScheduleInterval,
+ WatchdogTimeout: OverlappedDefaultWatchdogTimeout,
+ }
+}
+
+// NewOverlappedGetJSON constructs a [*Overlapped] for calling [GetJSON] with multiple URLs.
+func NewOverlappedGetJSON[Output any](config *Config) *Overlapped[Output] {
+ return newOverlappedWithFunc(func(ctx context.Context, epnt *Endpoint) (Output, error) {
+ return getJSON[Output](ctx, epnt, config)
+ })
+}
+
+// NewOverlappedGetRaw constructs a [*Overlapped] for calling [GetRaw] with multiple URLs.
+func NewOverlappedGetRaw(config *Config) *Overlapped[[]byte] {
+ return newOverlappedWithFunc(func(ctx context.Context, epnt *Endpoint) ([]byte, error) {
+ return getRaw(ctx, epnt, config)
+ })
+}
+
+// NewOverlappedGetXML constructs a [*Overlapped] for calling [GetXML] with multiple URLs.
+func NewOverlappedGetXML[Output any](config *Config) *Overlapped[Output] {
+ return newOverlappedWithFunc(func(ctx context.Context, epnt *Endpoint) (Output, error) {
+ return getXML[Output](ctx, epnt, config)
+ })
+}
+
+// NewOverlappedPostJSON constructs a [*Overlapped] for calling [PostJSON] with multiple URLs.
+func NewOverlappedPostJSON[Input, Output any](input Input, config *Config) *Overlapped[Output] {
+ return newOverlappedWithFunc(func(ctx context.Context, epnt *Endpoint) (Output, error) {
+ return postJSON[Input, Output](ctx, epnt, input, config)
+ })
+}
+
+// ErrGenericOverlappedFailure indicates that a generic [*Overlapped] failure occurred.
+var ErrGenericOverlappedFailure = errors.New("overlapped: generic failure")
+
+// Run runs the overlapped operations, returning the result of the first operation
+// that succeeds and its endpoint index, or the error that occurred.
+func (ovx *Overlapped[Output]) Run(ctx context.Context, epnts ...*Endpoint) (Output, int, error) {
+ return OverlappedReduce[Output](ovx.Map(ctx, epnts...))
+}
+
+// OverlappedErrorOr combines error information, result information and the endpoint index.
+type OverlappedErrorOr[Output any] struct {
+ // Err is the error or nil.
+ Err error
+
+ // Index is the endpoint index.
+ Index int
+
+ // Value is the result.
+ Value Output
+}
+
+// Map applies the [*Overlapped.RunFunc] function to each epnts entry, thus producing
+// a result for each entry. This function will cancel subsequent operations until there
+// is a success: subsequent results will be [context.Canceled] errors.
+//
+// Note that you SHOULD use [*Overlapped.Run] unless you want to observe the result
+// of each operation, which is mostly useful when running unit tests.
+//
+// Note that this function will return a zero length slice if epnts lenth is also zero.
+func (ovx *Overlapped[Output]) Map(ctx context.Context, epnts ...*Endpoint) []*OverlappedErrorOr[Output] {
+ // create cancellable context for early cancellation and also apply the
+ // watchdog timeout so that eventually this code returns.
+ //
+ // we are going to cancel this context as soon as we have a successful response so
+ // that we do not waste network resources by performing other attempts.
+ ctx, cancel := context.WithTimeout(ctx, ovx.WatchdogTimeout)
+ defer cancel()
+
+ // construct channel for collecting the results
+ //
+ // we're using this channel to communicate results back from goroutines running
+ // in the background and performing the real API call
+ output := make(chan *OverlappedErrorOr[Output])
+
+ // create ticker for scheduling subsequent attempts
+ //
+ // the ticker is going to tick at every schedule interval to start another
+ // attempt, if the previous attempt has not produced a result in time
+ ticker := time.NewTicker(ovx.ScheduleInterval)
+ defer ticker.Stop()
+
+ // create index for the next endpoint to try
+ idx := 0
+
+ // create vector for collecting results
+ //
+ // for simplicity, we're going to collect results from every goroutine
+ // including the ones cancelled by context after the previous success and
+ // then we're going to filter the results and produce a final result
+ results := []*OverlappedErrorOr[Output]{}
+
+ // keep looping until we have results for each endpoints
+ for len(results) < len(epnts) {
+
+ // if there are more endpoints to try, spawn a goroutine to try,
+ // and, otherwise, we can safely stop ticking
+ if idx < len(epnts) {
+ go ovx.transact(ctx, idx, epnts[idx], output)
+ idx++
+ } else {
+ ticker.Stop()
+ }
+
+ select {
+ // this event means that a child goroutine completed
+ // so we store the result; on success interrupt all the
+ // background goroutines and stop ticking
+ //
+ // note that we MUST continue reading until we have
+ // exactly `len(epnts)` results because the inner
+ // goroutine performs blocking writes on the channel
+ case res := <-output:
+ results = append(results, res)
+ if res.Err == nil {
+ ticker.Stop()
+ cancel()
+ }
+
+ // this means the ticker ticked, so we should loop again and
+ // attempt another endpoint because it's time to do that
+ case <-ticker.C:
+ }
+ }
+
+ // just send the results vector back to the caller
+ return results
+}
+
+// OverlappedReduce takes the results of [*Overlapped.Map] and returns either an Output or an error.
+//
+// Note that you SHOULD use [*Overlapped.Run] unless you want to observe the result
+// of each operation, which is mostly useful when running unit tests.
+//
+// The return value is (output, index, nil) on success and (zero, zero, error) on failure.
+func OverlappedReduce[Output any](results []*OverlappedErrorOr[Output]) (Output, int, error) {
+ // postprocess the results to check for success and
+ // aggregate all the errors that occurred
+ errorv := []error{}
+ for _, res := range results {
+ if res.Err == nil {
+ return res.Value, res.Index, nil
+ }
+ errorv = append(errorv, res.Err)
+ }
+
+ // handle the case where there's no error
+ //
+ // this happens if the user provided no endpoints to measure
+ if len(errorv) <= 0 {
+ errorv = append(errorv, ErrGenericOverlappedFailure)
+ }
+
+ // return zero value and errors list
+ //
+ // note that errors.Join returns nil if all the errors are nil or the
+ // list is nil, which is why we handle the corner case above
+ return *new(Output), 0, errors.Join(errorv...)
+}
+
+// transact performs an HTTP transaction with the given URL and writes results to the output channel.
+func (ovx *Overlapped[Output]) transact(
+ ctx context.Context, idx int, epnt *Endpoint, output chan<- *OverlappedErrorOr[Output]) {
+ // obtain the results
+ value, err := ovx.RunFunc(ctx, epnt)
+
+ // emit the results
+ //
+ // note that this unconditional channel write REQUIRES that we keep reading from
+ // the results channel in Run until we have a result per input endpoint
+ output <- &OverlappedErrorOr[Output]{
+ Err: err,
+ Index: idx,
+ Value: value,
+ }
+}
+
+// OverlappedIgnoreIndex is a filter that removes the index from [*Overlapped.Run] results.
+func OverlappedIgnoreIndex[Output any](value Output, _ int, err error) (Output, error) {
+ return value, err
+}
diff --git a/pkg/httpclientx/overlapped_test.go b/pkg/httpclientx/overlapped_test.go
new file mode 100644
index 000000000..64e8e1726
--- /dev/null
+++ b/pkg/httpclientx/overlapped_test.go
@@ -0,0 +1,529 @@
+package httpclientx
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "testing"
+ "time"
+
+ "github.com/apex/log"
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/must"
+ "github.com/ooni/probe-engine/pkg/netxlite"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/testingx"
+)
+
+// Implementation note: because top-level functions such as GetRaw always use
+// an [*Overlapped], we do not necessarily need to test that each top-level constructor
+// are WAI; rather, we should focus on the mechanics of multiple URLs.
+
+func TestNewOverlappedPostJSONFastRecoverFromEarlyErrors(t *testing.T) {
+
+ //
+ // Scenario:
+ //
+ // - 0.th.ooni.org is SNI blocked
+ // - 1.th.ooni.org is SNI blocked
+ // - 2.th.ooni.org is SNI blocked
+ // - 3.th.ooni.org WAIs
+ //
+ // We expect to get a response from 3.th.ooni.org.
+ //
+ // Because the first three THs fail fast but the schedule interval is the default (i.e.,
+ // 15 seconds), we're testing whether the algorithm allows us to recover quickly from
+ // failure and check the other endpoints without waiting for too much time.
+ //
+ // Note: before changing the algorithm, this test ran for 45 seconds. Now it runs
+ // for 1s because a previous goroutine terminating with error causes the next
+ // goroutine to start and attempt to fetch the resource.
+ //
+
+ zeroTh := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer zeroTh.Close()
+
+ oneTh := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer oneTh.Close()
+
+ twoTh := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer twoTh.Close()
+
+ expectedResponse := &apiResponse{
+ Age: 41,
+ Name: "sbs",
+ }
+
+ threeTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write(must.MarshalJSON(expectedResponse))
+ }))
+ defer threeTh.Close()
+
+ // Create client configuration. We don't care much about the
+ // JSON requests and reponses being aligned to reality.
+
+ apiReq := &apiRequest{
+ UserID: 117,
+ }
+
+ overlapped := NewOverlappedPostJSON[*apiRequest, *apiResponse](apiReq, &Config{
+ Authorization: "", // not relevant for this test
+ Client: http.DefaultClient,
+ Logger: log.Log,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ // Now we issue the requests and check we're getting the correct response.
+ //
+ // We're splitting the algorithm into its Map step and its Reduce step because
+ // this allows us to clearly observe what happened.
+
+ results := overlapped.Map(
+ context.Background(),
+ NewEndpoint(zeroTh.URL),
+ NewEndpoint(oneTh.URL),
+ NewEndpoint(twoTh.URL),
+ NewEndpoint(threeTh.URL),
+ )
+
+ runtimex.Assert(len(results) == 4, "unexpected number of results")
+
+ // the first three attempts should have failed with connection reset
+ // while the fourth result should be successful
+ for _, entry := range results {
+ t.Log(entry.Index, string(must.MarshalJSON(entry)))
+ switch entry.Index {
+ case 0, 1, 2:
+ if err := entry.Err; !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+ case 3:
+ if err := entry.Err; err != nil {
+ t.Fatal("unexpected error", err)
+ }
+ if diff := cmp.Diff(expectedResponse, entry.Value); diff != "" {
+ t.Fatal(diff)
+ }
+ default:
+ t.Fatal("unexpected index", entry.Index)
+ }
+ }
+
+ // Now run the reduce step of the algorithm and make sure we correctly
+ // return the first success and the nil error
+
+ apiResp, idx, err := OverlappedReduce(results)
+
+ // we do not expect to see a failure because threeTh is WAI
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if idx != 3 {
+ t.Fatal("unexpected success index", idx)
+ }
+
+ // compare response to expectation
+ if diff := cmp.Diff(expectedResponse, apiResp); diff != "" {
+ t.Fatal(diff)
+ }
+}
+
+func TestNewOverlappedPostJSONFirstCallSucceeds(t *testing.T) {
+
+ //
+ // Scenario:
+ //
+ // - 0.th.ooni.org is WAI
+ // - 1.th.ooni.org is WAI
+ // - 2.th.ooni.org is WAI
+ // - 3.th.ooni.org is WAI
+ //
+ // We expect to get a response from the first TH because it's the first goroutine
+ // that we schedule. Subsequent calls should be canceled.
+ //
+
+ expectedResponse := &apiResponse{
+ Age: 41,
+ Name: "sbs",
+ }
+
+ zeroTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write(must.MarshalJSON(expectedResponse))
+ }))
+ defer zeroTh.Close()
+
+ oneTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write(must.MarshalJSON(expectedResponse))
+ }))
+ defer oneTh.Close()
+
+ twoTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write(must.MarshalJSON(expectedResponse))
+ }))
+ defer twoTh.Close()
+
+ threeTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write(must.MarshalJSON(expectedResponse))
+ }))
+ defer threeTh.Close()
+
+ // Create client configuration. We don't care much about the
+ // JSON requests and reponses being aligned to reality.
+
+ apiReq := &apiRequest{
+ UserID: 117,
+ }
+
+ overlapped := NewOverlappedPostJSON[*apiRequest, *apiResponse](apiReq, &Config{
+ Authorization: "", // not relevant for this test
+ Client: http.DefaultClient,
+ Logger: log.Log,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ // make sure the schedule interval is high because we want
+ // all the goroutines but the first to be waiting for permission
+ // to fetch from their respective URLs.
+ overlapped.ScheduleInterval = 15 * time.Second
+
+ // Now we issue the requests and check we're getting the correct response.
+ //
+ // We're splitting the algorithm into its Map step and its Reduce step because
+ // this allows us to clearly observe what happened.
+
+ results := overlapped.Map(
+ context.Background(),
+ NewEndpoint(zeroTh.URL),
+ NewEndpoint(oneTh.URL),
+ NewEndpoint(twoTh.URL),
+ NewEndpoint(threeTh.URL),
+ )
+
+ runtimex.Assert(len(results) == 4, "unexpected number of results")
+
+ // the first attempt should succeed and subsequent ones should
+ // have failed with the context.Canceled error
+ for _, entry := range results {
+ t.Log(entry.Index, string(must.MarshalJSON(entry)))
+ switch entry.Index {
+ case 1, 2, 3:
+ if err := entry.Err; !errors.Is(err, context.Canceled) {
+ t.Fatal("unexpected error", err)
+ }
+ case 0:
+ if err := entry.Err; err != nil {
+ t.Fatal("unexpected error", err)
+ }
+ if diff := cmp.Diff(expectedResponse, entry.Value); diff != "" {
+ t.Fatal(diff)
+ }
+ default:
+ t.Fatal("unexpected index", entry.Index)
+ }
+ }
+
+ // Now run the reduce step of the algorithm and make sure we correctly
+ // return the first success and the nil error
+
+ apiResp, idx, err := OverlappedReduce(results)
+
+ // we do not expect to see a failure because all the THs are WAI
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if idx != 0 {
+ t.Fatal("unexpected success index", idx)
+ }
+
+ // compare response to expectation
+ if diff := cmp.Diff(expectedResponse, apiResp); diff != "" {
+ t.Fatal(diff)
+ }
+}
+
+func TestNewOverlappedPostJSONHandlesAllTimeouts(t *testing.T) {
+
+ //
+ // Scenario:
+ //
+ // - 0.th.ooni.org causes timeout
+ // - 1.th.ooni.org causes timeout
+ // - 2.th.ooni.org causes timeout
+ // - 3.th.ooni.org causes timeout
+ //
+ // We expect to loop for all endpoints and then discover that all of them
+ // failed. To make the test ~quick, we reduce the scheduling interval, and
+ // the watchdog timeout.
+ //
+
+ blockforever := make(chan any)
+
+ zeroTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ <-blockforever
+ w.WriteHeader(http.StatusBadGateway)
+ }))
+ defer zeroTh.Close()
+
+ oneTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ <-blockforever
+ w.WriteHeader(http.StatusBadGateway)
+ }))
+ defer oneTh.Close()
+
+ twoTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ <-blockforever
+ w.WriteHeader(http.StatusBadGateway)
+ }))
+ defer twoTh.Close()
+
+ threeTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ <-blockforever
+ w.WriteHeader(http.StatusBadGateway)
+ }))
+ defer threeTh.Close()
+
+ // Create client configuration. We don't care much about the
+ // JSON requests and reponses being aligned to reality.
+
+ apiReq := &apiRequest{
+ UserID: 117,
+ }
+
+ overlapped := NewOverlappedPostJSON[*apiRequest, *apiResponse](apiReq, &Config{
+ Authorization: "", // not relevant for this test
+ Client: http.DefaultClient,
+ Logger: log.Log,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ // make sure the schedule interval is low to make this test run faster.
+ overlapped.ScheduleInterval = 250 * time.Millisecond
+
+ // Now we issue the requests and check we're getting the correct response.
+ //
+ // We're splitting the algorithm into its Map step and its Reduce step because
+ // this allows us to clearly observe what happened.
+
+ // modify the watchdog timeout be much smaller than usual
+ overlapped.WatchdogTimeout = 2 * time.Second
+
+ results := overlapped.Map(
+ context.Background(),
+ NewEndpoint(zeroTh.URL),
+ NewEndpoint(oneTh.URL),
+ NewEndpoint(twoTh.URL),
+ NewEndpoint(threeTh.URL),
+ )
+
+ runtimex.Assert(len(results) == 4, "unexpected number of results")
+
+ // all the attempts should have failed with context deadline exceeded
+ for _, entry := range results {
+ t.Log(entry.Index, string(must.MarshalJSON(entry)))
+ switch entry.Index {
+ case 0, 1, 2, 3:
+ if err := entry.Err; !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatal("unexpected error", err)
+ }
+ default:
+ t.Fatal("unexpected index", entry.Index)
+ }
+ }
+
+ // Now run the reduce step of the algorithm and make sure we correctly
+ // return the first success and the nil error
+
+ apiResp, idx, err := OverlappedReduce(results)
+
+ // we expect to see a failure because the watchdog timeout should have fired
+ if !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatal("unexpected error", err)
+ }
+
+ if idx != 0 {
+ t.Fatal("unexpected index", idx)
+ }
+
+ // we expect the api response to be nil
+ if apiResp != nil {
+ t.Fatal("expected nil resp")
+ }
+
+ // now unblock the blocked goroutines
+ close(blockforever)
+}
+
+func TestNewOverlappedPostJSONResetTimeoutSuccessCanceled(t *testing.T) {
+
+ //
+ // Scenario:
+ //
+ // - 0.th.ooni.org resets the connection
+ // - 1.th.ooni.org causes timeout
+ // - 2.th.ooni.org is WAI
+ // - 3.th.ooni.org causes timeout
+ //
+ // We expect to see a success and to never attempt with 3.th.ooni.org.
+ //
+
+ blockforever := make(chan any)
+
+ zeroTh := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer zeroTh.Close()
+
+ oneTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ <-blockforever
+ w.WriteHeader(http.StatusBadGateway)
+ }))
+ defer oneTh.Close()
+
+ expectedResponse := &apiResponse{
+ Age: 41,
+ Name: "sbs",
+ }
+
+ twoTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write(must.MarshalJSON(expectedResponse))
+ }))
+ defer twoTh.Close()
+
+ threeTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ <-blockforever
+ w.WriteHeader(http.StatusBadGateway)
+ }))
+ defer threeTh.Close()
+
+ // Create client configuration. We don't care much about the
+ // JSON requests and reponses being aligned to reality.
+
+ apiReq := &apiRequest{
+ UserID: 117,
+ }
+
+ overlapped := NewOverlappedPostJSON[*apiRequest, *apiResponse](apiReq, &Config{
+ Authorization: "", // not relevant for this test
+ Client: http.DefaultClient,
+ Logger: log.Log,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ // make sure the schedule interval is low to make this test run faster.
+ overlapped.ScheduleInterval = 250 * time.Millisecond
+
+ // Now we issue the requests and check we're getting the correct response.
+ //
+ // We're splitting the algorithm into its Map step and its Reduce step because
+ // this allows us to clearly observe what happened.
+ //
+ // Note: we're running this test with the default watchdog timeout.
+
+ results := overlapped.Map(
+ context.Background(),
+ NewEndpoint(zeroTh.URL),
+ NewEndpoint(oneTh.URL),
+ NewEndpoint(twoTh.URL),
+ NewEndpoint(threeTh.URL),
+ )
+
+ runtimex.Assert(len(results) == 4, "unexpected number of results")
+
+ // attempt 0: should have seen connection reset
+ // attempt 1: should have seen the context canceled
+ // attempt 2: should be successful
+ // attempt 3: should have seen the context canceled
+ for _, entry := range results {
+ t.Log(entry.Index, string(must.MarshalJSON(entry)))
+ switch entry.Index {
+ case 0:
+ if err := entry.Err; !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+ case 1, 3:
+ if err := entry.Err; !errors.Is(err, context.Canceled) {
+ t.Fatal("unexpected error", err)
+ }
+ case 2:
+ if err := entry.Err; err != nil {
+ t.Fatal("unexpected error", err)
+ }
+ if diff := cmp.Diff(expectedResponse, entry.Value); diff != "" {
+ t.Fatal(diff)
+ }
+ default:
+ t.Fatal("unexpected index", entry.Index)
+ }
+ }
+
+ // Now run the reduce step of the algorithm and make sure we correctly
+ // return the first success and the nil error
+
+ apiResp, idx, err := OverlappedReduce(results)
+
+ // we do not expect to see a failure because one of the THs is WAI
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if idx != 2 {
+ t.Fatal("unexpected success index", idx)
+ }
+
+ // compare response to expectation
+ if diff := cmp.Diff(expectedResponse, apiResp); diff != "" {
+ t.Fatal(diff)
+ }
+
+ // now unblock the blocked goroutines
+ close(blockforever)
+}
+
+func TestNewOverlappedPostJSONWithNoURLs(t *testing.T) {
+
+ // Create client configuration. We don't care much about the
+ // JSON requests and reponses being aligned to reality.
+
+ apiReq := &apiRequest{
+ UserID: 117,
+ }
+
+ overlapped := NewOverlappedPostJSON[*apiRequest, *apiResponse](apiReq, &Config{
+ Authorization: "", // not relevant for this test
+ Client: http.DefaultClient,
+ Logger: log.Log,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ // Now we issue the requests without any URLs and make sure
+ // the result we get is the generic overlapped error
+
+ apiResp, idx, err := overlapped.Run(context.Background() /* no URLs here! */)
+
+ // we do expect to see the generic overlapped failure
+ if !errors.Is(err, ErrGenericOverlappedFailure) {
+ t.Fatal("unexpected error", err)
+ }
+
+ if idx != 0 {
+ t.Fatal("unexpected index", idx)
+ }
+
+ // we expect a nil response
+ if apiResp != nil {
+ t.Fatal("expected nil API response")
+ }
+}
+
+func TestNewOverlappedWithFuncDefaultsAreCorrect(t *testing.T) {
+ overlapped := newOverlappedWithFunc(func(ctx context.Context, e *Endpoint) (int, error) {
+ return 1, nil
+ })
+ if overlapped.ScheduleInterval != 15*time.Second {
+ t.Fatal("unexpected ScheduleInterval")
+ }
+ if overlapped.WatchdogTimeout != 5*time.Minute {
+ t.Fatal("unexpected WatchdogTimeout")
+ }
+}
diff --git a/pkg/httpclientx/postjson.go b/pkg/httpclientx/postjson.go
new file mode 100644
index 000000000..1dd2efe67
--- /dev/null
+++ b/pkg/httpclientx/postjson.go
@@ -0,0 +1,71 @@
+package httpclientx
+
+//
+// postjson.go - POST a JSON request and read a JSON response.
+//
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "net/http"
+)
+
+// PostJSON sends a POST request with a JSON body and reads a JSON response.
+//
+// Arguments:
+//
+// - ctx is the cancellable context;
+//
+// - epnt is the HTTP [*Endpoint] to use;
+//
+// - input is the input structure to JSON serialize as the request body;
+//
+// - config is the config to use.
+//
+// This function either returns an error or a valid Output.
+func PostJSON[Input, Output any](ctx context.Context, epnt *Endpoint, input Input, config *Config) (Output, error) {
+ return OverlappedIgnoreIndex(NewOverlappedPostJSON[Input, Output](input, config).Run(ctx, epnt))
+}
+
+func postJSON[Input, Output any](ctx context.Context, epnt *Endpoint, input Input, config *Config) (Output, error) {
+ // ensure we're not sending a nil map, pointer, or slice
+ if _, err := NilSafetyErrorIfNil(input); err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // serialize the request body
+ rawreqbody, err := json.Marshal(input)
+ if err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // log the raw request body
+ config.Logger.Debugf("POST %s: raw request body: %s", epnt.URL, string(rawreqbody))
+
+ // construct the request to use
+ req, err := http.NewRequestWithContext(ctx, "POST", epnt.URL, bytes.NewReader(rawreqbody))
+ if err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // assign the content type
+ req.Header.Set("Content-Type", "application/json")
+
+ // get the raw response body
+ rawrespbody, err := do(ctx, req, epnt, config)
+
+ // handle the case of error
+ if err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // parse the response body as JSON
+ var output Output
+ if err := json.Unmarshal(rawrespbody, &output); err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // avoid returning nil pointers, maps, slices
+ return NilSafetyErrorIfNil(output)
+}
diff --git a/pkg/httpclientx/postjson_test.go b/pkg/httpclientx/postjson_test.go
new file mode 100644
index 000000000..f2e828847
--- /dev/null
+++ b/pkg/httpclientx/postjson_test.go
@@ -0,0 +1,523 @@
+package httpclientx
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/must"
+ "github.com/ooni/probe-engine/pkg/netxlite"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/testingx"
+)
+
+type apiRequest struct {
+ UserID int
+}
+
+func TestPostJSON(t *testing.T) {
+ t.Run("when we cannot marshal the request body", func(t *testing.T) {
+ // a channel cannot be serialized
+ req := make(chan int)
+ close(req)
+
+ resp, err := PostJSON[chan int, *apiResponse](
+ context.Background(),
+ NewEndpoint(""),
+ req,
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ if err.Error() != `json: unsupported type: chan int` {
+ t.Fatal("unexpected error", err)
+ }
+
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when we cannot create a request", func(t *testing.T) {
+ req := &apiRequest{117}
+
+ resp, err := PostJSON[*apiRequest, *apiResponse](
+ context.Background(),
+ NewEndpoint("\t"), // <- invalid URL that we cannot parse
+ req,
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ },
+ )
+
+ t.Log(resp)
+ t.Log(err)
+
+ if err.Error() != `parse "\t": net/url: invalid control character in URL` {
+ t.Fatal("unexpected error", err)
+ }
+
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("in case of HTTP failure", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer server.Close()
+
+ req := &apiRequest{117}
+
+ resp, err := PostJSON[*apiRequest, *apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL),
+ req,
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when we cannot parse the response body", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte("[]"))
+ }))
+ defer server.Close()
+
+ req := &apiRequest{117}
+
+ resp, err := PostJSON[*apiRequest, *apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL),
+ req,
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if err.Error() != "json: cannot unmarshal array into Go value of type httpclientx.apiResponse" {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is nil.
+ if resp != nil {
+ t.Fatal("expected nil response")
+ }
+ })
+
+ t.Run("on success", func(t *testing.T) {
+ req := &apiRequest{117}
+
+ expect := &apiResponse{Name: "simone", Age: 41}
+
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var gotreq apiRequest
+ data := runtimex.Try1(netxlite.ReadAllContext(r.Context(), r.Body))
+ must.UnmarshalJSON(data, &gotreq)
+ if gotreq.UserID != req.UserID {
+ w.WriteHeader(404)
+ return
+ }
+ w.Write(must.MarshalJSON(expect))
+ }))
+ defer server.Close()
+
+ resp, err := PostJSON[*apiRequest, *apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL),
+ req,
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is OK.
+ if diff := cmp.Diff(expect, resp); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+}
+
+// This test ensures that PostJSON sets correct HTTP headers and sends the right body.
+func TestPostJSONCommunicationOkay(t *testing.T) {
+ var (
+ gothost string
+ gotheaders http.Header
+ gotrawbody []byte
+ gotmu sync.Mutex
+ )
+
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // read the raw response body
+ rawbody := runtimex.Try1(netxlite.ReadAllContext(r.Context(), r.Body))
+
+ // save the raw response body and headers
+ gotmu.Lock()
+ gothost = r.Host
+ gotrawbody = rawbody
+ gotheaders = r.Header
+ gotmu.Unlock()
+
+ // send a minimal 200 Ok response
+ w.WriteHeader(200)
+ w.Write([]byte(`{}`))
+ }))
+ defer server.Close()
+
+ // create and serialize the expected request body
+ apireq := &apiRequest{
+ UserID: 117,
+ }
+ rawapireq := must.MarshalJSON(apireq)
+
+ // send the request and receive the response
+ apiresp, err := PostJSON[*apiRequest, *apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL).WithHostOverride("www.cloudfront.com"),
+ apireq,
+ &Config{
+ Authorization: "scribai",
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ // we do not expect to see an error here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // given the handler, we expect to see an empty structure here
+ if apiresp.Age != 0 || apiresp.Name != "" {
+ t.Fatal("expected empty response")
+ }
+
+ // make sure there are no data races
+ defer gotmu.Unlock()
+ gotmu.Lock()
+
+ // now verify what the handler has read as the raw request body
+ if diff := cmp.Diff(rawapireq, gotrawbody); diff != "" {
+ t.Fatal(diff)
+ }
+
+ // make sure we have sent the authorization header
+ if value := gotheaders.Get("Authorization"); value != "scribai" {
+ t.Fatal("unexpected Authorization value", value)
+ }
+
+ // now make sure we have sent content-type
+ if value := gotheaders.Get("Content-Type"); value != "application/json" {
+ t.Fatal("unexpected Content-Type value", value)
+ }
+
+ // now make sure we have sent user-agent
+ if value := gotheaders.Get("User-Agent"); value != model.HTTPHeaderUserAgent {
+ t.Fatal("unexpected User-Agent value", value)
+ }
+
+ // now make sure we have sent accept-encoding
+ if value := gotheaders.Get("Accept-Encoding"); value != "gzip" {
+ t.Fatal("unexpected Accept-Encoding value", value)
+ }
+
+ // now make sure we could use cloudfronting
+ if gothost != "www.cloudfront.com" {
+ t.Fatal("unexpected Host value", gothost)
+ }
+}
+
+// This test ensures PostJSON logs the request and response body at Debug level.
+func TestPostJSONLoggingOkay(t *testing.T) {
+ req := &apiRequest{117}
+
+ expect := &apiResponse{Name: "simone", Age: 41}
+
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var gotreq apiRequest
+ data := runtimex.Try1(netxlite.ReadAllContext(r.Context(), r.Body))
+ must.UnmarshalJSON(data, &gotreq)
+ if gotreq.UserID != req.UserID {
+ w.WriteHeader(404)
+ return
+ }
+ w.Write(must.MarshalJSON(expect))
+ }))
+ defer server.Close()
+
+ // instantiate a logger that collects logs
+ logger := &testingx.Logger{}
+
+ resp, err := PostJSON[*apiRequest, *apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL),
+ req,
+ &Config{
+ Client: http.DefaultClient,
+ Logger: logger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is OK.
+ if diff := cmp.Diff(expect, resp); diff != "" {
+ t.Fatal(diff)
+ }
+
+ // collect and verify the debug lines
+ debuglines := logger.DebugLines()
+ t.Log(debuglines)
+ if len(debuglines) != 2 {
+ t.Fatal("expected to see a single debug line")
+ }
+ if !strings.Contains(debuglines[0], "raw request body:") {
+ t.Fatal("did not see raw request body log line")
+ }
+ if !strings.Contains(debuglines[1], "raw response body:") {
+ t.Fatal("did not see raw response body log line")
+ }
+}
+
+// TestPostJSONCorrectlyRejectsNilValues ensures we do not emit and correctly reject nil values.
+func TestPostJSONCorrectlyRejectsNilValues(t *testing.T) {
+
+ t.Run("when sending a nil map", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{}`))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := PostJSON[map[string]string, *apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL),
+ nil,
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when sending a nil struct pointer", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{}`))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := PostJSON[*apiRequest, *apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL),
+ nil,
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when sending a nil slice", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{}`))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := PostJSON[[]string, *apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL),
+ nil,
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when unmarshaling into a map", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`null`))
+ }))
+ defer server.Close()
+
+ // create an empty request
+ apireq := &apiRequest{}
+
+ // invoke the API
+ resp, err := PostJSON[*apiRequest, map[string]string](
+ context.Background(),
+ NewEndpoint(server.URL),
+ apireq,
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when unmarshaling into a struct pointer", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`null`))
+ }))
+ defer server.Close()
+
+ // create an empty request
+ apireq := &apiRequest{}
+
+ // invoke the API
+ resp, err := PostJSON[*apiRequest, *apiResponse](
+ context.Background(),
+ NewEndpoint(server.URL),
+ apireq,
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when unmarshaling into a slice", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`null`))
+ }))
+ defer server.Close()
+
+ // create an empty request
+ apireq := &apiRequest{}
+
+ // invoke the API
+ resp, err := PostJSON[*apiRequest, []string](
+ context.Background(),
+ NewEndpoint(server.URL),
+ apireq,
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+}
diff --git a/pkg/httpx/httpx.go b/pkg/httpx/httpx.go
deleted file mode 100644
index 69b83387b..000000000
--- a/pkg/httpx/httpx.go
+++ /dev/null
@@ -1,267 +0,0 @@
-// Package httpx contains http extensions.
-//
-// Deprecated: new code should use httpapi instead. While this package and httpapi
-// are basically using the same implementation, the API exposed by httpapi allows
-// us to try the same request with multiple HTTP endpoints.
-package httpx
-
-import (
- "bytes"
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "net/http"
- "net/url"
- "strings"
-
- "github.com/ooni/probe-engine/pkg/model"
- "github.com/ooni/probe-engine/pkg/netxlite"
-)
-
-// APIClientTemplate is a template for constructing an APIClient.
-type APIClientTemplate struct {
- // Accept contains the OPTIONAL accept header.
- Accept string
-
- // Authorization contains the OPTIONAL authorization header.
- Authorization string
-
- // BaseURL is the MANDATORY base URL of the API.
- BaseURL string
-
- // HTTPClient is the MANDATORY underlying http client to use.
- HTTPClient model.HTTPClient
-
- // Host allows to OPTIONALLY set a specific host header. This is useful
- // to implement, e.g., cloudfronting.
- Host string
-
- // LogBody is the OPTIONAL flag to force logging the bodies.
- LogBody bool
-
- // Logger is MANDATORY the logger to use.
- Logger model.Logger
-
- // UserAgent is the OPTIONAL user agent to use.
- UserAgent string
-}
-
-// WithBodyLogging enables logging of request and response bodies.
-func (tmpl *APIClientTemplate) WithBodyLogging() *APIClientTemplate {
- out := APIClientTemplate(*tmpl)
- out.LogBody = true
- return &out
-}
-
-// Build creates an APIClient from the APIClientTemplate.
-func (tmpl *APIClientTemplate) Build() APIClient {
- return tmpl.BuildWithAuthorization(tmpl.Authorization)
-}
-
-// BuildWithAuthorization creates an APIClient from the
-// APIClientTemplate and ensures it uses the given authorization
-// value for APIClient.Authorization in subsequent API calls.
-func (tmpl *APIClientTemplate) BuildWithAuthorization(authorization string) APIClient {
- ac := apiClient(*tmpl)
- ac.Authorization = authorization
- return &ac
-}
-
-// DefaultMaxBodySize is the default value for the maximum
-// body size you can fetch using an APIClient.
-const DefaultMaxBodySize = 1 << 22
-
-// APIClient is a client configured to call a given API identified
-// by a given baseURL and using a given model.HTTPClient.
-//
-// The resource path argument passed to APIClient methods is appended
-// to the base URL's path for determining the full URL's path.
-type APIClient interface {
- // GetJSON reads the JSON resource whose path is obtained concatenating
- // the baseURL's path with `resourcePath` and unmarshals the results
- // into `output`. The request is bounded by the lifetime of the
- // context passed as argument. Returns the error that occurred.
- GetJSON(ctx context.Context, resourcePath string, output interface{}) error
-
- // GetJSONWithQuery is like GetJSON but also has a query.
- GetJSONWithQuery(ctx context.Context, resourcePath string,
- query url.Values, output interface{}) error
-
- // PostJSON creates a JSON subresource of the resource whose
- // path is obtained concatenating the baseURL'spath with `resourcePath` using
- // the JSON document at `input` as value and returning the result into the
- // JSON document at output. The request is bounded by the context's
- // lifetime. Returns the error that occurred.
- PostJSON(ctx context.Context, resourcePath string, input, output interface{}) error
-
- // FetchResource fetches the specified resource and returns it.
- FetchResource(ctx context.Context, URLPath string) ([]byte, error)
-}
-
-// apiClient is an extended HTTP client. To construct this struct, make
-// sure you initialize all fields marked as MANDATORY.
-type apiClient struct {
- // Accept contains the OPTIONAL accept header.
- Accept string
-
- // Authorization contains the OPTIONAL authorization header.
- Authorization string
-
- // BaseURL is the MANDATORY base URL of the API.
- BaseURL string
-
- // HTTPClient is the MANDATORY underlying http client to use.
- HTTPClient model.HTTPClient
-
- // Host allows to OPTIONALLY set a specific host header. This is useful
- // to implement, e.g., cloudfronting.
- Host string
-
- // LogBody is the OPTIONAL flag to force logging the bodies.
- LogBody bool
-
- // Logger is MANDATORY the logger to use.
- Logger model.Logger
-
- // UserAgent is the OPTIONAL user agent to use.
- UserAgent string
-}
-
-// newRequestWithJSONBody creates a new request with a JSON body
-func (c *apiClient) newRequestWithJSONBody(
- ctx context.Context, method, resourcePath string,
- query url.Values, body interface{}) (*http.Request, error) {
- data, err := json.Marshal(body)
- if err != nil {
- return nil, err
- }
- c.Logger.Debugf("httpx: request body length: %d bytes", len(data))
- if c.LogBody {
- c.Logger.Debugf("httpx: request body: %s", string(data))
- }
- request, err := c.newRequest(
- ctx, method, resourcePath, query, bytes.NewReader(data))
- if err != nil {
- return nil, err
- }
- if body != nil {
- request.Header.Set("Content-Type", "application/json")
- }
- return request, nil
-}
-
-// joinURLPath appends resourcePath to the urlPath.
-func (c *apiClient) joinURLPath(urlPath, resourcePath string) string {
- if resourcePath == "" {
- if urlPath == "" {
- return "/"
- }
- return urlPath
- }
- if !strings.HasSuffix(urlPath, "/") {
- urlPath += "/"
- }
- resourcePath = strings.TrimPrefix(resourcePath, "/")
- return urlPath + resourcePath
-}
-
-// newRequest creates a new request.
-func (c *apiClient) newRequest(ctx context.Context, method, resourcePath string,
- query url.Values, body io.Reader) (*http.Request, error) {
- URL, err := url.Parse(c.BaseURL)
- if err != nil {
- return nil, err
- }
- // BaseURL and resource URL is joined if they have a path
- URL.Path = c.joinURLPath(URL.Path, resourcePath)
- if query != nil {
- URL.RawQuery = query.Encode()
- }
- request, err := http.NewRequestWithContext(ctx, method, URL.String(), body)
- if err != nil {
- return nil, err
- }
- request.Host = c.Host // allow cloudfronting
- if c.Authorization != "" {
- request.Header.Set("Authorization", c.Authorization)
- }
- if c.Accept != "" {
- request.Header.Set("Accept", c.Accept)
- }
- request.Header.Set("User-Agent", c.UserAgent)
- return request, nil
-}
-
-// ErrRequestFailed indicates that the server returned >= 400.
-var ErrRequestFailed = errors.New("httpx: request failed")
-
-// do performs the provided request and returns the response body or an error.
-func (c *apiClient) do(request *http.Request) ([]byte, error) {
- response, err := c.HTTPClient.Do(request)
- if err != nil {
- return nil, err
- }
- defer response.Body.Close()
- // Implementation note: always read and log the response body since
- // it's quite useful to see the response JSON on API error.
- r := io.LimitReader(response.Body, DefaultMaxBodySize)
- data, err := netxlite.ReadAllContext(request.Context(), r)
- if err != nil {
- return nil, err
- }
- c.Logger.Debugf("httpx: response body length: %d bytes", len(data))
- if c.LogBody {
- c.Logger.Debugf("httpx: response body: %s", string(data))
- }
- if response.StatusCode >= 400 {
- return nil, fmt.Errorf("%w: %s", ErrRequestFailed, response.Status)
- }
- return data, nil
-}
-
-// doJSON performs the provided request and unmarshals the JSON response body
-// into the provided output variable.
-func (c *apiClient) doJSON(request *http.Request, output interface{}) error {
- data, err := c.do(request)
- if err != nil {
- return err
- }
- return json.Unmarshal(data, output)
-}
-
-// GetJSON implements APIClient.GetJSON.
-func (c *apiClient) GetJSON(ctx context.Context, resourcePath string, output interface{}) error {
- return c.GetJSONWithQuery(ctx, resourcePath, nil, output)
-}
-
-// GetJSONWithQuery implements APIClient.GetJSONWithQuery.
-func (c *apiClient) GetJSONWithQuery(
- ctx context.Context, resourcePath string,
- query url.Values, output interface{}) error {
- request, err := c.newRequest(ctx, "GET", resourcePath, query, nil)
- if err != nil {
- return err
- }
- return c.doJSON(request, output)
-}
-
-// PostJSON implements APIClient.PostJSON.
-func (c *apiClient) PostJSON(
- ctx context.Context, resourcePath string, input, output interface{}) error {
- request, err := c.newRequestWithJSONBody(ctx, "POST", resourcePath, nil, input)
- if err != nil {
- return err
- }
- return c.doJSON(request, output)
-}
-
-// FetchResource implements APIClient.FetchResource.
-func (c *apiClient) FetchResource(ctx context.Context, URLPath string) ([]byte, error) {
- request, err := c.newRequest(ctx, "GET", URLPath, nil, nil)
- if err != nil {
- return nil, err
- }
- return c.do(request)
-}
diff --git a/pkg/httpx/httpx_test.go b/pkg/httpx/httpx_test.go
deleted file mode 100644
index 800e1e652..000000000
--- a/pkg/httpx/httpx_test.go
+++ /dev/null
@@ -1,698 +0,0 @@
-package httpx
-
-import (
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "net/http"
- "net/http/httptest"
- "net/url"
- "strings"
- "testing"
-
- "github.com/google/go-cmp/cmp"
- "github.com/ooni/probe-engine/pkg/mocks"
- "github.com/ooni/probe-engine/pkg/model"
- "github.com/ooni/probe-engine/pkg/netxlite"
- "github.com/ooni/probe-engine/pkg/testingx"
- "github.com/ooni/probe-engine/pkg/version"
-)
-
-// userAgent is the user agent used by this test suite
-var userAgent = fmt.Sprintf("ooniprobe-cli/%s", version.Version)
-
-func TestAPIClientTemplate(t *testing.T) {
- t.Run("WithBodyLogging", func(t *testing.T) {
- tmpl := &APIClientTemplate{
- HTTPClient: http.DefaultClient,
- LogBody: false, // explicit default initialization for clarity
- Logger: model.DiscardLogger,
- }
- child := tmpl.WithBodyLogging()
- if !child.LogBody {
- t.Fatal("expected body logging to be enabled")
- }
- if tmpl.LogBody {
- t.Fatal("expected body logging to still be disabled")
- }
- })
-
- t.Run("normal constructor", func(t *testing.T) {
- // Implementation note: the fakefiller will ignore the
- // fields it does not know how to fill, so we are filling
- // those fields with plausible values in advance
- tmpl := &APIClientTemplate{
- HTTPClient: http.DefaultClient,
- Logger: model.DiscardLogger,
- }
- ff := &testingx.FakeFiller{}
- ff.Fill(tmpl)
- ac := tmpl.Build()
- orig := apiClient(*tmpl)
- if diff := cmp.Diff(&orig, ac); diff != "" {
- t.Fatal(diff)
- }
- })
-
- t.Run("constructor with authorization", func(t *testing.T) {
- // Implementation note: the fakefiller will ignore the
- // fields it does not know how to fill, so we are filling
- // those fields with plausible values in advance
- tmpl := &APIClientTemplate{
- HTTPClient: http.DefaultClient,
- Logger: model.DiscardLogger,
- }
- ff := &testingx.FakeFiller{}
- ff.Fill(tmpl)
- tok := ""
- ff.Fill(&tok)
- ac := tmpl.BuildWithAuthorization(tok)
- // the authorization should be different now
- if tmpl.Authorization == ac.(*apiClient).Authorization {
- t.Fatal("we expect Authorization to be different")
- }
- // clear authorization for the comparison
- tmpl.Authorization = ""
- ac.(*apiClient).Authorization = ""
- orig := apiClient(*tmpl)
- if diff := cmp.Diff(&orig, ac); diff != "" {
- t.Fatal(diff)
- }
- })
-}
-
-// newAPIClient is an helper factory creating a client for testing.
-func newAPIClient() *apiClient {
- return &apiClient{
- BaseURL: "https://example.com",
- HTTPClient: http.DefaultClient,
- Logger: model.DiscardLogger,
- UserAgent: userAgent,
- }
-}
-
-func TestJoinURLPath(t *testing.T) {
- t.Run("the whole path is inside basePath and there's no resource path", func(t *testing.T) {
- ac := newAPIClient()
- ac.BaseURL = "https://example.com/robots.txt"
- req, err := ac.newRequest(context.Background(), "GET", "", nil, nil)
- if err != nil {
- t.Fatal(err)
- }
- if req.URL.String() != "https://example.com/robots.txt" {
- t.Fatal("unexpected result", req.URL.String())
- }
- })
-
- t.Run("empty baseURL path and slash-prefixed resource path", func(t *testing.T) {
- ac := newAPIClient()
- ac.BaseURL = "https://example.com"
- req, err := ac.newRequest(context.Background(), "GET", "/foo", nil, nil)
- if err != nil {
- t.Fatal(err)
- }
- if req.URL.String() != "https://example.com/foo" {
- t.Fatal("unexpected result", req.URL.String())
- }
- })
-
- t.Run("root baseURL path and slash-prefixed resource path", func(t *testing.T) {
- ac := newAPIClient()
- ac.BaseURL = "https://example.com/"
- req, err := ac.newRequest(context.Background(), "GET", "/foo", nil, nil)
- if err != nil {
- t.Fatal(err)
- }
- if req.URL.String() != "https://example.com/foo" {
- t.Fatal("unexpected result", req.URL.String())
- }
- })
-
- t.Run("empty baseURL path and empty resource path", func(t *testing.T) {
- ac := newAPIClient()
- ac.BaseURL = "https://example.com"
- req, err := ac.newRequest(context.Background(), "GET", "", nil, nil)
- if err != nil {
- t.Fatal(err)
- }
- if req.URL.String() != "https://example.com/" {
- t.Fatal("unexpected result", req.URL.String())
- }
- })
-
- t.Run("non-slash-terminated baseURL path and slash-prefixed resource path", func(t *testing.T) {
- ac := newAPIClient()
- ac.BaseURL = "http://example.com/foo"
- req, err := ac.newRequest(context.Background(), "GET", "/bar", nil, nil)
- if err != nil {
- t.Fatal(err)
- }
- if req.URL.String() != "http://example.com/foo/bar" {
- t.Fatal("unexpected result", req.URL.String())
- }
- })
-
- t.Run("slash-terminated baseURL path and slash-prefixed resource path", func(t *testing.T) {
- ac := newAPIClient()
- ac.BaseURL = "http://example.com/foo/"
- req, err := ac.newRequest(context.Background(), "GET", "/bar", nil, nil)
- if err != nil {
- t.Fatal(err)
- }
- if req.URL.String() != "http://example.com/foo/bar" {
- t.Fatal("unexpected result", req.URL.String())
- }
- })
-
- t.Run("slash-terminated baseURL path and non-slash-prefixed resource path", func(t *testing.T) {
- ac := newAPIClient()
- ac.BaseURL = "http://example.com/foo/"
- req, err := ac.newRequest(context.Background(), "GET", "bar", nil, nil)
- if err != nil {
- t.Fatal(err)
- }
- if req.URL.String() != "http://example.com/foo/bar" {
- t.Fatal("unexpected result", req.URL.String())
- }
- })
-}
-
-// fakeRequest is a fake request we serialize.
-type fakeRequest struct {
- Name string
- Age int
- Sleeping bool
- Attributes map[string][]string
-}
-
-func TestAPIClient(t *testing.T) {
- t.Run("newRequestWithJSONBody", func(t *testing.T) {
- t.Run("JSON marshal failure", func(t *testing.T) {
- client := newAPIClient()
- req, err := client.newRequestWithJSONBody(
- context.Background(), "GET", "/", nil, make(chan interface{}),
- )
- if err == nil || !strings.HasPrefix(err.Error(), "json: unsupported type") {
- t.Fatal("not the error we expected", err)
- }
- if req != nil {
- t.Fatal("expected nil request here")
- }
- })
-
- t.Run("newRequest failure", func(t *testing.T) {
- client := newAPIClient()
- client.BaseURL = "\t\t\t" // cause URL parse error
- req, err := client.newRequestWithJSONBody(
- context.Background(), "GET", "/", nil, nil,
- )
- if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
- t.Fatal("not the error we expected")
- }
- if req != nil {
- t.Fatal("expected nil request here")
- }
- })
-
- t.Run("sets the content-type properly", func(t *testing.T) {
- var jsonReq fakeRequest
- ff := &testingx.FakeFiller{}
- ff.Fill(&jsonReq)
- client := newAPIClient()
- req, err := client.newRequestWithJSONBody(
- context.Background(), "GET", "/", nil, jsonReq,
- )
- if err != nil {
- t.Fatal(err)
- }
- if req.Header.Get("Content-Type") != "application/json" {
- t.Fatal("did not set content-type properly")
- }
- })
- })
-
- t.Run("newRequest", func(t *testing.T) {
- t.Run("with invalid method", func(t *testing.T) {
- client := newAPIClient()
- req, err := client.newRequest(
- context.Background(), "\t\t\t", "/", nil, nil,
- )
- if err == nil || !strings.HasPrefix(err.Error(), "net/http: invalid method") {
- t.Fatal("not the error we expected")
- }
- if req != nil {
- t.Fatal("expected nil request here")
- }
- })
-
- t.Run("with query", func(t *testing.T) {
- client := newAPIClient()
- q := url.Values{}
- q.Add("antani", "mascetti")
- q.Add("melandri", "conte")
- req, err := client.newRequest(
- context.Background(), "GET", "/", q, nil,
- )
- if err != nil {
- t.Fatal(err)
- }
- if req.URL.Query().Get("antani") != "mascetti" {
- t.Fatal("expected different query string here")
- }
- if req.URL.Query().Get("melandri") != "conte" {
- t.Fatal("expected different query string here")
- }
- })
-
- t.Run("with authorization", func(t *testing.T) {
- client := newAPIClient()
- client.Authorization = "deadbeef"
- req, err := client.newRequest(
- context.Background(), "GET", "/", nil, nil,
- )
- if err != nil {
- t.Fatal(err)
- }
- if req.Header.Get("Authorization") != client.Authorization {
- t.Fatal("expected different Authorization here")
- }
- })
-
- t.Run("with accept", func(t *testing.T) {
- client := newAPIClient()
- client.Accept = "application/xml"
- req, err := client.newRequestWithJSONBody(
- context.Background(), "GET", "/", nil, []string{},
- )
- if err != nil {
- t.Fatal(err)
- }
- if req.Header.Get("Accept") != "application/xml" {
- t.Fatal("expected different Accept here")
- }
- })
-
- t.Run("with custom host header", func(t *testing.T) {
- client := newAPIClient()
- client.Host = "www.x.org"
- req, err := client.newRequest(
- context.Background(), "GET", "/", nil, nil,
- )
- if err != nil {
- t.Fatal(err)
- }
- if req.Host != client.Host {
- t.Fatal("expected different req.Host here")
- }
- })
-
- t.Run("with user agent", func(t *testing.T) {
- client := newAPIClient()
- req, err := client.newRequest(
- context.Background(), "GET", "/", nil, nil,
- )
- if err != nil {
- t.Fatal(err)
- }
- if req.Header.Get("User-Agent") != userAgent {
- t.Fatal("expected different User-Agent here")
- }
- })
- })
-
- t.Run("doJSON", func(t *testing.T) {
- t.Run("do failure", func(t *testing.T) {
- expected := errors.New("mocked error")
- client := newAPIClient()
- client.HTTPClient = &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- return nil, expected
- },
- }
- err := client.doJSON(&http.Request{URL: &url.URL{Scheme: "https", Host: "x.org"}}, nil)
- if !errors.Is(err, expected) {
- t.Fatal("not the error we expected")
- }
- })
-
- t.Run("response is not successful (i.e., >= 400)", func(t *testing.T) {
- client := newAPIClient()
- client.HTTPClient = &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- return &http.Response{
- StatusCode: 401,
- Body: io.NopCloser(strings.NewReader("{}")),
- }, nil
- },
- }
- err := client.doJSON(&http.Request{URL: &url.URL{Scheme: "https", Host: "x.org"}}, nil)
- if !errors.Is(err, ErrRequestFailed) {
- t.Fatal("not the error we expected", err)
- }
- })
-
- t.Run("cannot read body", func(t *testing.T) {
- expected := errors.New("mocked error")
- client := newAPIClient()
- client.HTTPClient = &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- return &http.Response{
- StatusCode: 200,
- Body: io.NopCloser(&mocks.Reader{
- MockRead: func(b []byte) (int, error) {
- return 0, expected
- },
- }),
- }, nil
- },
- }
- err := client.doJSON(&http.Request{URL: &url.URL{Scheme: "https", Host: "x.org"}}, nil)
- if !errors.Is(err, expected) {
- t.Fatal("not the error we expected")
- }
- })
-
- t.Run("response is not JSON", func(t *testing.T) {
- client := newAPIClient()
- client.HTTPClient = &mocks.HTTPClient{
- MockDo: func(req *http.Request) (*http.Response, error) {
- return &http.Response{
- StatusCode: 200,
- Body: io.NopCloser(strings.NewReader("[")),
- }, nil
- },
- }
- err := client.doJSON(&http.Request{URL: &url.URL{Scheme: "https", Host: "x.org"}}, nil)
- if err == nil || err.Error() != "unexpected end of JSON input" {
- t.Fatal("not the error we expected")
- }
- })
- })
-
- t.Run("GetJSON", func(t *testing.T) {
- t.Run("successful case", func(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(
- func(w http.ResponseWriter, r *http.Request) {
- w.Write([]byte(`["foo", "bar"]`))
- },
- ))
- defer server.Close()
- ctx := context.Background()
- var result []string
- err := (&apiClient{
- BaseURL: server.URL,
- HTTPClient: http.DefaultClient,
- Logger: model.DiscardLogger,
- }).GetJSON(ctx, "/", &result)
- if err != nil {
- t.Fatal(err)
- }
- if len(result) != 2 || result[0] != "foo" || result[1] != "bar" {
- t.Fatal("invalid result", result)
- }
- })
-
- t.Run("failure case", func(t *testing.T) {
- var headers []string
- client := newAPIClient()
- client.BaseURL = "\t\t\t\t"
- err := client.GetJSON(context.Background(), "/", &headers)
- if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
- t.Fatal("not the error we expected")
- }
- })
- })
-
- t.Run("PostJSON", func(t *testing.T) {
- t.Run("successful case", func(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(
- func(w http.ResponseWriter, r *http.Request) {
- var incoming []string
- data, err := netxlite.ReadAllContext(r.Context(), r.Body)
- if err != nil {
- w.WriteHeader(500)
- return
- }
- if err := json.Unmarshal(data, &incoming); err != nil {
- w.WriteHeader(500)
- return
- }
- w.Write(data)
- },
- ))
- defer server.Close()
- ctx := context.Background()
- incoming := []string{"foo", "bar"}
- var result []string
- err := (&apiClient{
- BaseURL: server.URL,
- HTTPClient: http.DefaultClient,
- Logger: model.DiscardLogger,
- }).PostJSON(ctx, "/", incoming, &result)
- if err != nil {
- t.Fatal(err)
- }
- if diff := cmp.Diff(incoming, result); diff != "" {
- t.Fatal(diff)
- }
- })
-
- t.Run("failure case", func(t *testing.T) {
- incoming := []string{"foo", "bar"}
- var result []string
- client := newAPIClient()
- client.BaseURL = "\t\t\t\t"
- err := client.PostJSON(context.Background(), "/", incoming, &result)
- if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
- t.Fatal("not the error we expected")
- }
- })
- })
-
- t.Run("FetchResource", func(t *testing.T) {
- t.Run("successful case", func(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(
- func(w http.ResponseWriter, r *http.Request) {
- w.Write([]byte("deadbeef"))
- },
- ))
- defer server.Close()
- ctx := context.Background()
- data, err := (&apiClient{
- BaseURL: server.URL,
- HTTPClient: http.DefaultClient,
- Logger: model.DiscardLogger,
- }).FetchResource(ctx, "/")
- if err != nil {
- t.Fatal(err)
- }
- if string(data) != "deadbeef" {
- t.Fatal("invalid data")
- }
- })
-
- t.Run("failure case", func(t *testing.T) {
- client := newAPIClient()
- client.BaseURL = "\t\t\t\t"
- data, err := client.FetchResource(context.Background(), "/")
- if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
- t.Fatal("not the error we expected")
- }
- if data != nil {
- t.Fatal("unexpected data")
- }
- })
- })
-
- t.Run("we honour context", func(t *testing.T) {
- // It should suffice to check one of the public methods here
- client := newAPIClient()
- ctx, cancel := context.WithCancel(context.Background())
- cancel() // test should fail
- data, err := client.FetchResource(ctx, "/")
- if !errors.Is(err, context.Canceled) {
- t.Fatal("unexpected err", err)
- }
- if data != nil {
- t.Fatal("unexpected data")
- }
- })
-
- t.Run("body logging", func(t *testing.T) {
- t.Run("logging enabled and 200 Ok", func(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(
- func(w http.ResponseWriter, r *http.Request) {
- w.Write([]byte("[]"))
- },
- ))
- logs := make(chan string, 1024)
- defer server.Close()
- var (
- input []string
- output []string
- )
- ctx := context.Background()
- err := (&apiClient{
- BaseURL: server.URL,
- HTTPClient: http.DefaultClient,
- LogBody: true,
- Logger: &mocks.Logger{
- MockDebugf: func(format string, v ...interface{}) {
- logs <- fmt.Sprintf(format, v...)
- },
- },
- }).PostJSON(ctx, "/", input, &output)
- var found int
- close(logs)
- for entry := range logs {
- if strings.HasPrefix(entry, "httpx: request body: ") {
- found |= 1 << 0
- continue
- }
- if strings.HasPrefix(entry, "httpx: response body: ") {
- found |= 1 << 1
- continue
- }
- }
- if found != (1<<0 | 1<<1) {
- t.Fatal("did not find logs")
- }
- if err != nil {
- t.Fatal(err)
- }
- })
-
- t.Run("logging enabled and 401 Unauthorized", func(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(
- func(w http.ResponseWriter, r *http.Request) {
- w.WriteHeader(401)
- w.Write([]byte("[]"))
- },
- ))
- logs := make(chan string, 1024)
- defer server.Close()
- var (
- input []string
- output []string
- )
- ctx := context.Background()
- err := (&apiClient{
- BaseURL: server.URL,
- HTTPClient: http.DefaultClient,
- LogBody: true,
- Logger: &mocks.Logger{
- MockDebugf: func(format string, v ...interface{}) {
- logs <- fmt.Sprintf(format, v...)
- },
- },
- }).PostJSON(ctx, "/", input, &output)
- var found int
- close(logs)
- for entry := range logs {
- if strings.HasPrefix(entry, "httpx: request body: ") {
- found |= 1 << 0
- continue
- }
- if strings.HasPrefix(entry, "httpx: response body: ") {
- found |= 1 << 1
- continue
- }
- }
- if found != (1<<0 | 1<<1) {
- t.Fatal("did not find logs")
- }
- if !errors.Is(err, ErrRequestFailed) {
- t.Fatal("unexpected err", err)
- }
- })
-
- t.Run("logging NOT enabled and 200 Ok", func(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(
- func(w http.ResponseWriter, r *http.Request) {
- w.Write([]byte("[]"))
- },
- ))
- logs := make(chan string, 1024)
- defer server.Close()
- var (
- input []string
- output []string
- )
- ctx := context.Background()
- err := (&apiClient{
- BaseURL: server.URL,
- HTTPClient: http.DefaultClient,
- LogBody: false, // explicit initialization
- Logger: &mocks.Logger{
- MockDebugf: func(format string, v ...interface{}) {
- logs <- fmt.Sprintf(format, v...)
- },
- },
- }).PostJSON(ctx, "/", input, &output)
- var found int
- close(logs)
- for entry := range logs {
- if strings.HasPrefix(entry, "httpx: request body: ") {
- found |= 1 << 0
- continue
- }
- if strings.HasPrefix(entry, "httpx: response body: ") {
- found |= 1 << 1
- continue
- }
- }
- if found != 0 {
- t.Fatal("did find logs")
- }
- if err != nil {
- t.Fatal(err)
- }
- })
-
- t.Run("logging NOT enabled and 401 Unauthorized", func(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(
- func(w http.ResponseWriter, r *http.Request) {
- w.WriteHeader(401)
- w.Write([]byte("[]"))
- },
- ))
- logs := make(chan string, 1024)
- defer server.Close()
- var (
- input []string
- output []string
- )
- ctx := context.Background()
- err := (&apiClient{
- BaseURL: server.URL,
- HTTPClient: http.DefaultClient,
- LogBody: false, // explicit initialization
- Logger: &mocks.Logger{
- MockDebugf: func(format string, v ...interface{}) {
- logs <- fmt.Sprintf(format, v...)
- },
- },
- }).PostJSON(ctx, "/", input, &output)
- var found int
- close(logs)
- for entry := range logs {
- if strings.HasPrefix(entry, "httpx: request body: ") {
- found |= 1 << 0
- continue
- }
- if strings.HasPrefix(entry, "httpx: response body: ") {
- found |= 1 << 1
- continue
- }
- }
- if found != 0 {
- t.Fatal("did find logs")
- }
- if !errors.Is(err, ErrRequestFailed) {
- t.Fatal("unexpected err", err)
- }
- })
- })
-}
diff --git a/pkg/legacy/measurex/easy.go b/pkg/legacy/measurex/easy.go
index 5aebf41ed..a27d6dee8 100644
--- a/pkg/legacy/measurex/easy.go
+++ b/pkg/legacy/measurex/easy.go
@@ -49,7 +49,7 @@ func (mx *Measurer) EasyHTTPRoundTripGET(ctx context.Context, timeout time.Durat
failure := err.Error()
return NewArchivalMeasurement(db.AsMeasurement()), &failure
}
- resp.Body.Close()
+ _ = resp.Body.Close()
return NewArchivalMeasurement(db.AsMeasurement()), nil
}
@@ -61,7 +61,7 @@ type EasyTLSConfig struct {
// NewEasyTLSConfig creates a new EasyTLSConfig instance.
func NewEasyTLSConfig() *EasyTLSConfig {
return &EasyTLSConfig{
- config: &tls.Config{
+ config: &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
// Because here we use nil, this causes netxlite to use
// a cached copy of Mozilla's CA pool. We don't create a
// new pool every time for performance reasons. See
@@ -98,7 +98,7 @@ func (easy *EasyTLSConfig) RootCAs(v *x509.CertPool) *EasyTLSConfig {
// asTLSConfig converts an *EasyTLSConfig to a *tls.Config.
func (easy *EasyTLSConfig) asTLSConfig() *tls.Config {
if easy == nil || easy.config == nil {
- return &tls.Config{}
+ return &tls.Config{} // #nosec G402 - we need to use a large TLS versions range for measuring
}
return easy.config
}
@@ -135,7 +135,7 @@ func (mx *Measurer) EasyTLSConnectAndHandshake(ctx context.Context, endpoint str
failure := err.Error()
return NewArchivalMeasurement(db.AsMeasurement()), &failure
}
- conn.Close()
+ _ = conn.Close()
return NewArchivalMeasurement(db.AsMeasurement()), nil
}
@@ -168,7 +168,7 @@ func (mx *Measurer) EasyTCPConnect(ctx context.Context,
failure := err.Error()
return NewArchivalMeasurement(db.AsMeasurement()), &failure
}
- conn.Close()
+ _ = conn.Close()
return NewArchivalMeasurement(db.AsMeasurement()), nil
}
@@ -272,6 +272,6 @@ func (mx *Measurer) EasyOBFS4ConnectAndHandshake(ctx context.Context,
failure := err.Error()
return NewArchivalMeasurement(db.AsMeasurement()), &failure
}
- o4conn.Close()
+ _ = o4conn.Close()
return NewArchivalMeasurement(db.AsMeasurement()), nil
}
diff --git a/pkg/legacy/measurex/http.go b/pkg/legacy/measurex/http.go
index bfe299196..237af04d9 100644
--- a/pkg/legacy/measurex/http.go
+++ b/pkg/legacy/measurex/http.go
@@ -97,7 +97,9 @@ func (mx *Measurer) NewHTTPTransportWithTLSConn(
func (mx *Measurer) NewHTTPTransportWithQUICConn(
logger model.Logger, db WritableDB, qconn quic.EarlyConnection) *HTTPTransportDB {
return mx.WrapHTTPTransport(db, netxlite.NewHTTP3Transport(
- logger, netxlite.NewSingleUseQUICDialer(qconn), &tls.Config{}))
+ logger, netxlite.NewSingleUseQUICDialer(qconn),
+ &tls.Config{}, // #nosec G402 - we need to use a large TLS versions range for measuring
+ ))
}
// HTTPTransportDB is an implementation of HTTPTransport that
diff --git a/pkg/legacy/measurex/measurer.go b/pkg/legacy/measurex/measurer.go
index 46a521eb9..77c95e809 100644
--- a/pkg/legacy/measurex/measurer.go
+++ b/pkg/legacy/measurex/measurer.go
@@ -251,7 +251,7 @@ func (mx *Measurer) TCPConnect(ctx context.Context, address string) *EndpointMea
conn, _ := mx.TCPConnectWithDB(ctx, db, address)
measurement := db.AsMeasurement()
if conn != nil {
- conn.Close()
+ _ = conn.Close()
}
return &EndpointMeasurement{
Network: NetworkTCP,
@@ -322,7 +322,7 @@ func (mx *Measurer) TLSConnectAndHandshake(ctx context.Context,
conn, _ := mx.TLSConnectAndHandshakeWithDB(ctx, db, address, config)
measurement := db.AsMeasurement()
if conn != nil {
- conn.Close()
+ _ = conn.Close()
}
return &EndpointMeasurement{
Network: NetworkTCP,
@@ -393,7 +393,7 @@ func (mx *Measurer) QUICHandshake(ctx context.Context, address string,
measurement := db.AsMeasurement()
if qconn != nil {
// TODO(bassosimone): close connection with correct message
- qconn.CloseWithError(0, "")
+ _ = qconn.CloseWithError(0, "")
}
return &EndpointMeasurement{
Network: NetworkUDP,
@@ -449,7 +449,7 @@ func (mx *Measurer) HTTPEndpointGet(
ctx context.Context, epnt *HTTPEndpoint, jar http.CookieJar) *HTTPEndpointMeasurement {
resp, m, _ := mx.httpEndpointGet(ctx, epnt, jar)
if resp != nil {
- resp.Body.Close()
+ _ = resp.Body.Close()
}
return m
}
@@ -561,11 +561,12 @@ func (mx *Measurer) httpEndpointGetHTTPS(ctx context.Context,
db WritableDB, epnt *HTTPEndpoint, jar http.CookieJar) (*http.Response, error) {
// Using a nil cert pool here forces netxlite to use a cached copy of Mozilla's
// CA bundle. See https://github.com/ooni/probe/issues/2413 for context.
- conn, err := mx.TLSConnectAndHandshakeWithDB(ctx, db, epnt.Address, &tls.Config{
- ServerName: epnt.SNI,
- NextProtos: epnt.ALPN,
- RootCAs: nil,
- })
+ conn, err := mx.TLSConnectAndHandshakeWithDB(ctx, db, epnt.Address,
+ &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
+ ServerName: epnt.SNI,
+ NextProtos: epnt.ALPN,
+ RootCAs: nil,
+ })
if err != nil {
return nil, err
}
@@ -581,11 +582,12 @@ func (mx *Measurer) httpEndpointGetQUIC(ctx context.Context,
db WritableDB, epnt *HTTPEndpoint, jar http.CookieJar) (*http.Response, error) {
// Using a nil cert pool here forces netxlite to use a cached copy of Mozilla's
// CA bundle. See https://github.com/ooni/probe/issues/2413 for context.
- qconn, err := mx.QUICHandshakeWithDB(ctx, db, epnt.Address, &tls.Config{
- ServerName: epnt.SNI,
- NextProtos: epnt.ALPN,
- RootCAs: nil,
- })
+ qconn, err := mx.QUICHandshakeWithDB(ctx, db, epnt.Address,
+ &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
+ ServerName: epnt.SNI,
+ NextProtos: epnt.ALPN,
+ RootCAs: nil,
+ })
if err != nil {
return nil, err
}
diff --git a/pkg/multierror/example_test.go b/pkg/legacy/multierror/example_test.go
similarity index 80%
rename from pkg/multierror/example_test.go
rename to pkg/legacy/multierror/example_test.go
index 09ffc9efb..35d942a53 100644
--- a/pkg/multierror/example_test.go
+++ b/pkg/legacy/multierror/example_test.go
@@ -4,7 +4,7 @@ import (
"errors"
"fmt"
- "github.com/ooni/probe-engine/pkg/multierror"
+ "github.com/ooni/probe-engine/pkg/legacy/multierror"
)
func ExampleUnion() {
diff --git a/pkg/multierror/multierror.go b/pkg/legacy/multierror/multierror.go
similarity index 78%
rename from pkg/multierror/multierror.go
rename to pkg/legacy/multierror/multierror.go
index 560a56b0c..346f80c9e 100644
--- a/pkg/multierror/multierror.go
+++ b/pkg/legacy/multierror/multierror.go
@@ -25,6 +25,9 @@ func New(root error) *Union {
}
// Unwrap returns the Root error of the Union error.
+//
+// QUIRK: we cannot change this function to be `Unwrap() []error` as
+// explained by https://github.com/ooni/probe-cli/pull/1587.
func (err Union) Unwrap() error {
return err.Root
}
@@ -56,10 +59,16 @@ func (err Union) Is(target error) bool {
// Error returns a string representation of the Union error.
func (err Union) Error() string {
+ return BuildErrorString(err.Root.Error(), err.Children...)
+}
+
+// BuildErrorString builds the error string returned by [*Union.Error] using the
+// given prefix string as the prefix and the given list of errors.
+func BuildErrorString(prefix string, errs ...error) string {
var sb strings.Builder
- sb.WriteString(err.Root.Error())
+ sb.WriteString(prefix)
sb.WriteString(": [")
- for _, c := range err.Children {
+ for _, c := range errs {
sb.WriteString(" ")
sb.WriteString(c.Error())
sb.WriteString(";")
diff --git a/pkg/multierror/multierror_test.go b/pkg/legacy/multierror/multierror_test.go
similarity index 97%
rename from pkg/multierror/multierror_test.go
rename to pkg/legacy/multierror/multierror_test.go
index f786ff8c5..1725054fc 100644
--- a/pkg/multierror/multierror_test.go
+++ b/pkg/legacy/multierror/multierror_test.go
@@ -8,7 +8,7 @@ import (
"testing"
"github.com/google/go-cmp/cmp"
- "github.com/ooni/probe-engine/pkg/multierror"
+ "github.com/ooni/probe-engine/pkg/legacy/multierror"
)
func TestEmpty(t *testing.T) {
diff --git a/pkg/measurexlite/conn.go b/pkg/measurexlite/conn.go
index 5adfdb128..5c6534459 100644
--- a/pkg/measurexlite/conn.go
+++ b/pkg/measurexlite/conn.go
@@ -39,11 +39,29 @@ type connTrace struct {
var _ net.Conn = &connTrace{}
+type remoteAddrProvider interface {
+ RemoteAddr() net.Addr
+}
+
+func safeRemoteAddrNetwork(rap remoteAddrProvider) (result string) {
+ if addr := rap.RemoteAddr(); addr != nil {
+ result = addr.Network()
+ }
+ return result
+}
+
+func safeRemoteAddrString(rap remoteAddrProvider) (result string) {
+ if addr := rap.RemoteAddr(); addr != nil {
+ result = addr.String()
+ }
+ return result
+}
+
// Read implements net.Conn.Read and saves network events.
func (c *connTrace) Read(b []byte) (int, error) {
// collect preliminary stats when the connection is surely active
- network := c.RemoteAddr().Network()
- addr := c.RemoteAddr().String()
+ network := safeRemoteAddrNetwork(c)
+ addr := safeRemoteAddrString(c)
started := c.tx.TimeSince(c.tx.ZeroTime())
// perform the underlying network operation
@@ -99,8 +117,8 @@ func (tx *Trace) CloneBytesReceivedMap() (out map[string]int64) {
// Write implements net.Conn.Write and saves network events.
func (c *connTrace) Write(b []byte) (int, error) {
- network := c.RemoteAddr().Network()
- addr := c.RemoteAddr().String()
+ network := safeRemoteAddrNetwork(c)
+ addr := safeRemoteAddrString(c)
started := c.tx.TimeSince(c.tx.ZeroTime())
count, err := c.Conn.Write(b)
diff --git a/pkg/measurexlite/conn_test.go b/pkg/measurexlite/conn_test.go
index b9a170809..6b45400ea 100644
--- a/pkg/measurexlite/conn_test.go
+++ b/pkg/measurexlite/conn_test.go
@@ -12,6 +12,43 @@ import (
"github.com/ooni/probe-engine/pkg/testingx"
)
+func TestRemoteAddrProvider(t *testing.T) {
+ t.Run("for nil address", func(t *testing.T) {
+ conn := &mocks.Conn{
+ MockRemoteAddr: func() net.Addr {
+ return nil
+ },
+ }
+ if safeRemoteAddrNetwork(conn) != "" {
+ t.Fatal("expected empty network")
+ }
+ if safeRemoteAddrString(conn) != "" {
+ t.Fatal("expected empty string")
+ }
+ })
+
+ t.Run("for common case", func(t *testing.T) {
+ conn := &mocks.Conn{
+ MockRemoteAddr: func() net.Addr {
+ return &mocks.Addr{
+ MockString: func() string {
+ return "1.1.1.1:443"
+ },
+ MockNetwork: func() string {
+ return "tcp"
+ },
+ }
+ },
+ }
+ if safeRemoteAddrNetwork(conn) != "tcp" {
+ t.Fatal("unexpected network")
+ }
+ if safeRemoteAddrString(conn) != "1.1.1.1:443" {
+ t.Fatal("unexpected string")
+ }
+ })
+}
+
func TestMaybeClose(t *testing.T) {
t.Run("with nil conn", func(t *testing.T) {
var conn net.Conn = nil
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/measurement.json
index 9192dbc6b..9b07bc403 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -188,7 +188,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/measurement.json
index 3616e06a6..fc180b797 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -188,7 +188,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/measurement.json
index bc1352ce5..2240f056e 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -305,7 +305,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/badSSLWithWrongServerName/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/badSSLWithWrongServerName/measurement.json
index 7645c9f0f..207ef4dc0 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/badSSLWithWrongServerName/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/badSSLWithWrongServerName/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -188,7 +188,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/cloudflareCAPTCHAWithHTTP/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/cloudflareCAPTCHAWithHTTP/measurement.json
index 261fd1ba9..bd3a3ac77 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/cloudflareCAPTCHAWithHTTP/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/cloudflareCAPTCHAWithHTTP/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -323,7 +323,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/cloudflareCAPTCHAWithHTTPS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/cloudflareCAPTCHAWithHTTPS/measurement.json
index 0edbec3e2..4b83894cc 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/cloudflareCAPTCHAWithHTTPS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/cloudflareCAPTCHAWithHTTPS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -308,7 +308,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPSWebsite/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPSWebsite/measurement.json
index fb2ee6123..8ef0ceca1 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPSWebsite/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPSWebsite/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org",
@@ -273,7 +273,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
@@ -284,7 +284,7 @@
},
"control": null,
"x_conn_priority_log": null,
- "control_failure": "unknown_failure: httpapi: all endpoints failed: [ connection_reset; connection_reset; connection_reset; connection_reset;]",
+ "control_failure": "connection_reset",
"x_dns_flags": 0,
"dns_experiment_failure": null,
"dns_consistency": null,
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPWebsite/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPWebsite/measurement.json
index afb7d1f76..c26a547ac 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPWebsite/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPWebsite/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org",
@@ -288,7 +288,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
@@ -299,7 +299,7 @@
},
"control": null,
"x_conn_priority_log": null,
- "control_failure": "unknown_failure: httpapi: all endpoints failed: [ connection_reset; connection_reset; connection_reset; connection_reset;]",
+ "control_failure": "connection_reset",
"x_dns_flags": 0,
"dns_experiment_failure": null,
"dns_consistency": null,
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/dnsBlockingAndroidDNSCacheNoData/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/dnsBlockingAndroidDNSCacheNoData/measurement.json
index 4de015804..dc22bb4e9 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/dnsBlockingAndroidDNSCacheNoData/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/dnsBlockingAndroidDNSCacheNoData/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -262,7 +262,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/measurement.json
index 061005b82..ab3c70e9d 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -284,7 +284,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/dnsBlockingNXDOMAIN/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/dnsBlockingNXDOMAIN/measurement.json
index b061c4c47..ef0cde814 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/dnsBlockingNXDOMAIN/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/dnsBlockingNXDOMAIN/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -262,7 +262,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/dnsHijackingToLocalhostWithHTTP/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/dnsHijackingToLocalhostWithHTTP/measurement.json
index f3432a6a9..d8b013e8f 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/dnsHijackingToLocalhostWithHTTP/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/dnsHijackingToLocalhostWithHTTP/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -280,7 +280,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/dnsHijackingToLocalhostWithHTTPS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/dnsHijackingToLocalhostWithHTTPS/measurement.json
index 9b309cf17..93b0d79ef 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/dnsHijackingToLocalhostWithHTTPS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/dnsHijackingToLocalhostWithHTTPS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -266,7 +266,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPSURL/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPSURL/measurement.json
index f1eabc940..f92572884 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPSURL/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPSURL/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -306,7 +306,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPURL/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPURL/measurement.json
index dfc05e40a..ca38de7db 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPURL/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/dnsHijackingToProxyWithHTTPURL/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -331,7 +331,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/ghostDNSBlockingWithHTTP/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/ghostDNSBlockingWithHTTP/measurement.json
index e3828c9e5..e288a9a30 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/ghostDNSBlockingWithHTTP/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/ghostDNSBlockingWithHTTP/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -257,7 +257,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/ghostDNSBlockingWithHTTPS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/ghostDNSBlockingWithHTTPS/measurement.json
index 50823c31b..6a0c06a7f 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/ghostDNSBlockingWithHTTPS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/ghostDNSBlockingWithHTTPS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -162,7 +162,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/httpBlockingConnectionReset/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/httpBlockingConnectionReset/measurement.json
index 893fbd4a2..b340c2f17 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/httpBlockingConnectionReset/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/httpBlockingConnectionReset/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -266,7 +266,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/httpDiffWithConsistentDNS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/httpDiffWithConsistentDNS/measurement.json
index 7f3137834..aede853bb 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/httpDiffWithConsistentDNS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/httpDiffWithConsistentDNS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -266,7 +266,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/measurement.json
index f1a074c3e..d2d91f45e 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -314,7 +314,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/idnaWithoutCensorshipLowercase/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/idnaWithoutCensorshipLowercase/measurement.json
index f30be11f2..3c197aa58 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/idnaWithoutCensorshipLowercase/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/idnaWithoutCensorshipLowercase/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -41,7 +41,7 @@
"answers": [
{
"asn": 208398,
- "as_org_name": "Teletech d.o.o. Beograd",
+ "as_org_name": "Edge Technology Plus d.o.o. Beograd",
"answer_type": "A",
"ipv4": "5.255.255.80",
"ttl": null
@@ -79,7 +79,7 @@
"answers": [
{
"asn": 208398,
- "as_org_name": "Teletech d.o.o. Beograd",
+ "as_org_name": "Edge Technology Plus d.o.o. Beograd",
"answer_type": "A",
"ipv4": "5.255.255.80",
"ttl": null
@@ -103,7 +103,7 @@
"answers": [
{
"asn": 208398,
- "as_org_name": "Teletech d.o.o. Beograd",
+ "as_org_name": "Edge Technology Plus d.o.o. Beograd",
"answer_type": "A",
"ipv4": "5.255.255.80",
"ttl": null
@@ -127,7 +127,7 @@
"answers": [
{
"asn": 208398,
- "as_org_name": "Teletech d.o.o. Beograd",
+ "as_org_name": "Edge Technology Plus d.o.o. Beograd",
"answer_type": "A",
"ipv4": "5.255.255.80",
"ttl": null
@@ -151,7 +151,7 @@
"answers": [
{
"asn": 208398,
- "as_org_name": "Teletech d.o.o. Beograd",
+ "as_org_name": "Edge Technology Plus d.o.o. Beograd",
"answer_type": "A",
"ipv4": "5.255.255.80",
"ttl": null
@@ -189,7 +189,7 @@
"answers": [
{
"asn": 208398,
- "as_org_name": "Teletech d.o.o. Beograd",
+ "as_org_name": "Edge Technology Plus d.o.o. Beograd",
"answer_type": "A",
"ipv4": "5.255.255.80",
"ttl": null
@@ -227,7 +227,7 @@
"answers": [
{
"asn": 208398,
- "as_org_name": "Teletech d.o.o. Beograd",
+ "as_org_name": "Edge Technology Plus d.o.o. Beograd",
"answer_type": "A",
"ipv4": "5.255.255.80",
"ttl": null
@@ -644,7 +644,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/idnaWithoutCensorshipWithFirstLetterUppercase/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/idnaWithoutCensorshipWithFirstLetterUppercase/measurement.json
index 3ffa23730..64b34d447 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/idnaWithoutCensorshipWithFirstLetterUppercase/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/idnaWithoutCensorshipWithFirstLetterUppercase/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -41,7 +41,7 @@
"answers": [
{
"asn": 208398,
- "as_org_name": "Teletech d.o.o. Beograd",
+ "as_org_name": "Edge Technology Plus d.o.o. Beograd",
"answer_type": "A",
"ipv4": "5.255.255.80",
"ttl": null
@@ -79,7 +79,7 @@
"answers": [
{
"asn": 208398,
- "as_org_name": "Teletech d.o.o. Beograd",
+ "as_org_name": "Edge Technology Plus d.o.o. Beograd",
"answer_type": "A",
"ipv4": "5.255.255.80",
"ttl": null
@@ -103,7 +103,7 @@
"answers": [
{
"asn": 208398,
- "as_org_name": "Teletech d.o.o. Beograd",
+ "as_org_name": "Edge Technology Plus d.o.o. Beograd",
"answer_type": "A",
"ipv4": "5.255.255.80",
"ttl": null
@@ -127,7 +127,7 @@
"answers": [
{
"asn": 208398,
- "as_org_name": "Teletech d.o.o. Beograd",
+ "as_org_name": "Edge Technology Plus d.o.o. Beograd",
"answer_type": "A",
"ipv4": "5.255.255.80",
"ttl": null
@@ -151,7 +151,7 @@
"answers": [
{
"asn": 208398,
- "as_org_name": "Teletech d.o.o. Beograd",
+ "as_org_name": "Edge Technology Plus d.o.o. Beograd",
"answer_type": "A",
"ipv4": "5.255.255.80",
"ttl": null
@@ -189,7 +189,7 @@
"answers": [
{
"asn": 208398,
- "as_org_name": "Teletech d.o.o. Beograd",
+ "as_org_name": "Edge Technology Plus d.o.o. Beograd",
"answer_type": "A",
"ipv4": "5.255.255.80",
"ttl": null
@@ -227,7 +227,7 @@
"answers": [
{
"asn": 208398,
- "as_org_name": "Teletech d.o.o. Beograd",
+ "as_org_name": "Edge Technology Plus d.o.o. Beograd",
"answer_type": "A",
"ipv4": "5.255.255.80",
"ttl": null
@@ -644,7 +644,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/largeFileWithHTTP/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/largeFileWithHTTP/measurement.json
index 67d10a757..1edcb5321 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/largeFileWithHTTP/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/largeFileWithHTTP/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -283,7 +283,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/largeFileWithHTTPS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/largeFileWithHTTPS/measurement.json
index aeb14c5b1..cda151e8f 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/largeFileWithHTTPS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/largeFileWithHTTPS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -268,7 +268,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/localhostWithHTTP/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/localhostWithHTTP/measurement.json
index e742cd9d2..bd9007ff2 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/localhostWithHTTP/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/localhostWithHTTP/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -145,7 +145,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/localhostWithHTTPS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/localhostWithHTTPS/measurement.json
index 9cab4b1c5..b23c19fd2 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/localhostWithHTTPS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/localhostWithHTTPS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -145,7 +145,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithBrokenLocationForHTTP/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithBrokenLocationForHTTP/measurement.json
index a2482742b..d0f1e5312 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithBrokenLocationForHTTP/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithBrokenLocationForHTTP/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -283,7 +283,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithBrokenLocationForHTTPS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithBrokenLocationForHTTPS/measurement.json
index 8fbf7bd2e..cf46ebc63 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithBrokenLocationForHTTPS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithBrokenLocationForHTTPS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -268,7 +268,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionRefusedForHTTP/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionRefusedForHTTP/measurement.json
index 9e8c095a9..58524261b 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionRefusedForHTTP/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionRefusedForHTTP/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -360,7 +360,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionRefusedForHTTPS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionRefusedForHTTPS/measurement.json
index 38ae634f9..9e52c878b 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionRefusedForHTTPS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionRefusedForHTTPS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -345,7 +345,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionResetForHTTP/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionResetForHTTP/measurement.json
index 1f5ec7241..b14d8e4cb 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionResetForHTTP/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionResetForHTTP/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -438,7 +438,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionResetForHTTPS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionResetForHTTPS/measurement.json
index f16d215d4..b001e653b 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionResetForHTTPS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenConnectionResetForHTTPS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -363,7 +363,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenEOFForHTTP/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenEOFForHTTP/measurement.json
index 15b9faf20..8cdc5517a 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenEOFForHTTP/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenEOFForHTTP/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -438,7 +438,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenEOFForHTTPS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenEOFForHTTPS/measurement.json
index 9abce6b04..8d2bcc672 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenEOFForHTTPS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenEOFForHTTPS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -363,7 +363,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/measurement.json
index c6a6142ce..52848b139 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -316,7 +316,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenTimeoutForHTTP/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenTimeoutForHTTP/measurement.json
index 521e339fa..3cfaaac82 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenTimeoutForHTTP/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenTimeoutForHTTP/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -438,7 +438,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenTimeoutForHTTPS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenTimeoutForHTTPS/measurement.json
index fa47a9b24..5954615d6 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenTimeoutForHTTPS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenTimeoutForHTTPS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -363,7 +363,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTP/analysis.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTP/analysis.json
new file mode 100644
index 000000000..369df6460
--- /dev/null
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTP/analysis.json
@@ -0,0 +1,1362 @@
+{
+ "ControlExpectations": {
+ "DNSAddresses": [
+ "172.67.144.64"
+ ],
+ "FinalResponseFailure": "unknown_error"
+ },
+ "DNSLookupSuccess": [
+ 10001,
+ 20001,
+ 30001
+ ],
+ "DNSLookupSuccessWithInvalidAddresses": [],
+ "DNSLookupSuccessWithValidAddress": [
+ 10001,
+ 20001,
+ 30001
+ ],
+ "DNSLookupSuccessWithBogonAddresses": [],
+ "DNSLookupSuccessWithInvalidAddressesClassic": [],
+ "DNSLookupSuccessWithValidAddressClassic": [
+ 10001,
+ 20001,
+ 30001
+ ],
+ "DNSLookupUnexpectedFailure": [],
+ "DNSLookupUnexplainedFailure": [],
+ "DNSExperimentFailure": null,
+ "DNSLookupExpectedFailure": [],
+ "DNSLookupExpectedSuccess": [],
+ "TCPConnectExpectedFailure": [],
+ "TCPConnectUnexpectedFailure": [],
+ "TCPConnectUnexpectedFailureDuringWebFetch": [],
+ "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
+ "TCPConnectUnexplainedFailure": [],
+ "TCPConnectUnexplainedFailureDuringWebFetch": [],
+ "TCPConnectUnexplainedFailureDuringConnectivityCheck": [],
+ "TLSHandshakeExpectedFailure": [],
+ "TLSHandshakeUnexpectedFailure": [],
+ "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+ "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
+ "TLSHandshakeUnexplainedFailure": [],
+ "TLSHandshakeUnexplainedFailureDuringWebFetch": [],
+ "TLSHandshakeUnexplainedFailureDuringConnectivityCheck": [],
+ "HTTPRoundTripUnexpectedFailure": [],
+ "HTTPRoundTripUnexplainedFailure": [],
+ "HTTPFinalResponseSuccessTLSWithoutControl": null,
+ "HTTPFinalResponseSuccessTLSWithControl": null,
+ "HTTPFinalResponseSuccessTCPWithoutControl": null,
+ "HTTPFinalResponseSuccessTCPWithControl": null,
+ "HTTPFinalResponseDiffBodyProportionFactor": null,
+ "HTTPFinalResponseDiffStatusCodeMatch": null,
+ "HTTPFinalResponseDiffTitleDifferentLongWords": null,
+ "HTTPFinalResponseDiffUncommonHeadersIntersection": null,
+ "Linear": [
+ {
+ "TagDepth": 10,
+ "Type": 3,
+ "Failure": "unknown_failure: stopped after too many redirects",
+ "TransactionID": 40011,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40011,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/5",
+ "HTTPFailure": "unknown_failure: stopped after too many redirects",
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 10,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50011,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50011,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 9,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40010,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40010,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/6",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/5",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 9,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50010,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50010,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 8,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40009,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40009,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/7",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/6",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 8,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50009,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50009,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 7,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40008,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40008,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/8",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/7",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 7,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50008,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50008,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 6,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40007,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40007,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/9",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/8",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 6,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50007,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50007,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 5,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40006,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40006,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/10",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/9",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 5,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50006,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50006,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 4,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40005,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40005,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/11",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/10",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 4,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50005,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50005,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 3,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40004,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40004,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/12",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/11",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 3,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50004,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50004,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 2,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40003,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40003,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/13",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/12",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 2,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50003,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50003,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 1,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40002,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40002,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/14",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/13",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 1,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50002,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50002,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40001,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40001,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/15",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/14",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50001,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50001,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "",
+ "TransactionID": 30001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": "A",
+ "DNSEngine": "doh",
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "",
+ "TransactionID": 20001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 20001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": "A",
+ "DNSEngine": "udp",
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "",
+ "TransactionID": 10001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 10001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": "ANY",
+ "DNSEngine": "getaddrinfo",
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "dns_no_answer",
+ "TransactionID": 30001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "dns_no_answer",
+ "DNSQueryType": "AAAA",
+ "DNSEngine": "doh",
+ "DNSResolvedAddrs": null,
+ "IPAddressOrigin": null,
+ "IPAddress": null,
+ "IPAddressASN": null,
+ "IPAddressBogon": null,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "dns_no_answer",
+ "TransactionID": 20001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 20001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "dns_no_answer",
+ "DNSQueryType": "AAAA",
+ "DNSEngine": "udp",
+ "DNSResolvedAddrs": null,
+ "IPAddressOrigin": null,
+ "IPAddress": null,
+ "IPAddressASN": null,
+ "IPAddressBogon": null,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ }
+ ]
+}
\ No newline at end of file
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTP/analysis_classic.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTP/analysis_classic.json
new file mode 100644
index 000000000..cd8382948
--- /dev/null
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTP/analysis_classic.json
@@ -0,0 +1,655 @@
+{
+ "ControlExpectations": {
+ "DNSAddresses": [
+ "172.67.144.64"
+ ],
+ "FinalResponseFailure": "unknown_error"
+ },
+ "DNSLookupSuccess": [
+ 10001
+ ],
+ "DNSLookupSuccessWithInvalidAddresses": [],
+ "DNSLookupSuccessWithValidAddress": [
+ 10001
+ ],
+ "DNSLookupSuccessWithBogonAddresses": [],
+ "DNSLookupSuccessWithInvalidAddressesClassic": [],
+ "DNSLookupSuccessWithValidAddressClassic": [
+ 10001
+ ],
+ "DNSLookupUnexpectedFailure": [],
+ "DNSLookupUnexplainedFailure": [],
+ "DNSExperimentFailure": null,
+ "DNSLookupExpectedFailure": [],
+ "DNSLookupExpectedSuccess": [],
+ "TCPConnectExpectedFailure": [],
+ "TCPConnectUnexpectedFailure": [],
+ "TCPConnectUnexpectedFailureDuringWebFetch": [],
+ "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
+ "TCPConnectUnexplainedFailure": [],
+ "TCPConnectUnexplainedFailureDuringWebFetch": [],
+ "TCPConnectUnexplainedFailureDuringConnectivityCheck": [],
+ "TLSHandshakeExpectedFailure": [],
+ "TLSHandshakeUnexpectedFailure": [],
+ "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+ "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
+ "TLSHandshakeUnexplainedFailure": [],
+ "TLSHandshakeUnexplainedFailureDuringWebFetch": [],
+ "TLSHandshakeUnexplainedFailureDuringConnectivityCheck": [],
+ "HTTPRoundTripUnexpectedFailure": [],
+ "HTTPRoundTripUnexplainedFailure": [],
+ "HTTPFinalResponseSuccessTLSWithoutControl": null,
+ "HTTPFinalResponseSuccessTLSWithControl": null,
+ "HTTPFinalResponseSuccessTCPWithoutControl": null,
+ "HTTPFinalResponseSuccessTCPWithControl": null,
+ "HTTPFinalResponseDiffBodyProportionFactor": null,
+ "HTTPFinalResponseDiffStatusCodeMatch": null,
+ "HTTPFinalResponseDiffTitleDifferentLongWords": null,
+ "HTTPFinalResponseDiffUncommonHeadersIntersection": null,
+ "Linear": [
+ {
+ "TagDepth": 10,
+ "Type": 3,
+ "Failure": "unknown_failure: stopped after too many redirects",
+ "TransactionID": 40011,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40011,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/5",
+ "HTTPFailure": "unknown_failure: stopped after too many redirects",
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 9,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40010,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40010,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/6",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/5",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 8,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40009,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40009,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/7",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/6",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 7,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40008,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40008,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/8",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/7",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 6,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40007,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40007,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/9",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/8",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 5,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40006,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40006,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/10",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/9",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 4,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40005,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40005,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/11",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/10",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 3,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40004,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40004,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/12",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/11",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 2,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40003,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40003,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/13",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/12",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 1,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40002,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40002,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/14",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/13",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40001,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40001,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/15",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/14",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "",
+ "TransactionID": 10001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 10001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": "ANY",
+ "DNSEngine": "getaddrinfo",
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ }
+ ]
+}
\ No newline at end of file
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTP/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTP/measurement.json
new file mode 100644
index 000000000..79a76a7ad
--- /dev/null
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTP/measurement.json
@@ -0,0 +1,1606 @@
+{
+ "data_format_version": "0.2.0",
+ "extensions": {
+ "dnst": 0,
+ "httpt": 0,
+ "netevents": 0,
+ "tcpconnect": 0,
+ "tlshandshake": 0,
+ "tunnel": 0
+ },
+ "input": "http://httpbin.com/redirect/15",
+ "measurement_start_time": "2024-02-12 20:33:47",
+ "probe_asn": "AS137",
+ "probe_cc": "IT",
+ "probe_ip": "127.0.0.1",
+ "probe_network_name": "Consortium GARR",
+ "report_id": "",
+ "resolver_asn": "AS137",
+ "resolver_ip": "130.192.3.21",
+ "resolver_network_name": "Consortium GARR",
+ "software_name": "ooniprobe",
+ "software_version": "3.22.0-alpha",
+ "test_helpers": {
+ "backend": {
+ "address": "https://0.th.ooni.org/",
+ "type": "https"
+ }
+ },
+ "test_keys": {
+ "agent": "redirect",
+ "client_resolver": "130.192.3.21",
+ "retries": null,
+ "socksproxy": null,
+ "network_events": null,
+ "x_dns_whoami": null,
+ "x_doh": null,
+ "x_do53": null,
+ "x_dns_duplicate_responses": null,
+ "queries": [
+ {
+ "answers": [
+ {
+ "asn": 13335,
+ "as_org_name": "Cloudflare Inc",
+ "answer_type": "A",
+ "ipv4": "172.67.144.64",
+ "ttl": null
+ }
+ ],
+ "engine": "doh",
+ "failure": null,
+ "hostname": "httpbin.com",
+ "query_type": "A",
+ "resolver_hostname": null,
+ "resolver_port": null,
+ "resolver_address": "https://dns.google/dns-query",
+ "t": 0,
+ "tags": [
+ "depth=0"
+ ],
+ "transaction_id": 30001
+ },
+ {
+ "answers": null,
+ "engine": "doh",
+ "failure": "dns_no_answer",
+ "hostname": "httpbin.com",
+ "query_type": "AAAA",
+ "resolver_hostname": null,
+ "resolver_port": null,
+ "resolver_address": "https://dns.google/dns-query",
+ "t": 0,
+ "tags": [
+ "depth=0"
+ ],
+ "transaction_id": 30001
+ },
+ {
+ "answers": [
+ {
+ "asn": 13335,
+ "as_org_name": "Cloudflare Inc",
+ "answer_type": "A",
+ "ipv4": "172.67.144.64",
+ "ttl": null
+ }
+ ],
+ "engine": "getaddrinfo",
+ "failure": null,
+ "hostname": "httpbin.com",
+ "query_type": "ANY",
+ "resolver_hostname": null,
+ "resolver_port": null,
+ "resolver_address": "",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=0"
+ ],
+ "transaction_id": 10001
+ },
+ {
+ "answers": [
+ {
+ "asn": 13335,
+ "as_org_name": "Cloudflare Inc",
+ "answer_type": "A",
+ "ipv4": "172.67.144.64",
+ "ttl": null
+ }
+ ],
+ "engine": "udp",
+ "failure": null,
+ "hostname": "httpbin.com",
+ "query_type": "A",
+ "resolver_hostname": null,
+ "resolver_port": null,
+ "resolver_address": "1.1.1.1:53",
+ "t": 0,
+ "tags": [
+ "depth=0"
+ ],
+ "transaction_id": 20001
+ },
+ {
+ "answers": null,
+ "engine": "udp",
+ "failure": "dns_no_answer",
+ "hostname": "httpbin.com",
+ "query_type": "AAAA",
+ "resolver_hostname": null,
+ "resolver_port": null,
+ "resolver_address": "1.1.1.1:53",
+ "t": 0,
+ "tags": [
+ "depth=0"
+ ],
+ "transaction_id": 20001
+ }
+ ],
+ "requests": [
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:80",
+ "failure": "unknown_failure: stopped after too many redirects",
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "http://httpbin.com/redirect/6"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "http://httpbin.com/redirect/6",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "http://httpbin.com/redirect/5"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/4"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/4"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=10",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40011
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:80",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "http://httpbin.com/redirect/7"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "http://httpbin.com/redirect/7",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "http://httpbin.com/redirect/6"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/5"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/5"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=9",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40010
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:80",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "http://httpbin.com/redirect/8"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "http://httpbin.com/redirect/8",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "http://httpbin.com/redirect/7"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/6"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/6"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=8",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40009
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:80",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "http://httpbin.com/redirect/9"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "http://httpbin.com/redirect/9",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "http://httpbin.com/redirect/8"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/7"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/7"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=7",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40008
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:80",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "http://httpbin.com/redirect/10"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "http://httpbin.com/redirect/10",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "http://httpbin.com/redirect/9"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/8"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/8"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=6",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40007
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:80",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "http://httpbin.com/redirect/11"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "http://httpbin.com/redirect/11",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "http://httpbin.com/redirect/10"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/9"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/9"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=5",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40006
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:80",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "http://httpbin.com/redirect/12"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "http://httpbin.com/redirect/12",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "http://httpbin.com/redirect/11"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/10"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/10"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=4",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40005
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:80",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "http://httpbin.com/redirect/13"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "http://httpbin.com/redirect/13",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "http://httpbin.com/redirect/12"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/11"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/11"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=3",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40004
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:80",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "http://httpbin.com/redirect/14"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "http://httpbin.com/redirect/14",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "http://httpbin.com/redirect/13"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/12"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/12"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=2",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40003
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:80",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "http://httpbin.com/redirect/15"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "http://httpbin.com/redirect/15",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "http://httpbin.com/redirect/14"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/13"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/13"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=1",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40002
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:80",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ ""
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "http://httpbin.com/redirect/15"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/14"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/14"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "tcptls_experiment",
+ "depth=0",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40001
+ }
+ ],
+ "tcp_connect": [
+ {
+ "ip": "172.67.144.64",
+ "port": 80,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "tcptls_experiment",
+ "depth=0",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40001
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 80,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=1",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40002
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 80,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=2",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40003
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 80,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=3",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40004
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 80,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=4",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40005
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 80,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=5",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40006
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 80,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=6",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40007
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 80,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=7",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40008
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 80,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=8",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40009
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 80,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=9",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40010
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 80,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=10",
+ "fetch_body=true"
+ ],
+ "transaction_id": 40011
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "tcptls_experiment",
+ "depth=0",
+ "fetch_body=false"
+ ],
+ "transaction_id": 50001
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=1",
+ "fetch_body=false"
+ ],
+ "transaction_id": 50002
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=2",
+ "fetch_body=false"
+ ],
+ "transaction_id": 50003
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=3",
+ "fetch_body=false"
+ ],
+ "transaction_id": 50004
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=4",
+ "fetch_body=false"
+ ],
+ "transaction_id": 50005
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=5",
+ "fetch_body=false"
+ ],
+ "transaction_id": 50006
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=6",
+ "fetch_body=false"
+ ],
+ "transaction_id": 50007
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=7",
+ "fetch_body=false"
+ ],
+ "transaction_id": 50008
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=8",
+ "fetch_body=false"
+ ],
+ "transaction_id": 50009
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=9",
+ "fetch_body=false"
+ ],
+ "transaction_id": 50010
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=10",
+ "fetch_body=false"
+ ],
+ "transaction_id": 50011
+ }
+ ],
+ "tls_handshakes": [
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "tcptls_experiment",
+ "depth=0",
+ "fetch_body=false"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50001
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=1",
+ "fetch_body=false"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50002
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=2",
+ "fetch_body=false"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50003
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=3",
+ "fetch_body=false"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50004
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=4",
+ "fetch_body=false"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50005
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=5",
+ "fetch_body=false"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50006
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=6",
+ "fetch_body=false"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50007
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=7",
+ "fetch_body=false"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50008
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=8",
+ "fetch_body=false"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50009
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=9",
+ "fetch_body=false"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50010
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=10",
+ "fetch_body=false"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50011
+ }
+ ],
+ "x_control_request": {
+ "http_request": "http://httpbin.com/redirect/15",
+ "http_request_headers": {
+ "Accept": [
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ "Accept-Language": [
+ "en-US,en;q=0.9"
+ ],
+ "User-Agent": [
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
+ ]
+ },
+ "tcp_connect": [
+ "172.67.144.64:443",
+ "172.67.144.64:80"
+ ],
+ "x_quic_enabled": false
+ },
+ "control": {
+ "tcp_connect": {
+ "172.67.144.64:443": {
+ "status": true,
+ "failure": null
+ },
+ "172.67.144.64:80": {
+ "status": true,
+ "failure": null
+ }
+ },
+ "tls_handshake": {
+ "172.67.144.64:443": {
+ "server_name": "httpbin.com",
+ "status": true,
+ "failure": null
+ }
+ },
+ "quic_handshake": {},
+ "http_request": {
+ "body_length": -1,
+ "discovered_h3_endpoint": "",
+ "failure": "unknown_error",
+ "title": "",
+ "headers": {},
+ "status_code": -1
+ },
+ "http3_request": null,
+ "dns": {
+ "failure": null,
+ "addrs": [
+ "172.67.144.64"
+ ]
+ },
+ "ip_info": {
+ "172.67.144.64": {
+ "asn": 13335,
+ "flags": 11
+ }
+ }
+ },
+ "x_conn_priority_log": null,
+ "control_failure": null,
+ "x_dns_flags": 0,
+ "dns_experiment_failure": null,
+ "dns_consistency": "consistent",
+ "http_experiment_failure": "unknown_failure: stopped after too many redirects",
+ "x_blocking_flags": 0,
+ "x_null_null_flags": 0,
+ "body_proportion": 0,
+ "body_length_match": null,
+ "headers_match": null,
+ "status_code_match": null,
+ "title_match": null,
+ "blocking": false,
+ "accessible": false
+ },
+ "test_name": "web_connectivity",
+ "test_runtime": 0,
+ "test_start_time": "2024-02-12 20:33:47",
+ "test_version": "0.5.28"
+}
\ No newline at end of file
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTP/observations.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTP/observations.json
new file mode 100644
index 000000000..8e626fcef
--- /dev/null
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTP/observations.json
@@ -0,0 +1,1319 @@
+{
+ "DNSLookupFailures": [
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "dns_no_answer",
+ "TransactionID": 30001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "dns_no_answer",
+ "DNSQueryType": "AAAA",
+ "DNSEngine": "doh",
+ "DNSResolvedAddrs": null,
+ "IPAddressOrigin": null,
+ "IPAddress": null,
+ "IPAddressASN": null,
+ "IPAddressBogon": null,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "dns_no_answer",
+ "TransactionID": 20001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 20001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "dns_no_answer",
+ "DNSQueryType": "AAAA",
+ "DNSEngine": "udp",
+ "DNSResolvedAddrs": null,
+ "IPAddressOrigin": null,
+ "IPAddress": null,
+ "IPAddressASN": null,
+ "IPAddressBogon": null,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ }
+ ],
+ "DNSLookupSuccesses": [
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "",
+ "TransactionID": 30001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": "A",
+ "DNSEngine": "doh",
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "",
+ "TransactionID": 10001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 10001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": "ANY",
+ "DNSEngine": "getaddrinfo",
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "",
+ "TransactionID": 20001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 20001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": "A",
+ "DNSEngine": "udp",
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ }
+ ],
+ "KnownTCPEndpoints": {
+ "40001": {
+ "TagDepth": 0,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40001,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40001,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/15",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/14",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40002": {
+ "TagDepth": 1,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40002,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40002,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/14",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/13",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40003": {
+ "TagDepth": 2,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40003,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40003,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/13",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/12",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40004": {
+ "TagDepth": 3,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40004,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40004,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/12",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/11",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40005": {
+ "TagDepth": 4,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40005,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40005,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/11",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/10",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40006": {
+ "TagDepth": 5,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40006,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40006,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/10",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/9",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40007": {
+ "TagDepth": 6,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40007,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40007,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/9",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/8",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40008": {
+ "TagDepth": 7,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40008,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40008,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/8",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/7",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40009": {
+ "TagDepth": 8,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40009,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40009,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/7",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/6",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40010": {
+ "TagDepth": 9,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40010,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40010,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/6",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/5",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40011": {
+ "TagDepth": 10,
+ "Type": 3,
+ "Failure": "unknown_failure: stopped after too many redirects",
+ "TransactionID": 40011,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40011,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/5",
+ "HTTPFailure": "unknown_failure: stopped after too many redirects",
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50001": {
+ "TagDepth": 0,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50001,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50001,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50002": {
+ "TagDepth": 1,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50002,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50002,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50003": {
+ "TagDepth": 2,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50003,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50003,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50004": {
+ "TagDepth": 3,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50004,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50004,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50005": {
+ "TagDepth": 4,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50005,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50005,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50006": {
+ "TagDepth": 5,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50006,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50006,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50007": {
+ "TagDepth": 6,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50007,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50007,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50008": {
+ "TagDepth": 7,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50008,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50008,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50009": {
+ "TagDepth": 8,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50009,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50009,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50010": {
+ "TagDepth": 9,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50010,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50010,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50011": {
+ "TagDepth": 10,
+ "Type": 2,
+ "Failure": "",
+ "TransactionID": 50011,
+ "TagFetchBody": false,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50011,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ }
+ },
+ "ControlExpectations": {
+ "DNSAddresses": [
+ "172.67.144.64"
+ ],
+ "FinalResponseFailure": "unknown_error"
+ }
+}
\ No newline at end of file
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTP/observations_classic.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTP/observations_classic.json
new file mode 100644
index 000000000..dc912b928
--- /dev/null
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTP/observations_classic.json
@@ -0,0 +1,617 @@
+{
+ "DNSLookupFailures": [],
+ "DNSLookupSuccesses": [
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "",
+ "TransactionID": 10001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 10001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": "ANY",
+ "DNSEngine": "getaddrinfo",
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ }
+ ],
+ "KnownTCPEndpoints": {
+ "40001": {
+ "TagDepth": 0,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40001,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40001,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/15",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/14",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40002": {
+ "TagDepth": 1,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40002,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40002,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/14",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/13",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40003": {
+ "TagDepth": 2,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40003,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40003,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/13",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/12",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40004": {
+ "TagDepth": 3,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40004,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40004,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/12",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/11",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40005": {
+ "TagDepth": 4,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40005,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40005,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/11",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/10",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40006": {
+ "TagDepth": 5,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40006,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40006,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/10",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/9",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40007": {
+ "TagDepth": 6,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40007,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40007,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/9",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/8",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40008": {
+ "TagDepth": 7,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40008,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40008,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/8",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/7",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40009": {
+ "TagDepth": 8,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40009,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40009,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/7",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/6",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40010": {
+ "TagDepth": 9,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 40010,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40010,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/6",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/5",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "40011": {
+ "TagDepth": 10,
+ "Type": 3,
+ "Failure": "unknown_failure: stopped after too many redirects",
+ "TransactionID": 40011,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 40011,
+ "EndpointProto": "tcp",
+ "EndpointPort": "80",
+ "EndpointAddress": "172.67.144.64:80",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": "http://httpbin.com/redirect/5",
+ "HTTPFailure": "unknown_failure: stopped after too many redirects",
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ }
+ },
+ "ControlExpectations": {
+ "DNSAddresses": [
+ "172.67.144.64"
+ ],
+ "FinalResponseFailure": "unknown_error"
+ }
+}
\ No newline at end of file
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTPS/analysis.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTPS/analysis.json
new file mode 100644
index 000000000..5c2f64379
--- /dev/null
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTPS/analysis.json
@@ -0,0 +1,845 @@
+{
+ "ControlExpectations": {
+ "DNSAddresses": [
+ "172.67.144.64"
+ ],
+ "FinalResponseFailure": "unknown_error"
+ },
+ "DNSLookupSuccess": [
+ 10001,
+ 20001,
+ 30001
+ ],
+ "DNSLookupSuccessWithInvalidAddresses": [],
+ "DNSLookupSuccessWithValidAddress": [
+ 10001,
+ 20001,
+ 30001
+ ],
+ "DNSLookupSuccessWithBogonAddresses": [],
+ "DNSLookupSuccessWithInvalidAddressesClassic": [],
+ "DNSLookupSuccessWithValidAddressClassic": [
+ 10001,
+ 20001,
+ 30001
+ ],
+ "DNSLookupUnexpectedFailure": [],
+ "DNSLookupUnexplainedFailure": [],
+ "DNSExperimentFailure": null,
+ "DNSLookupExpectedFailure": [],
+ "DNSLookupExpectedSuccess": [],
+ "TCPConnectExpectedFailure": [],
+ "TCPConnectUnexpectedFailure": [],
+ "TCPConnectUnexpectedFailureDuringWebFetch": [],
+ "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
+ "TCPConnectUnexplainedFailure": [],
+ "TCPConnectUnexplainedFailureDuringWebFetch": [],
+ "TCPConnectUnexplainedFailureDuringConnectivityCheck": [],
+ "TLSHandshakeExpectedFailure": [],
+ "TLSHandshakeUnexpectedFailure": [],
+ "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+ "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
+ "TLSHandshakeUnexplainedFailure": [],
+ "TLSHandshakeUnexplainedFailureDuringWebFetch": [],
+ "TLSHandshakeUnexplainedFailureDuringConnectivityCheck": [],
+ "HTTPRoundTripUnexpectedFailure": [],
+ "HTTPRoundTripUnexplainedFailure": [],
+ "HTTPFinalResponseSuccessTLSWithoutControl": null,
+ "HTTPFinalResponseSuccessTLSWithControl": null,
+ "HTTPFinalResponseSuccessTCPWithoutControl": null,
+ "HTTPFinalResponseSuccessTCPWithControl": null,
+ "HTTPFinalResponseDiffBodyProportionFactor": null,
+ "HTTPFinalResponseDiffStatusCodeMatch": null,
+ "HTTPFinalResponseDiffTitleDifferentLongWords": null,
+ "HTTPFinalResponseDiffUncommonHeadersIntersection": null,
+ "Linear": [
+ {
+ "TagDepth": 10,
+ "Type": 3,
+ "Failure": "unknown_failure: stopped after too many redirects",
+ "TransactionID": 50011,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50011,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/5",
+ "HTTPFailure": "unknown_failure: stopped after too many redirects",
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 9,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50010,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50010,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/6",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/5",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 8,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50009,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50009,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/7",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/6",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 7,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50008,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50008,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/8",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/7",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 6,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50007,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50007,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/9",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/8",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 5,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50006,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50006,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/10",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/9",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 4,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50005,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50005,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/11",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/10",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 3,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50004,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50004,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/12",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/11",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 2,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50003,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50003,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/13",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/12",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 1,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50002,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50002,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/14",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/13",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50001,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50001,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/15",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/14",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "",
+ "TransactionID": 30001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": "A",
+ "DNSEngine": "doh",
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "",
+ "TransactionID": 20001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 20001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": "A",
+ "DNSEngine": "udp",
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "",
+ "TransactionID": 10001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 10001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": "ANY",
+ "DNSEngine": "getaddrinfo",
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "dns_no_answer",
+ "TransactionID": 30001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "dns_no_answer",
+ "DNSQueryType": "AAAA",
+ "DNSEngine": "doh",
+ "DNSResolvedAddrs": null,
+ "IPAddressOrigin": null,
+ "IPAddress": null,
+ "IPAddressASN": null,
+ "IPAddressBogon": null,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "dns_no_answer",
+ "TransactionID": 20001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 20001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "dns_no_answer",
+ "DNSQueryType": "AAAA",
+ "DNSEngine": "udp",
+ "DNSResolvedAddrs": null,
+ "IPAddressOrigin": null,
+ "IPAddress": null,
+ "IPAddressASN": null,
+ "IPAddressBogon": null,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ }
+ ]
+}
\ No newline at end of file
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTPS/analysis_classic.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTPS/analysis_classic.json
new file mode 100644
index 000000000..430090a7f
--- /dev/null
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTPS/analysis_classic.json
@@ -0,0 +1,656 @@
+{
+ "ControlExpectations": {
+ "DNSAddresses": [
+ "172.67.144.64"
+ ],
+ "FinalResponseFailure": "unknown_error"
+ },
+ "DNSLookupSuccess": [
+ 10001
+ ],
+ "DNSLookupSuccessWithInvalidAddresses": [],
+ "DNSLookupSuccessWithValidAddress": [
+ 10001,
+ 30001
+ ],
+ "DNSLookupSuccessWithBogonAddresses": [],
+ "DNSLookupSuccessWithInvalidAddressesClassic": [],
+ "DNSLookupSuccessWithValidAddressClassic": [
+ 10001
+ ],
+ "DNSLookupUnexpectedFailure": [],
+ "DNSLookupUnexplainedFailure": [],
+ "DNSExperimentFailure": null,
+ "DNSLookupExpectedFailure": [],
+ "DNSLookupExpectedSuccess": [],
+ "TCPConnectExpectedFailure": [],
+ "TCPConnectUnexpectedFailure": [],
+ "TCPConnectUnexpectedFailureDuringWebFetch": [],
+ "TCPConnectUnexpectedFailureDuringConnectivityCheck": [],
+ "TCPConnectUnexplainedFailure": [],
+ "TCPConnectUnexplainedFailureDuringWebFetch": [],
+ "TCPConnectUnexplainedFailureDuringConnectivityCheck": [],
+ "TLSHandshakeExpectedFailure": [],
+ "TLSHandshakeUnexpectedFailure": [],
+ "TLSHandshakeUnexpectedFailureDuringWebFetch": [],
+ "TLSHandshakeUnexpectedFailureDuringConnectivityCheck": [],
+ "TLSHandshakeUnexplainedFailure": [],
+ "TLSHandshakeUnexplainedFailureDuringWebFetch": [],
+ "TLSHandshakeUnexplainedFailureDuringConnectivityCheck": [],
+ "HTTPRoundTripUnexpectedFailure": [],
+ "HTTPRoundTripUnexplainedFailure": [],
+ "HTTPFinalResponseSuccessTLSWithoutControl": null,
+ "HTTPFinalResponseSuccessTLSWithControl": null,
+ "HTTPFinalResponseSuccessTCPWithoutControl": null,
+ "HTTPFinalResponseSuccessTCPWithControl": null,
+ "HTTPFinalResponseDiffBodyProportionFactor": null,
+ "HTTPFinalResponseDiffStatusCodeMatch": null,
+ "HTTPFinalResponseDiffTitleDifferentLongWords": null,
+ "HTTPFinalResponseDiffUncommonHeadersIntersection": null,
+ "Linear": [
+ {
+ "TagDepth": 10,
+ "Type": 3,
+ "Failure": "unknown_failure: stopped after too many redirects",
+ "TransactionID": 50011,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50011,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/5",
+ "HTTPFailure": "unknown_failure: stopped after too many redirects",
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 9,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50010,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50010,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/6",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/5",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 8,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50009,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50009,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/7",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/6",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 7,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50008,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50008,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/8",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/7",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 6,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50007,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50007,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/9",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/8",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 5,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50006,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50006,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/10",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/9",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 4,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50005,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50005,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/11",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/10",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 3,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50004,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50004,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/12",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/11",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 2,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50003,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50003,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/13",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/12",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 1,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50002,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50002,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/14",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/13",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50001,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50001,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/15",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/14",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "",
+ "TransactionID": 10001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 10001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": "ANY",
+ "DNSEngine": "getaddrinfo",
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ }
+ ]
+}
\ No newline at end of file
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTPS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTPS/measurement.json
new file mode 100644
index 000000000..95bdb8305
--- /dev/null
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTPS/measurement.json
@@ -0,0 +1,1447 @@
+{
+ "data_format_version": "0.2.0",
+ "extensions": {
+ "dnst": 0,
+ "httpt": 0,
+ "netevents": 0,
+ "tcpconnect": 0,
+ "tlshandshake": 0,
+ "tunnel": 0
+ },
+ "input": "https://httpbin.com/redirect/15",
+ "measurement_start_time": "2024-02-12 20:33:47",
+ "probe_asn": "AS137",
+ "probe_cc": "IT",
+ "probe_ip": "127.0.0.1",
+ "probe_network_name": "Consortium GARR",
+ "report_id": "",
+ "resolver_asn": "AS137",
+ "resolver_ip": "130.192.3.21",
+ "resolver_network_name": "Consortium GARR",
+ "software_name": "ooniprobe",
+ "software_version": "3.22.0-alpha",
+ "test_helpers": {
+ "backend": {
+ "address": "https://0.th.ooni.org/",
+ "type": "https"
+ }
+ },
+ "test_keys": {
+ "agent": "redirect",
+ "client_resolver": "130.192.3.21",
+ "retries": null,
+ "socksproxy": null,
+ "network_events": null,
+ "x_dns_whoami": null,
+ "x_doh": null,
+ "x_do53": null,
+ "x_dns_duplicate_responses": null,
+ "queries": [
+ {
+ "answers": [
+ {
+ "asn": 13335,
+ "as_org_name": "Cloudflare Inc",
+ "answer_type": "A",
+ "ipv4": "172.67.144.64",
+ "ttl": null
+ }
+ ],
+ "engine": "doh",
+ "failure": null,
+ "hostname": "httpbin.com",
+ "query_type": "A",
+ "resolver_hostname": null,
+ "resolver_port": null,
+ "resolver_address": "https://dns.google/dns-query",
+ "t": 0,
+ "tags": [
+ "depth=0"
+ ],
+ "transaction_id": 30001
+ },
+ {
+ "answers": null,
+ "engine": "doh",
+ "failure": "dns_no_answer",
+ "hostname": "httpbin.com",
+ "query_type": "AAAA",
+ "resolver_hostname": null,
+ "resolver_port": null,
+ "resolver_address": "https://dns.google/dns-query",
+ "t": 0,
+ "tags": [
+ "depth=0"
+ ],
+ "transaction_id": 30001
+ },
+ {
+ "answers": [
+ {
+ "asn": 13335,
+ "as_org_name": "Cloudflare Inc",
+ "answer_type": "A",
+ "ipv4": "172.67.144.64",
+ "ttl": null
+ }
+ ],
+ "engine": "getaddrinfo",
+ "failure": null,
+ "hostname": "httpbin.com",
+ "query_type": "ANY",
+ "resolver_hostname": null,
+ "resolver_port": null,
+ "resolver_address": "",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=0"
+ ],
+ "transaction_id": 10001
+ },
+ {
+ "answers": [
+ {
+ "asn": 13335,
+ "as_org_name": "Cloudflare Inc",
+ "answer_type": "A",
+ "ipv4": "172.67.144.64",
+ "ttl": null
+ }
+ ],
+ "engine": "udp",
+ "failure": null,
+ "hostname": "httpbin.com",
+ "query_type": "A",
+ "resolver_hostname": null,
+ "resolver_port": null,
+ "resolver_address": "1.1.1.1:53",
+ "t": 0,
+ "tags": [
+ "depth=0"
+ ],
+ "transaction_id": 20001
+ },
+ {
+ "answers": null,
+ "engine": "udp",
+ "failure": "dns_no_answer",
+ "hostname": "httpbin.com",
+ "query_type": "AAAA",
+ "resolver_hostname": null,
+ "resolver_port": null,
+ "resolver_address": "1.1.1.1:53",
+ "t": 0,
+ "tags": [
+ "depth=0"
+ ],
+ "transaction_id": 20001
+ }
+ ],
+ "requests": [
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "alpn": "http/1.1",
+ "failure": "unknown_failure: stopped after too many redirects",
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "https://httpbin.com/redirect/6"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "https://httpbin.com/redirect/6",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "https://httpbin.com/redirect/5"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/4"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/4"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=10",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50011
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "alpn": "http/1.1",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "https://httpbin.com/redirect/7"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "https://httpbin.com/redirect/7",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "https://httpbin.com/redirect/6"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/5"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/5"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=9",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50010
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "alpn": "http/1.1",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "https://httpbin.com/redirect/8"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "https://httpbin.com/redirect/8",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "https://httpbin.com/redirect/7"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/6"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/6"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=8",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50009
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "alpn": "http/1.1",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "https://httpbin.com/redirect/9"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "https://httpbin.com/redirect/9",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "https://httpbin.com/redirect/8"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/7"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/7"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=7",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50008
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "alpn": "http/1.1",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "https://httpbin.com/redirect/10"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "https://httpbin.com/redirect/10",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "https://httpbin.com/redirect/9"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/8"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/8"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=6",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50007
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "alpn": "http/1.1",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "https://httpbin.com/redirect/11"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "https://httpbin.com/redirect/11",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "https://httpbin.com/redirect/10"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/9"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/9"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=5",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50006
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "alpn": "http/1.1",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "https://httpbin.com/redirect/12"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "https://httpbin.com/redirect/12",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "https://httpbin.com/redirect/11"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/10"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/10"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=4",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50005
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "alpn": "http/1.1",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "https://httpbin.com/redirect/13"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "https://httpbin.com/redirect/13",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "https://httpbin.com/redirect/12"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/11"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/11"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=3",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50004
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "alpn": "http/1.1",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "https://httpbin.com/redirect/14"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "https://httpbin.com/redirect/14",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "https://httpbin.com/redirect/13"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/12"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/12"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=2",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50003
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "alpn": "http/1.1",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ "https://httpbin.com/redirect/15"
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "https://httpbin.com/redirect/15",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "https://httpbin.com/redirect/14"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/13"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/13"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=1",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50002
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "alpn": "http/1.1",
+ "failure": null,
+ "request": {
+ "body": "",
+ "body_is_truncated": false,
+ "headers_list": [
+ [
+ "Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ [
+ "Accept-Language",
+ "en-US,en;q=0.9"
+ ],
+ [
+ "Host",
+ "httpbin.com"
+ ],
+ [
+ "Referer",
+ ""
+ ],
+ [
+ "User-Agent",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ ]
+ ],
+ "headers": {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Host": "httpbin.com",
+ "Referer": "",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/[scrubbed] Safari/537.3"
+ },
+ "method": "GET",
+ "tor": {
+ "exit_ip": null,
+ "exit_name": null,
+ "is_tor": false
+ },
+ "x_transport": "tcp",
+ "url": "https://httpbin.com/redirect/15"
+ },
+ "response": {
+ "body": "",
+ "body_is_truncated": false,
+ "code": 302,
+ "headers_list": [
+ [
+ "Content-Length",
+ "0"
+ ],
+ [
+ "Date",
+ "Thu, 24 Aug 2023 14:35:29 GMT"
+ ],
+ [
+ "Location",
+ "/redirect/14"
+ ]
+ ],
+ "headers": {
+ "Content-Length": "0",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ "Location": "/redirect/14"
+ }
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "tcptls_experiment",
+ "depth=0",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50001
+ }
+ ],
+ "tcp_connect": [
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "tcptls_experiment",
+ "depth=0",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50001
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=1",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50002
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=2",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50003
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=3",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50004
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=4",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50005
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=5",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50006
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=6",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50007
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=7",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50008
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=8",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50009
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=9",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50010
+ },
+ {
+ "ip": "172.67.144.64",
+ "port": 443,
+ "status": {
+ "failure": null,
+ "success": true
+ },
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=10",
+ "fetch_body=true"
+ ],
+ "transaction_id": 50011
+ }
+ ],
+ "tls_handshakes": [
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "tcptls_experiment",
+ "depth=0",
+ "fetch_body=true"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50001
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=1",
+ "fetch_body=true"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50002
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=2",
+ "fetch_body=true"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50003
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=3",
+ "fetch_body=true"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50004
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=4",
+ "fetch_body=true"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50005
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=5",
+ "fetch_body=true"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50006
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=6",
+ "fetch_body=true"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50007
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=7",
+ "fetch_body=true"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50008
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=8",
+ "fetch_body=true"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50009
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=9",
+ "fetch_body=true"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50010
+ },
+ {
+ "network": "tcp",
+ "address": "172.67.144.64:443",
+ "cipher_suite": "TLS_AES_128_GCM_SHA256",
+ "failure": null,
+ "negotiated_protocol": "http/1.1",
+ "no_tls_verify": false,
+ "peer_certificates": null,
+ "server_name": "httpbin.com",
+ "t": 0,
+ "tags": [
+ "classic",
+ "depth=10",
+ "fetch_body=true"
+ ],
+ "tls_version": "TLSv1.3",
+ "transaction_id": 50011
+ }
+ ],
+ "x_control_request": {
+ "http_request": "https://httpbin.com/redirect/15",
+ "http_request_headers": {
+ "Accept": [
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ ],
+ "Accept-Language": [
+ "en-US,en;q=0.9"
+ ],
+ "User-Agent": [
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
+ ]
+ },
+ "tcp_connect": [
+ "172.67.144.64:443",
+ "172.67.144.64:80"
+ ],
+ "x_quic_enabled": false
+ },
+ "control": {
+ "tcp_connect": {
+ "172.67.144.64:443": {
+ "status": true,
+ "failure": null
+ }
+ },
+ "tls_handshake": {
+ "172.67.144.64:443": {
+ "server_name": "httpbin.com",
+ "status": true,
+ "failure": null
+ }
+ },
+ "quic_handshake": {},
+ "http_request": {
+ "body_length": -1,
+ "discovered_h3_endpoint": "",
+ "failure": "unknown_error",
+ "title": "",
+ "headers": {},
+ "status_code": -1
+ },
+ "http3_request": null,
+ "dns": {
+ "failure": null,
+ "addrs": [
+ "172.67.144.64"
+ ]
+ },
+ "ip_info": {
+ "172.67.144.64": {
+ "asn": 13335,
+ "flags": 11
+ }
+ }
+ },
+ "x_conn_priority_log": null,
+ "control_failure": null,
+ "x_dns_flags": 0,
+ "dns_experiment_failure": null,
+ "dns_consistency": "consistent",
+ "http_experiment_failure": "unknown_failure: stopped after too many redirects",
+ "x_blocking_flags": 0,
+ "x_null_null_flags": 0,
+ "body_proportion": 0,
+ "body_length_match": null,
+ "headers_match": null,
+ "status_code_match": null,
+ "title_match": null,
+ "blocking": false,
+ "accessible": false
+ },
+ "test_name": "web_connectivity",
+ "test_runtime": 0,
+ "test_start_time": "2024-02-12 20:33:47",
+ "test_version": "0.5.28"
+}
\ No newline at end of file
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTPS/observations.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTPS/observations.json
new file mode 100644
index 000000000..e0f4907b9
--- /dev/null
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTPS/observations.json
@@ -0,0 +1,802 @@
+{
+ "DNSLookupFailures": [
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "dns_no_answer",
+ "TransactionID": 30001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "dns_no_answer",
+ "DNSQueryType": "AAAA",
+ "DNSEngine": "doh",
+ "DNSResolvedAddrs": null,
+ "IPAddressOrigin": null,
+ "IPAddress": null,
+ "IPAddressASN": null,
+ "IPAddressBogon": null,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "dns_no_answer",
+ "TransactionID": 20001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 20001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "dns_no_answer",
+ "DNSQueryType": "AAAA",
+ "DNSEngine": "udp",
+ "DNSResolvedAddrs": null,
+ "IPAddressOrigin": null,
+ "IPAddress": null,
+ "IPAddressASN": null,
+ "IPAddressBogon": null,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ }
+ ],
+ "DNSLookupSuccesses": [
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "",
+ "TransactionID": 30001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": "A",
+ "DNSEngine": "doh",
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "",
+ "TransactionID": 10001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 10001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": "ANY",
+ "DNSEngine": "getaddrinfo",
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "",
+ "TransactionID": 20001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 20001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": "A",
+ "DNSEngine": "udp",
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ }
+ ],
+ "KnownTCPEndpoints": {
+ "50001": {
+ "TagDepth": 0,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50001,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50001,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/15",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/14",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50002": {
+ "TagDepth": 1,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50002,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50002,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/14",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/13",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50003": {
+ "TagDepth": 2,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50003,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50003,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/13",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/12",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50004": {
+ "TagDepth": 3,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50004,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50004,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/12",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/11",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50005": {
+ "TagDepth": 4,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50005,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50005,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/11",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/10",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50006": {
+ "TagDepth": 5,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50006,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50006,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/10",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/9",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50007": {
+ "TagDepth": 6,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50007,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50007,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/9",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/8",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50008": {
+ "TagDepth": 7,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50008,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50008,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/8",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/7",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50009": {
+ "TagDepth": 8,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50009,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50009,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/7",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/6",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50010": {
+ "TagDepth": 9,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50010,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50010,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/6",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/5",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50011": {
+ "TagDepth": 10,
+ "Type": 3,
+ "Failure": "unknown_failure: stopped after too many redirects",
+ "TransactionID": 50011,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50011,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/5",
+ "HTTPFailure": "unknown_failure: stopped after too many redirects",
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ }
+ },
+ "ControlExpectations": {
+ "DNSAddresses": [
+ "172.67.144.64"
+ ],
+ "FinalResponseFailure": "unknown_error"
+ }
+}
\ No newline at end of file
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTPS/observations_classic.json b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTPS/observations_classic.json
new file mode 100644
index 000000000..8d324784a
--- /dev/null
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/redirectWithMoreThanTenRedirectsAndHTTPS/observations_classic.json
@@ -0,0 +1,617 @@
+{
+ "DNSLookupFailures": [],
+ "DNSLookupSuccesses": [
+ {
+ "TagDepth": 0,
+ "Type": 0,
+ "Failure": "",
+ "TransactionID": 10001,
+ "TagFetchBody": null,
+ "DNSTransactionID": 10001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": "ANY",
+ "DNSEngine": "getaddrinfo",
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": null,
+ "EndpointProto": null,
+ "EndpointPort": null,
+ "EndpointAddress": null,
+ "TCPConnectFailure": null,
+ "TLSHandshakeFailure": null,
+ "TLSServerName": null,
+ "HTTPRequestURL": null,
+ "HTTPFailure": null,
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": null,
+ "ControlTLSHandshakeFailure": null,
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ }
+ ],
+ "KnownTCPEndpoints": {
+ "50001": {
+ "TagDepth": 0,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50001,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50001,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/15",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/14",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50002": {
+ "TagDepth": 1,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50002,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50002,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/14",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/13",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50003": {
+ "TagDepth": 2,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50003,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50003,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/13",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/12",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50004": {
+ "TagDepth": 3,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50004,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50004,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/12",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/11",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50005": {
+ "TagDepth": 4,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50005,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50005,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/11",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/10",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50006": {
+ "TagDepth": 5,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50006,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50006,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/10",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/9",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50007": {
+ "TagDepth": 6,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50007,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50007,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/9",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/8",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50008": {
+ "TagDepth": 7,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50008,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50008,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/8",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/7",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50009": {
+ "TagDepth": 8,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50009,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50009,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/7",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/6",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50010": {
+ "TagDepth": 9,
+ "Type": 3,
+ "Failure": "",
+ "TransactionID": 50010,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50010,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/6",
+ "HTTPFailure": "",
+ "HTTPResponseStatusCode": 302,
+ "HTTPResponseBodyLength": 0,
+ "HTTPResponseBodyIsTruncated": false,
+ "HTTPResponseHeadersKeys": {
+ "Content-Length": true,
+ "Date": true,
+ "Location": true
+ },
+ "HTTPResponseLocation": "/redirect/5",
+ "HTTPResponseTitle": "",
+ "HTTPResponseIsFinal": false,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ },
+ "50011": {
+ "TagDepth": 10,
+ "Type": 3,
+ "Failure": "unknown_failure: stopped after too many redirects",
+ "TransactionID": 50011,
+ "TagFetchBody": true,
+ "DNSTransactionID": 30001,
+ "DNSDomain": "httpbin.com",
+ "DNSLookupFailure": "",
+ "DNSQueryType": null,
+ "DNSEngine": null,
+ "DNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "IPAddressOrigin": "dns",
+ "IPAddress": "172.67.144.64",
+ "IPAddressASN": 13335,
+ "IPAddressBogon": false,
+ "EndpointTransactionID": 50011,
+ "EndpointProto": "tcp",
+ "EndpointPort": "443",
+ "EndpointAddress": "172.67.144.64:443",
+ "TCPConnectFailure": "",
+ "TLSHandshakeFailure": "",
+ "TLSServerName": "httpbin.com",
+ "HTTPRequestURL": "https://httpbin.com/redirect/5",
+ "HTTPFailure": "unknown_failure: stopped after too many redirects",
+ "HTTPResponseStatusCode": null,
+ "HTTPResponseBodyLength": null,
+ "HTTPResponseBodyIsTruncated": null,
+ "HTTPResponseHeadersKeys": null,
+ "HTTPResponseLocation": null,
+ "HTTPResponseTitle": null,
+ "HTTPResponseIsFinal": null,
+ "ControlDNSDomain": "httpbin.com",
+ "ControlDNSLookupFailure": "",
+ "ControlDNSResolvedAddrs": [
+ "172.67.144.64"
+ ],
+ "ControlTCPConnectFailure": "",
+ "ControlTLSHandshakeFailure": "",
+ "ControlHTTPFailure": "unknown_error",
+ "ControlHTTPResponseStatusCode": null,
+ "ControlHTTPResponseBodyLength": null,
+ "ControlHTTPResponseHeadersKeys": null,
+ "ControlHTTPResponseTitle": null
+ }
+ },
+ "ControlExpectations": {
+ "DNSAddresses": [
+ "172.67.144.64"
+ ],
+ "FinalResponseFailure": "unknown_error"
+ }
+}
\ No newline at end of file
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/successWithHTTP/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/successWithHTTP/measurement.json
index 67aa6c253..7b1971579 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/successWithHTTP/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/successWithHTTP/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -288,7 +288,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/successWithHTTPS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/successWithHTTPS/measurement.json
index be52dec78..ff7419f4f 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/successWithHTTPS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/successWithHTTPS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -273,7 +273,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectTimeout/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectTimeout/measurement.json
index 82ec226a5..11406e0a8 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectTimeout/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectTimeout/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -168,7 +168,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/measurement.json
index d373370e2..1179cb904 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -316,7 +316,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/throttlingWithHTTP/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/throttlingWithHTTP/measurement.json
index 0fac157d9..7f2403762 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/throttlingWithHTTP/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/throttlingWithHTTP/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -283,7 +283,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/throttlingWithHTTPS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/throttlingWithHTTPS/measurement.json
index 8fca4a3f3..da00524e5 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/throttlingWithHTTPS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/throttlingWithHTTPS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -268,7 +268,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/tlsBlockingConnectionResetWithConsistentDNS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/tlsBlockingConnectionResetWithConsistentDNS/measurement.json
index e38835fdc..a3dbbaed7 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/tlsBlockingConnectionResetWithConsistentDNS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/tlsBlockingConnectionResetWithConsistentDNS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -188,7 +188,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/tlsBlockingConnectionResetWithInconsistentDNS/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/tlsBlockingConnectionResetWithInconsistentDNS/measurement.json
index 9d93d6a81..1333a5d89 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/tlsBlockingConnectionResetWithInconsistentDNS/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/tlsBlockingConnectionResetWithInconsistentDNS/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -221,7 +221,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/websiteDownNXDOMAIN/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/websiteDownNXDOMAIN/measurement.json
index 800085e87..ee793bdf8 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/websiteDownNXDOMAIN/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/websiteDownNXDOMAIN/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -131,7 +131,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": null,
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/websiteDownNoAddrs/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/websiteDownNoAddrs/measurement.json
index ab344bcd7..4c80d7781 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/websiteDownNoAddrs/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/websiteDownNoAddrs/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -151,7 +151,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": null,
diff --git a/pkg/minipipeline/testdata/webconnectivity/generated/websiteDownTCPConnect/measurement.json b/pkg/minipipeline/testdata/webconnectivity/generated/websiteDownTCPConnect/measurement.json
index 915e801e4..bc43a718f 100644
--- a/pkg/minipipeline/testdata/webconnectivity/generated/websiteDownTCPConnect/measurement.json
+++ b/pkg/minipipeline/testdata/webconnectivity/generated/websiteDownTCPConnect/measurement.json
@@ -19,7 +19,7 @@
"resolver_ip": "130.192.3.21",
"resolver_network_name": "Consortium GARR",
"software_name": "ooniprobe",
- "software_version": "3.21.0-alpha",
+ "software_version": "3.22.0-alpha",
"test_helpers": {
"backend": {
"address": "https://0.th.ooni.org/",
@@ -168,7 +168,7 @@
"en-US,en;q=0.9"
],
"User-Agent": [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.3"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
]
},
"tcp_connect": [
diff --git a/pkg/mocks/session.go b/pkg/mocks/session.go
index 44567b552..e18f4030c 100644
--- a/pkg/mocks/session.go
+++ b/pkg/mocks/session.go
@@ -55,7 +55,7 @@ type Session struct {
MockNewSubmitter func(ctx context.Context) (model.Submitter, error)
MockCheckIn func(ctx context.Context,
- config *model.OOAPICheckInConfig) (*model.OOAPICheckInResultNettests, error)
+ config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error)
}
func (sess *Session) GetTestHelpersByName(name string) ([]model.OOAPIService, bool) {
@@ -148,6 +148,6 @@ func (sess *Session) NewSubmitter(ctx context.Context) (model.Submitter, error)
}
func (sess *Session) CheckIn(ctx context.Context,
- config *model.OOAPICheckInConfig) (*model.OOAPICheckInResultNettests, error) {
+ config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) {
return sess.MockCheckIn(ctx, config)
}
diff --git a/pkg/mocks/session_test.go b/pkg/mocks/session_test.go
index b52c4228f..cb326c3e8 100644
--- a/pkg/mocks/session_test.go
+++ b/pkg/mocks/session_test.go
@@ -326,7 +326,7 @@ func TestSession(t *testing.T) {
t.Run("CheckIn", func(t *testing.T) {
expected := errors.New("mocked err")
s := &Session{
- MockCheckIn: func(ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResultNettests, error) {
+ MockCheckIn: func(ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) {
return nil, expected
},
}
diff --git a/pkg/model/experiment.go b/pkg/model/experiment.go
index 29385fa2e..410233f6f 100644
--- a/pkg/model/experiment.go
+++ b/pkg/model/experiment.go
@@ -7,8 +7,12 @@ package model
import (
"context"
+ "errors"
)
+// ErrNoAvailableTestHelpers is emitted when there are no available test helpers.
+var ErrNoAvailableTestHelpers = errors.New("no available helpers")
+
// ExperimentSession is the experiment's view of a session.
type ExperimentSession interface {
// GetTestHelpersByName returns a list of test helpers with the given name.
@@ -196,11 +200,6 @@ type Experiment interface {
// when used with an asynchronous experiment.
MeasureWithContext(ctx context.Context, input string) (measurement *Measurement, err error)
- // SaveMeasurement saves a measurement on the specified file path.
- //
- // Deprecated: new code should use a Saver.
- SaveMeasurement(measurement *Measurement, filePath string) error
-
// SubmitAndUpdateMeasurementContext submits a measurement and updates the
// fields whose value has changed as part of the submission.
//
diff --git a/pkg/model/http.go b/pkg/model/http.go
index f717274b8..9e04df769 100644
--- a/pkg/model/http.go
+++ b/pkg/model/http.go
@@ -13,9 +13,9 @@ const (
HTTPHeaderAcceptLanguage = "en-US,en;q=0.9"
// HTTPHeaderUserAgent is the User-Agent header used for measuring. The current header
- // is 28.39% of the browser population as of 2023-12-13 according to the
+ // is 36.86% of the browser population as of 2024-05-13 according to the
// https://www.useragents.me/ webpage.
- HTTPHeaderUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3"
+ HTTPHeaderUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.3"
)
// Additional strings used to report HTTP errors. They're currently only used by
diff --git a/pkg/must/must.go b/pkg/must/must.go
index 75d6e2fc7..7b45e7a41 100644
--- a/pkg/must/must.go
+++ b/pkg/must/must.go
@@ -20,7 +20,7 @@ import (
// CreateFile is like [os.Create] but calls
// [runtimex.PanicOnError] on failure.
func CreateFile(name string) *File {
- fp, err := os.Create(name)
+ fp, err := os.Create(name) // #nosec G304 - this is working as intended
runtimex.PanicOnError(err, "os.Create failed")
return &File{fp}
}
@@ -28,7 +28,7 @@ func CreateFile(name string) *File {
// OpenFile is like [os.Open] but calls
// [runtimex.PanicOnError] on failure.
func OpenFile(name string) *File {
- fp, err := os.Open(name)
+ fp, err := os.Open(name) // #nosec G304 - this is working as intended
runtimex.PanicOnError(err, "os.Open failed")
return &File{fp}
}
@@ -143,7 +143,7 @@ func WriteFile(filename string, content []byte, mode fs.FileMode) {
// ReadFile is like [os.ReadFile] but calls
// [runtimex.PanicOnError] on failure.
func ReadFile(filename string) []byte {
- data, err := os.ReadFile(filename)
+ data, err := os.ReadFile(filename) // #nosec G304 - this is working as intended
runtimex.PanicOnError(err, "os.ReadFile failed")
return data
}
diff --git a/pkg/netemx/cloudflare.go b/pkg/netemx/cloudflare.go
index f8a6c516b..45c4a318a 100644
--- a/pkg/netemx/cloudflare.go
+++ b/pkg/netemx/cloudflare.go
@@ -192,7 +192,7 @@ func CloudflareCAPTCHAHandler() http.Handler {
if address == DefaultClientAddress {
log.Printf("CLOUDFLARE_CACHE: request from %s => 503", address)
w.WriteHeader(http.StatusServiceUnavailable)
- w.Write(cloudflareCAPTCHAWebPage)
+ _, _ = w.Write(cloudflareCAPTCHAWebPage)
return
}
@@ -200,6 +200,6 @@ func CloudflareCAPTCHAHandler() http.Handler {
// otherwise => 200
log.Printf("CLOUDFLARE_CACHE: request from %s => 200", address)
w.WriteHeader(http.StatusOK)
- w.Write([]byte(ExampleWebPage))
+ _, _ = w.Write([]byte(ExampleWebPage))
})
}
diff --git a/pkg/netemx/http.go b/pkg/netemx/http.go
index a529c3532..1f38d6fb2 100644
--- a/pkg/netemx/http.go
+++ b/pkg/netemx/http.go
@@ -99,7 +99,7 @@ func (srv *httpCleartextServer) mustListenPortLocked(handler http.Handler, ipAdd
listener := runtimex.Try1(srv.unet.ListenTCP("tcp", addr))
// serve requests in a background goroutine
- srvr := &http.Server{Handler: handler}
+ srvr := &http.Server{Handler: handler} // #nosec G112 - just a testing server
go srvr.Serve(listener)
// make sure we track the server (the .Serve method will close the
diff --git a/pkg/netemx/httpbin.go b/pkg/netemx/httpbin.go
index 2cb6a7b99..d563c4b3e 100644
--- a/pkg/netemx/httpbin.go
+++ b/pkg/netemx/httpbin.go
@@ -1,8 +1,11 @@
package netemx
import (
+ "fmt"
"net"
"net/http"
+ "strconv"
+ "strings"
"github.com/ooni/netem"
)
@@ -29,7 +32,7 @@ func HTTPBinHandlerFactory() HTTPHandlerFactory {
// Any other request URL causes a 404 respose.
func HTTPBinHandler() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Header().Add("Date", "Thu, 24 Aug 2023 14:35:29 GMT")
+ w.Header().Set("Date", "Thu, 24 Aug 2023 14:35:29 GMT")
// missing address => 500
address, _, err := net.SplitHostPort(r.RemoteAddr)
@@ -44,6 +47,20 @@ func HTTPBinHandler() http.Handler {
secureRedirect := r.URL.Path == "/broken-redirect-https"
switch {
+ // redirect with count
+ case strings.HasPrefix(r.URL.Path, "/redirect/"):
+ count, err := strconv.Atoi(strings.TrimPrefix(r.URL.Path, "/redirect/"))
+ if err != nil {
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ if count <= 0 {
+ w.WriteHeader(http.StatusOK)
+ return
+ }
+ w.Header().Set("Location", fmt.Sprintf("/redirect/%d", count-1))
+ w.WriteHeader(http.StatusFound)
+
// broken HTTP redirect for clients
case cleartextRedirect && client:
w.Header().Set("Location", "http://")
diff --git a/pkg/netemx/httpbin_test.go b/pkg/netemx/httpbin_test.go
index 64630fee1..b85cd2789 100644
--- a/pkg/netemx/httpbin_test.go
+++ b/pkg/netemx/httpbin_test.go
@@ -25,6 +25,88 @@ func TestHTTPBinHandler(t *testing.T) {
}
})
+ t.Run("/redirect/{n} with invalid number", func(t *testing.T) {
+ req := &http.Request{
+ URL: &url.URL{Scheme: "https://", Path: "/redirect/antani"},
+ Body: http.NoBody,
+ Close: false,
+ Host: "httpbin.com",
+ RemoteAddr: net.JoinHostPort("8.8.8.8", "54321"),
+ }
+ rr := httptest.NewRecorder()
+ handler := HTTPBinHandler()
+ handler.ServeHTTP(rr, req)
+ result := rr.Result()
+ if result.StatusCode != http.StatusBadRequest {
+ t.Fatal("unexpected status code", result.StatusCode)
+ }
+ })
+
+ t.Run("/redirect/0", func(t *testing.T) {
+ req := &http.Request{
+ URL: &url.URL{Scheme: "https://", Path: "/redirect/0"},
+ Body: http.NoBody,
+ Close: false,
+ Host: "httpbin.com",
+ RemoteAddr: net.JoinHostPort("8.8.8.8", "54321"),
+ }
+ rr := httptest.NewRecorder()
+ handler := HTTPBinHandler()
+ handler.ServeHTTP(rr, req)
+ result := rr.Result()
+ if result.StatusCode != http.StatusOK {
+ t.Fatal("unexpected status code", result.StatusCode)
+ }
+ })
+
+ t.Run("/redirect/1", func(t *testing.T) {
+ req := &http.Request{
+ URL: &url.URL{Scheme: "https://", Path: "/redirect/1"},
+ Body: http.NoBody,
+ Close: false,
+ Host: "httpbin.com",
+ RemoteAddr: net.JoinHostPort("8.8.8.8", "54321"),
+ }
+ rr := httptest.NewRecorder()
+ handler := HTTPBinHandler()
+ handler.ServeHTTP(rr, req)
+ result := rr.Result()
+ if result.StatusCode != http.StatusFound {
+ t.Fatal("unexpected status code", result.StatusCode)
+ }
+ location, err := result.Location()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if location.Path != "/redirect/0" {
+ t.Fatal("unexpected location.Path", location.Path)
+ }
+ })
+
+ t.Run("/redirect/2", func(t *testing.T) {
+ req := &http.Request{
+ URL: &url.URL{Scheme: "https://", Path: "/redirect/2"},
+ Body: http.NoBody,
+ Close: false,
+ Host: "httpbin.com",
+ RemoteAddr: net.JoinHostPort("8.8.8.8", "54321"),
+ }
+ rr := httptest.NewRecorder()
+ handler := HTTPBinHandler()
+ handler.ServeHTTP(rr, req)
+ result := rr.Result()
+ if result.StatusCode != http.StatusFound {
+ t.Fatal("unexpected status code", result.StatusCode)
+ }
+ location, err := result.Location()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if location.Path != "/redirect/1" {
+ t.Fatal("unexpected location.Path", location.Path)
+ }
+ })
+
t.Run("/broken-redirect-http with client address", func(t *testing.T) {
req := &http.Request{
URL: &url.URL{Scheme: "http://", Path: "/broken-redirect-http"},
diff --git a/pkg/netemx/https.go b/pkg/netemx/https.go
index e0fd7b558..d82d81fed 100644
--- a/pkg/netemx/https.go
+++ b/pkg/netemx/https.go
@@ -97,7 +97,7 @@ func (srv *httpSecureServer) mustListenPortLocked(handler http.Handler, ipAddr n
tlsConfig := srv.unet.MustNewServerTLSConfig(srv.serverNameMain, srv.serverNameExtras...)
// serve requests in a background goroutine
- srvr := &http.Server{
+ srvr := &http.Server{ // #nosec G112 - just a testing server
Handler: handler,
TLSConfig: tlsConfig,
}
diff --git a/pkg/netemx/largefile.go b/pkg/netemx/largefile.go
index b899622ef..4a069bf5d 100644
--- a/pkg/netemx/largefile.go
+++ b/pkg/netemx/largefile.go
@@ -24,6 +24,6 @@ func LargeFileHandler(reader func(b []byte) (n int, err error)) http.Handler {
w.WriteHeader(http.StatusInternalServerError)
return
}
- w.Write(data)
+ _, _ = w.Write(data)
})
}
diff --git a/pkg/netemx/ooapi.go b/pkg/netemx/ooapi.go
index 43002d921..a5214f6e7 100644
--- a/pkg/netemx/ooapi.go
+++ b/pkg/netemx/ooapi.go
@@ -35,7 +35,7 @@ func (p *OOAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
}
-func (p *OOAPIHandler) getApiV1TestHelpers(w http.ResponseWriter, r *http.Request) {
+func (p *OOAPIHandler) getApiV1TestHelpers(w http.ResponseWriter, _ *http.Request) {
resp := map[string][]model.OOAPIService{
"web-connectivity": {
{
@@ -57,5 +57,5 @@ func (p *OOAPIHandler) getApiV1TestHelpers(w http.ResponseWriter, r *http.Reques
},
}
w.Header().Add("Content-Type", "application/json")
- w.Write(runtimex.Try1(json.Marshal(resp)))
+ _, _ = w.Write(runtimex.Try1(json.Marshal(resp)))
}
diff --git a/pkg/netemx/oohelperd_test.go b/pkg/netemx/oohelperd_test.go
index 7661e8d2a..91b2bb92b 100644
--- a/pkg/netemx/oohelperd_test.go
+++ b/pkg/netemx/oohelperd_test.go
@@ -73,13 +73,7 @@ func TestOOHelperDHandler(t *testing.T) {
Failure: nil,
},
},
- QUICHandshake: map[string]model.THTLSHandshakeResult{
- "93.184.216.34:443": {
- ServerName: "www.example.com",
- Status: true,
- Failure: nil,
- },
- },
+ QUICHandshake: map[string]model.THTLSHandshakeResult{}, // since https://github.com/ooni/probe-cli/pull/1549
HTTPRequest: model.THHTTPRequestResult{
BodyLength: 1533,
DiscoveredH3Endpoint: "www.example.com:443",
@@ -93,19 +87,7 @@ func TestOOHelperDHandler(t *testing.T) {
},
StatusCode: 200,
},
- HTTP3Request: &model.THHTTPRequestResult{
- BodyLength: 1533,
- DiscoveredH3Endpoint: "",
- Failure: nil,
- Title: "Default Web Page",
- Headers: map[string]string{
- "Alt-Svc": `h3=":443"`,
- "Content-Length": "1533",
- "Content-Type": "text/html; charset=utf-8",
- "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
- },
- StatusCode: 200,
- },
+ HTTP3Request: nil, // since https://github.com/ooni/probe-cli/pull/1549
DNS: model.THDNSResult{
Failure: nil,
Addrs: []string{"93.184.216.34"},
diff --git a/pkg/netemx/qaenv.go b/pkg/netemx/qaenv.go
index 4bdc37fa2..afa53e270 100644
--- a/pkg/netemx/qaenv.go
+++ b/pkg/netemx/qaenv.go
@@ -238,8 +238,8 @@ func (env *QAEnv) mustNewNetStacks(config *qaEnvConfig) (closables []io.Closer)
// AddRecordToAllResolvers adds the given DNS record to all DNS resolvers. You can safely
// add new DNS records from concurrent goroutines at any time.
func (env *QAEnv) AddRecordToAllResolvers(domain string, cname string, addrs ...string) {
- env.ISPResolverConfig().AddRecord(domain, cname, addrs...)
- env.OtherResolversConfig().AddRecord(domain, cname, addrs...)
+ runtimex.Try0(env.ISPResolverConfig().AddRecord(domain, cname, addrs...))
+ runtimex.Try0(env.OtherResolversConfig().AddRecord(domain, cname, addrs...))
}
// ISPResolverConfig returns the [*netem.DNSConfig] of the ISP resolver. Note that can safely
@@ -288,11 +288,11 @@ func (env *QAEnv) Close() error {
env.once.Do(func() {
// first close all the possible closables we track
for _, c := range env.closables {
- c.Close()
+ _ = c.Close()
}
// finally close the whole network topology
- env.topology.Close()
+ _ = env.topology.Close()
})
return nil
}
diff --git a/pkg/netemx/web.go b/pkg/netemx/web.go
index 39365db2b..73224fcfe 100644
--- a/pkg/netemx/web.go
+++ b/pkg/netemx/web.go
@@ -74,7 +74,7 @@ func ExampleWebPageHandler() http.Handler {
switch host {
case "www.example.com", "www.example.org":
- w.Write([]byte(ExampleWebPage))
+ _, _ = w.Write([]byte(ExampleWebPage))
case "example.com":
w.Header().Add("Location", "https://www.example.com/")
@@ -118,7 +118,7 @@ func BlockpageHandlerFactory() HTTPHandlerFactory {
return HTTPHandlerFactoryFunc(func(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Date", "Thu, 24 Aug 2023 14:35:29 GMT")
- w.Write([]byte(Blockpage))
+ _, _ = w.Write([]byte(Blockpage))
})
})
}
diff --git a/pkg/netemx/yandex.go b/pkg/netemx/yandex.go
index 72e74fa81..e3bdac8f6 100644
--- a/pkg/netemx/yandex.go
+++ b/pkg/netemx/yandex.go
@@ -32,7 +32,7 @@ func YandexHandler() http.Handler {
switch host {
case "ya.ru":
- w.Write([]byte(ExampleWebPage))
+ _, _ = w.Write([]byte(ExampleWebPage))
case "yandex.com":
w.Header().Add("Location", "https://ya.ru/")
diff --git a/pkg/netxlite/certifi.go b/pkg/netxlite/certifi.go
index 5685cb627..41f51b51b 100644
--- a/pkg/netxlite/certifi.go
+++ b/pkg/netxlite/certifi.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// 2024-03-22 14:00:05.981523 +0100 CET m=+1.360890834
+// 2024-05-13 18:43:22.647694 +0200 CEST m=+0.938119584
// https://curl.haxx.se/ca/cacert.pem
package netxlite
diff --git a/pkg/netxlite/dnsovertcp.go b/pkg/netxlite/dnsovertcp.go
index fb42d260c..025483938 100644
--- a/pkg/netxlite/dnsovertcp.go
+++ b/pkg/netxlite/dnsovertcp.go
@@ -88,7 +88,7 @@ func (t *DNSOverTCPTransport) RoundTrip(
}
defer conn.Close()
const iotimeout = 10 * time.Second
- conn.SetDeadline(time.Now().Add(iotimeout))
+ _ = conn.SetDeadline(time.Now().Add(iotimeout))
// Write request
buf := []byte{byte(len(rawQuery) >> 8)}
buf = append(buf, byte(len(rawQuery)))
diff --git a/pkg/netxlite/dnsoverudp.go b/pkg/netxlite/dnsoverudp.go
index 9a4e8ef76..a9bcf2ed0 100644
--- a/pkg/netxlite/dnsoverudp.go
+++ b/pkg/netxlite/dnsoverudp.go
@@ -95,16 +95,16 @@ func (t *DNSOverUDPTransport) RoundTrip(
if err != nil {
return nil, err
}
- conn.SetDeadline(deadline) // time to dial (usually ~zero) already factored in
+ _ = conn.SetDeadline(deadline) // time to dial (usually ~zero) already factored in
joinedch := make(chan bool)
myaddr := conn.LocalAddr().String()
if _, err := conn.Write(rawQuery); err != nil {
- conn.Close() // we still own the conn
+ _ = conn.Close() // we still own the conn
return nil, err
}
resp, err := t.recv(query, conn)
if err != nil {
- conn.Close() // we still own the conn
+ _ = conn.Close() // we still own the conn
return nil, err
}
// start a goroutine to listen for any delayed DNS response and
diff --git a/pkg/netxlite/errno.go b/pkg/netxlite/errno.go
index 8b3a1504c..0dadf82b5 100644
--- a/pkg/netxlite/errno.go
+++ b/pkg/netxlite/errno.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2024-03-22 14:00:06.725152 +0100 CET m=+0.418657251
+// Generated: 2024-05-13 18:43:23.514043 +0200 CEST m=+0.480330667
package netxlite
diff --git a/pkg/netxlite/errno_darwin.go b/pkg/netxlite/errno_darwin.go
index c28b3ebfd..4df9980ce 100644
--- a/pkg/netxlite/errno_darwin.go
+++ b/pkg/netxlite/errno_darwin.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2024-03-22 14:00:06.307199 +0100 CET m=+0.000694251
+// Generated: 2024-05-13 18:43:23.034437 +0200 CEST m=+0.000713334
package netxlite
diff --git a/pkg/netxlite/errno_darwin_test.go b/pkg/netxlite/errno_darwin_test.go
index b62370cf7..cc990de9e 100644
--- a/pkg/netxlite/errno_darwin_test.go
+++ b/pkg/netxlite/errno_darwin_test.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2024-03-22 14:00:06.515736 +0100 CET m=+0.209235918
+// Generated: 2024-05-13 18:43:23.30085 +0200 CEST m=+0.267132417
package netxlite
diff --git a/pkg/netxlite/errno_freebsd.go b/pkg/netxlite/errno_freebsd.go
index 0b2c1ddf7..0fc15fa20 100644
--- a/pkg/netxlite/errno_freebsd.go
+++ b/pkg/netxlite/errno_freebsd.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2024-03-22 14:00:06.545439 +0100 CET m=+0.238939543
+// Generated: 2024-05-13 18:43:23.327874 +0200 CEST m=+0.294157167
package netxlite
diff --git a/pkg/netxlite/errno_freebsd_test.go b/pkg/netxlite/errno_freebsd_test.go
index a81983005..6972334e4 100644
--- a/pkg/netxlite/errno_freebsd_test.go
+++ b/pkg/netxlite/errno_freebsd_test.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2024-03-22 14:00:06.570521 +0100 CET m=+0.264022418
+// Generated: 2024-05-13 18:43:23.354839 +0200 CEST m=+0.321123042
package netxlite
diff --git a/pkg/netxlite/errno_linux.go b/pkg/netxlite/errno_linux.go
index 8190eae29..7a4a2cf62 100644
--- a/pkg/netxlite/errno_linux.go
+++ b/pkg/netxlite/errno_linux.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2024-03-22 14:00:06.637061 +0100 CET m=+0.330564126
+// Generated: 2024-05-13 18:43:23.42483 +0200 CEST m=+0.391115126
package netxlite
diff --git a/pkg/netxlite/errno_linux_test.go b/pkg/netxlite/errno_linux_test.go
index 9c20533fc..b7409b697 100644
--- a/pkg/netxlite/errno_linux_test.go
+++ b/pkg/netxlite/errno_linux_test.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2024-03-22 14:00:06.660915 +0100 CET m=+0.354418043
+// Generated: 2024-05-13 18:43:23.448805 +0200 CEST m=+0.415090876
package netxlite
diff --git a/pkg/netxlite/errno_openbsd.go b/pkg/netxlite/errno_openbsd.go
index 84721ae33..85f89d009 100644
--- a/pkg/netxlite/errno_openbsd.go
+++ b/pkg/netxlite/errno_openbsd.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2024-03-22 14:00:06.591822 +0100 CET m=+0.285323709
+// Generated: 2024-05-13 18:43:23.377383 +0200 CEST m=+0.343667459
package netxlite
diff --git a/pkg/netxlite/errno_openbsd_test.go b/pkg/netxlite/errno_openbsd_test.go
index 66ec31ecc..a9283453b 100644
--- a/pkg/netxlite/errno_openbsd_test.go
+++ b/pkg/netxlite/errno_openbsd_test.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2024-03-22 14:00:06.615325 +0100 CET m=+0.308827543
+// Generated: 2024-05-13 18:43:23.402976 +0200 CEST m=+0.369261209
package netxlite
diff --git a/pkg/netxlite/errno_windows.go b/pkg/netxlite/errno_windows.go
index c36af2dc1..33d27476c 100644
--- a/pkg/netxlite/errno_windows.go
+++ b/pkg/netxlite/errno_windows.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2024-03-22 14:00:06.682241 +0100 CET m=+0.375744876
+// Generated: 2024-05-13 18:43:23.47073 +0200 CEST m=+0.437016626
package netxlite
diff --git a/pkg/netxlite/errno_windows_test.go b/pkg/netxlite/errno_windows_test.go
index 9e9485d4a..5a95d8aa0 100644
--- a/pkg/netxlite/errno_windows_test.go
+++ b/pkg/netxlite/errno_windows_test.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2024-03-22 14:00:06.703734 +0100 CET m=+0.397238084
+// Generated: 2024-05-13 18:43:23.492462 +0200 CEST m=+0.458748876
package netxlite
diff --git a/pkg/netxlite/http3.go b/pkg/netxlite/http3.go
index 23ef21ae5..3b75ee830 100644
--- a/pkg/netxlite/http3.go
+++ b/pkg/netxlite/http3.go
@@ -39,7 +39,7 @@ func (txp *http3Transport) RoundTrip(req *http.Request) (*http.Response, error)
// CloseIdleConnections implements HTTPTransport.CloseIdleConnections.
func (txp *http3Transport) CloseIdleConnections() {
- txp.child.Close()
+ _ = txp.child.Close()
txp.dialer.CloseIdleConnections()
}
diff --git a/pkg/netxlite/httptimeout.go b/pkg/netxlite/httptimeout.go
index b2c4b80e9..c978df199 100644
--- a/pkg/netxlite/httptimeout.go
+++ b/pkg/netxlite/httptimeout.go
@@ -62,7 +62,7 @@ func (d *httpTLSDialerWithReadTimeout) DialTLSContext(
}
tconn, okay := conn.(TLSConn) // part of the contract but let's be graceful
if !okay {
- conn.Close() // we own the conn here
+ _ = conn.Close() // we own the conn here
return nil, ErrNotTLSConn
}
return &httpTLSConnWithReadTimeout{tconn}, nil
@@ -95,7 +95,7 @@ const httpConnReadTimeout = 300 * time.Second
// Read implements Conn.Read.
func (c *httpConnWithReadTimeout) Read(b []byte) (int, error) {
- c.Conn.SetReadDeadline(time.Now().Add(httpConnReadTimeout))
+ _ = c.Conn.SetReadDeadline(time.Now().Add(httpConnReadTimeout))
defer c.Conn.SetReadDeadline(time.Time{})
return c.Conn.Read(b)
}
@@ -108,7 +108,7 @@ type httpTLSConnWithReadTimeout struct {
// Read implements Conn.Read.
func (c *httpTLSConnWithReadTimeout) Read(b []byte) (int, error) {
- c.TLSConn.SetReadDeadline(time.Now().Add(httpConnReadTimeout))
+ _ = c.TLSConn.SetReadDeadline(time.Now().Add(httpConnReadTimeout))
defer c.TLSConn.SetReadDeadline(time.Time{})
return c.TLSConn.Read(b)
}
diff --git a/pkg/netxlite/internal/gencertifi/main.go b/pkg/netxlite/internal/gencertifi/main.go
index 4d5fd1e2e..60d92a7f2 100644
--- a/pkg/netxlite/internal/gencertifi/main.go
+++ b/pkg/netxlite/internal/gencertifi/main.go
@@ -41,7 +41,7 @@ func main() {
}
url := os.Args[1]
- resp, err := http.Get(url)
+ resp, err := http.Get(url) // #nosec G107 -- this is working as intended
if err != nil {
log.Fatal(err)
}
diff --git a/pkg/netxlite/internal/generrno/main.go b/pkg/netxlite/internal/generrno/main.go
index f54e7136c..0f8b355a3 100644
--- a/pkg/netxlite/internal/generrno/main.go
+++ b/pkg/netxlite/internal/generrno/main.go
@@ -191,7 +191,7 @@ func mapSystemToLibrary(system string) string {
}
func fileCreate(filename string) *os.File {
- filep, err := os.Create(filename)
+ filep, err := os.Create(filename) // #nosec G304 - this is working as intended
if err != nil {
log.Fatal(err)
}
diff --git a/pkg/netxlite/netx_test.go b/pkg/netxlite/netx_test.go
index af5660857..f2139b8ef 100644
--- a/pkg/netxlite/netx_test.go
+++ b/pkg/netxlite/netx_test.go
@@ -68,7 +68,7 @@ func TestNetxWithNetem(t *testing.T) {
webServerUDPListener := runtimex.Try1(webServerStack.ListenUDP("udp", webServerUDPAddress))
webServerUDPServer := &http3.Server{
TLSConfig: webServerTLSConfig,
- QuicConfig: &quic.Config{},
+ QUICConfig: &quic.Config{},
Handler: webServerHandler,
}
go webServerUDPServer.Serve(webServerUDPListener)
diff --git a/pkg/netxlite/quic.go b/pkg/netxlite/quic.go
index 0512eb7ff..46cc8f00d 100644
--- a/pkg/netxlite/quic.go
+++ b/pkg/netxlite/quic.go
@@ -139,7 +139,7 @@ func (d *quicDialerQUICGo) DialContext(ctx context.Context,
err = MaybeNewErrWrapper(ClassifyQUICHandshakeError, QUICHandshakeOperation, err)
trace.OnQUICHandshakeDone(started, address, qconn, tlsConfig, err, finished)
if err != nil {
- pconn.Close() // we own it on failure
+ _ = pconn.Close() // we own it on failure
return nil, err
}
return newQUICConnectionOwnsConn(qconn, pconn), nil
@@ -200,7 +200,7 @@ func (d *quicDialerHandshakeCompleter) DialContext(
case <-conn.HandshakeComplete():
return conn, nil
case <-ctx.Done():
- conn.CloseWithError(0, "") // we own the conn
+ _ = conn.CloseWithError(0, "") // we own the conn
return nil, ctx.Err()
}
}
@@ -227,7 +227,7 @@ func newQUICConnectionOwnsConn(qconn quic.EarlyConnection, pconn model.UDPLikeCo
func (qconn *quicConnectionOwnsConn) CloseWithError(
code quic.ApplicationErrorCode, reason string) error {
err := qconn.EarlyConnection.CloseWithError(code, reason)
- qconn.conn.Close()
+ _ = qconn.conn.Close()
return err
}
diff --git a/pkg/netxlite/tls.go b/pkg/netxlite/tls.go
index 6c60065a1..ee8b43e71 100644
--- a/pkg/netxlite/tls.go
+++ b/pkg/netxlite/tls.go
@@ -211,7 +211,7 @@ func (h *tlsHandshakerConfigurable) Handshake(
timeout = 10 * time.Second
}
defer conn.SetDeadline(time.Time{})
- conn.SetDeadline(time.Now().Add(timeout))
+ _ = conn.SetDeadline(time.Now().Add(timeout))
if config.RootCAs == nil {
config = config.Clone()
// See https://github.com/ooni/probe/issues/2413 for context
@@ -318,7 +318,7 @@ func (d *tlsDialer) DialTLSContext(ctx context.Context, network, address string)
config := d.config(host, port)
tlsconn, err := d.TLSHandshaker.Handshake(ctx, conn, config)
if err != nil {
- conn.Close()
+ _ = conn.Close()
return nil, err
}
return tlsconn, nil
diff --git a/pkg/ooapi/checkin.go b/pkg/ooapi/checkin.go
deleted file mode 100644
index e47c04d69..000000000
--- a/pkg/ooapi/checkin.go
+++ /dev/null
@@ -1,39 +0,0 @@
-package ooapi
-
-//
-// CheckIn API
-//
-
-import (
- "encoding/json"
- "net/http"
-
- "github.com/ooni/probe-engine/pkg/httpapi"
- "github.com/ooni/probe-engine/pkg/model"
- "github.com/ooni/probe-engine/pkg/runtimex"
-)
-
-// NewDescriptorCheckIn creates a new [httpapi.Descriptor] describing how
-// to issue an HTTP call to the CheckIn API.
-func NewDescriptorCheckIn(
- config *model.OOAPICheckInConfig,
-) *httpapi.Descriptor[*model.OOAPICheckInConfig, *model.OOAPICheckInResult] {
- rawRequest, err := json.Marshal(config)
- runtimex.PanicOnError(err, "json.Marshal failed unexpectedly")
- return &httpapi.Descriptor[*model.OOAPICheckInConfig, *model.OOAPICheckInResult]{
- Accept: httpapi.ApplicationJSON,
- AcceptEncodingGzip: true, // we want a small response
- Authorization: "",
- ContentType: httpapi.ApplicationJSON,
- LogBody: true,
- MaxBodySize: 0,
- Method: http.MethodPost,
- Request: &httpapi.RequestDescriptor[*model.OOAPICheckInConfig]{
- Body: rawRequest,
- },
- Response: &httpapi.JSONResponseDescriptor[model.OOAPICheckInResult]{},
- Timeout: 0,
- URLPath: "/api/v1/check-in",
- URLQuery: nil,
- }
-}
diff --git a/pkg/ooapi/checkin_test.go b/pkg/ooapi/checkin_test.go
deleted file mode 100644
index 51cab71b7..000000000
--- a/pkg/ooapi/checkin_test.go
+++ /dev/null
@@ -1,79 +0,0 @@
-package ooapi
-
-import (
- "net/http"
- "reflect"
- "testing"
-
- "github.com/ooni/probe-engine/pkg/httpapi"
- "github.com/ooni/probe-engine/pkg/model"
-)
-
-func TestNewDescriptorCheckIn(t *testing.T) {
- // Implementation note: this test uses reflection such that new
- // fields added to a Descriptor will cause an error if they aren't
- // initialized as expected (which may be zero-initialized).
-
- desc := NewDescriptorCheckIn(&model.OOAPICheckInConfig{})
-
- rdesc := reflect.ValueOf(desc).Elem()
- typ := rdesc.Type()
- for idx := 0; idx < rdesc.NumField(); idx++ {
- field := rdesc.Field(idx)
- name := typ.Field(idx).Name
-
- switch name {
- case "AcceptEncodingGzip":
- if !field.Interface().(bool) {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "Accept":
- if field.Interface().(string) != httpapi.ApplicationJSON {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "Authorization":
- if !field.IsZero() {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "ContentType":
- if field.Interface().(string) != httpapi.ApplicationJSON {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "LogBody":
- if field.IsZero() {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "MaxBodySize":
- if !field.IsZero() {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "Method":
- if field.Interface().(string) != http.MethodPost {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "Request":
- req := field.Interface().(*httpapi.RequestDescriptor[*model.OOAPICheckInConfig])
- if len(req.Body) <= 2 {
- t.Fatalf("unexpected desc.%s length", name)
- }
- case "Response":
- if field.IsZero() {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "Timeout":
- if !field.IsZero() {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "URLPath":
- if field.Interface().(string) != "/api/v1/check-in" {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "URLQuery":
- if !field.IsZero() {
- t.Fatalf("unexpected desc.%s", name)
- }
- default:
- t.Fatalf("unhandled field %s", name)
- }
- }
-}
diff --git a/pkg/ooapi/doc.go b/pkg/ooapi/doc.go
deleted file mode 100644
index 24c8c5623..000000000
--- a/pkg/ooapi/doc.go
+++ /dev/null
@@ -1,2 +0,0 @@
-// Package ooapi describes how the OONI API works.
-package ooapi
diff --git a/pkg/ooapi/th.go b/pkg/ooapi/th.go
deleted file mode 100644
index 09ca4cfbf..000000000
--- a/pkg/ooapi/th.go
+++ /dev/null
@@ -1,39 +0,0 @@
-package ooapi
-
-//
-// Web Connectivity Test Helper (TH).
-//
-
-import (
- "encoding/json"
- "net/http"
-
- "github.com/ooni/probe-engine/pkg/httpapi"
- "github.com/ooni/probe-engine/pkg/model"
- "github.com/ooni/probe-engine/pkg/runtimex"
-)
-
-// NewDescriptorTH creates a new [httpapi.Descriptor] describing how
-// to issue an HTTP call to the Web Connectivity Test Helper (TH).
-func NewDescriptorTH(
- creq *model.THRequest,
-) *httpapi.Descriptor[*model.THRequest, *model.THResponse] {
- rawRequest, err := json.Marshal(creq)
- runtimex.PanicOnError(err, "json.Marshal failed unexpectedly")
- return &httpapi.Descriptor[*model.THRequest, *model.THResponse]{
- Accept: httpapi.ApplicationJSON,
- AcceptEncodingGzip: false,
- Authorization: "",
- ContentType: httpapi.ApplicationJSON,
- LogBody: true,
- MaxBodySize: 0,
- Method: http.MethodPost,
- Request: &httpapi.RequestDescriptor[*model.THRequest]{
- Body: rawRequest,
- },
- Response: &httpapi.JSONResponseDescriptor[model.THResponse]{},
- Timeout: 0,
- URLPath: "/",
- URLQuery: nil,
- }
-}
diff --git a/pkg/ooapi/th_test.go b/pkg/ooapi/th_test.go
deleted file mode 100644
index 4e9184276..000000000
--- a/pkg/ooapi/th_test.go
+++ /dev/null
@@ -1,79 +0,0 @@
-package ooapi
-
-import (
- "net/http"
- "reflect"
- "testing"
-
- "github.com/ooni/probe-engine/pkg/httpapi"
- "github.com/ooni/probe-engine/pkg/model"
-)
-
-func TestNewDescriptorTH(t *testing.T) {
- // Implementation note: this test uses reflection such that new
- // fields added to a Descriptor will cause an error if they aren't
- // initialized as expected (which may be zero-initialized).
-
- desc := NewDescriptorTH(&model.THRequest{})
-
- rdesc := reflect.ValueOf(desc).Elem()
- typ := rdesc.Type()
- for idx := 0; idx < rdesc.NumField(); idx++ {
- field := rdesc.Field(idx)
- name := typ.Field(idx).Name
-
- switch name {
- case "AcceptEncodingGzip":
- if !field.IsZero() {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "Accept":
- if field.Interface().(string) != httpapi.ApplicationJSON {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "Authorization":
- if !field.IsZero() {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "ContentType":
- if field.Interface().(string) != httpapi.ApplicationJSON {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "LogBody":
- if !field.Interface().(bool) {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "MaxBodySize":
- if !field.IsZero() {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "Method":
- if field.Interface().(string) != http.MethodPost {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "Request":
- req := field.Interface().(*httpapi.RequestDescriptor[*model.THRequest])
- if len(req.Body) <= 2 {
- t.Fatalf("unexpected desc.%s length", name)
- }
- case "Response":
- if field.IsZero() {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "Timeout":
- if !field.IsZero() {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "URLPath":
- if field.Interface().(string) != "/" {
- t.Fatalf("unexpected desc.%s", name)
- }
- case "URLQuery":
- if !field.IsZero() {
- t.Fatalf("unexpected desc.%s", name)
- }
- default:
- t.Fatalf("unhandled non-zero field %s", name)
- }
- }
-}
diff --git a/pkg/oohelperd/handler.go b/pkg/oohelperd/handler.go
index c1ed0461c..9c6342f71 100644
--- a/pkg/oohelperd/handler.go
+++ b/pkg/oohelperd/handler.go
@@ -11,6 +11,7 @@ import (
"io"
"net/http"
"net/http/cookiejar"
+ "os"
"strings"
"sync/atomic"
"time"
@@ -31,6 +32,9 @@ const maxAcceptableBodySize = 1 << 24
//
// The zero value is invalid; construct using [NewHandler].
type Handler struct {
+ // EnableQUIC OPTIONALLY enables QUIC.
+ EnableQUIC bool
+
// baseLogger is the MANDATORY logger to use.
baseLogger model.Logger
@@ -69,9 +73,13 @@ type Handler struct {
var _ http.Handler = &Handler{}
+// enableQUIC allows to control whether to enable QUIC by using environment variables.
+var enableQUIC = (os.Getenv("OOHELPERD_ENABLE_QUIC") == "1")
+
// NewHandler constructs the [handler].
func NewHandler(logger model.Logger, netx *netxlite.Netx) *Handler {
return &Handler{
+ EnableQUIC: enableQUIC,
baseLogger: logger,
countRequests: &atomic.Int64{},
indexer: &atomic.Int64{},
@@ -148,7 +156,20 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
version.Version,
))
- // we only handle the POST method
+ // handle GET method for health check
+ if req.Method == "GET" {
+ metricRequestsCount.WithLabelValues("200", "ok").Inc()
+ resp := map[string]string{
+ "message": "Hello OONItarian!",
+ }
+ data, err := json.Marshal(resp)
+ runtimex.PanicOnError(err, "json.Marshal failed")
+ w.Header().Add("Content-Type", "application/json")
+ _, _ = w.Write(data)
+ return
+ }
+
+ // we only handle the POST method for response generation
if req.Method != "POST" {
metricRequestsCount.WithLabelValues("400", "bad_request_method").Inc()
w.WriteHeader(400)
@@ -202,7 +223,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
data, err = json.Marshal(cresp)
runtimex.PanicOnError(err, "json.Marshal failed")
w.Header().Add("Content-Type", "application/json")
- w.Write(data)
+ _, _ = w.Write(data)
}
// newResolver creates a new [model.Resolver] suitable for serving
diff --git a/pkg/oohelperd/handler_test.go b/pkg/oohelperd/handler_test.go
index ba0c4e382..f25898874 100644
--- a/pkg/oohelperd/handler_test.go
+++ b/pkg/oohelperd/handler_test.go
@@ -100,12 +100,20 @@ func TestHandlerWorkingAsIntended(t *testing.T) {
expectations := []expectationSpec{{
name: "check for invalid method",
- reqMethod: "GET",
+ reqMethod: "PUT",
reqContentType: "",
reqBody: strings.NewReader(""),
respStatusCode: 400,
respContentType: "",
parseBody: false,
+ }, {
+ name: "check for health message",
+ reqMethod: "GET",
+ reqContentType: "",
+ reqBody: strings.NewReader(""),
+ respStatusCode: 200,
+ respContentType: "application/json",
+ parseBody: true,
}, {
name: "check for error reading request body",
reqMethod: "POST",
@@ -254,3 +262,10 @@ func TestHandlerWorkingAsIntended(t *testing.T) {
})
}
}
+
+func TestNewHandlerEnableQUIC(t *testing.T) {
+ handler := NewHandler(log.Log, &netxlite.Netx{Underlying: nil})
+ if handler.EnableQUIC != false {
+ t.Fatal("expected to see false here (is the the environment variable OOHELPERD_ENABLE_QUIC set?!)")
+ }
+}
diff --git a/pkg/oohelperd/measure.go b/pkg/oohelperd/measure.go
index 24bfe870e..01fbfdccc 100644
--- a/pkg/oohelperd/measure.go
+++ b/pkg/oohelperd/measure.go
@@ -125,7 +125,7 @@ func measure(ctx context.Context, config *Handler, creq *ctrlRequest) (*ctrlResp
// In the v3.17.x and possibly v3.18.x release cycles, QUIC is disabled by
// default but clients that know QUIC can enable it. We will eventually remove
// this flag and enable QUIC measurements for all clients.
- if creq.XQUICEnabled && cresp.HTTPRequest.DiscoveredH3Endpoint != "" {
+ if config.EnableQUIC && creq.XQUICEnabled && cresp.HTTPRequest.DiscoveredH3Endpoint != "" {
// quicconnect: start over all the endpoints
for _, endpoint := range endpoints {
wg.Add(1)
diff --git a/pkg/oohelperd/qa_test.go b/pkg/oohelperd/qa_test.go
new file mode 100644
index 000000000..da7bea37c
--- /dev/null
+++ b/pkg/oohelperd/qa_test.go
@@ -0,0 +1,137 @@
+package oohelperd_test
+
+import (
+ "bytes"
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/apex/log"
+ "github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/must"
+ "github.com/ooni/probe-engine/pkg/netemx"
+ "github.com/ooni/probe-engine/pkg/netxlite"
+ "github.com/ooni/probe-engine/pkg/oohelperd"
+ "github.com/ooni/probe-engine/pkg/optional"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+)
+
+// TestQAEnableDisableQUIC ensures that we can enable and disable QUIC.
+func TestQAEnableDisableQUIC(t *testing.T) {
+ // testcase is a test case for this function
+ type testcase struct {
+ name string
+ enableQUIC optional.Value[bool]
+ }
+
+ cases := []testcase{{
+ name: "with the default settings",
+ enableQUIC: optional.None[bool](),
+ }, {
+ name: "with explicit false",
+ enableQUIC: optional.Some(false),
+ }, {
+ name: "with explicit true",
+ enableQUIC: optional.Some(true),
+ }}
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ // create a new testing scenario
+ env := netemx.MustNewScenario(netemx.InternetScenario)
+ defer env.Close()
+
+ // create a new handler
+ handler := oohelperd.NewHandler(
+ log.Log,
+ &netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}},
+ )
+
+ // optionally and conditionally enable QUIC
+ if !tc.enableQUIC.IsNone() {
+ handler.EnableQUIC = tc.enableQUIC.Unwrap()
+ }
+
+ // create request body
+ reqbody := &model.THRequest{
+ HTTPRequest: "https://www.example.com/",
+ HTTPRequestHeaders: map[string][]string{
+ "Accept-Language": {model.HTTPHeaderAcceptLanguage},
+ "Accept": {model.HTTPHeaderAccept},
+ "User-Agent": {model.HTTPHeaderUserAgent},
+ },
+ TCPConnect: []string{netemx.AddressWwwExampleCom},
+ XQUICEnabled: true,
+ }
+
+ // create request
+ req := runtimex.Try1(http.NewRequest(
+ "POST",
+ "http://127.0.0.1:8080/",
+ bytes.NewReader(must.MarshalJSON(reqbody)),
+ ))
+
+ // create response recorder
+ resprec := httptest.NewRecorder()
+
+ // invoke the handler
+ handler.ServeHTTP(resprec, req)
+
+ // get the response
+ resp := resprec.Result()
+ defer resp.Body.Close()
+
+ // make sure the status code indicates success
+ if resp.StatusCode != 200 {
+ t.Fatal("expected 200 Ok")
+ }
+
+ // make sure the content-type is OK
+ if v := resp.Header.Get("Content-Type"); v != "application/json" {
+ t.Fatal("unexpected content-type", v)
+ }
+
+ // read the response body
+ respbody := runtimex.Try1(netxlite.ReadAllContext(context.Background(), resp.Body))
+
+ // parse the response body
+ var jsonresp model.THResponse
+ must.UnmarshalJSON(respbody, &jsonresp)
+
+ // check whether we have an HTTP3 response
+ switch {
+ case !tc.enableQUIC.IsNone() && tc.enableQUIC.Unwrap() && jsonresp.HTTP3Request != nil:
+ // all good: we have QUIC enabled and we get an HTTP/3 response
+
+ case (tc.enableQUIC.IsNone() || tc.enableQUIC.Unwrap() == false) && jsonresp.HTTP3Request == nil:
+ // all good: either default behavior or QUIC not enabled and not HTTP/3 response
+
+ default:
+ t.Fatalf(
+ "tc.enableQUIC.IsNone() = %v, tc.enableQUIC.UnwrapOr(false) = %v, jsonresp.HTTP3Request = %v",
+ tc.enableQUIC.IsNone(),
+ tc.enableQUIC.UnwrapOr(false),
+ jsonresp.HTTP3Request,
+ )
+ }
+
+ // check whether we have QUIC handshakes
+ switch {
+ case !tc.enableQUIC.IsNone() && tc.enableQUIC.Unwrap() && len(jsonresp.QUICHandshake) > 0:
+ // all good: we have QUIC enabled and we get QUIC handshakes
+
+ case (tc.enableQUIC.IsNone() || tc.enableQUIC.Unwrap() == false) && len(jsonresp.QUICHandshake) <= 0:
+ // all good: either default behavior or QUIC not enabled and no QUIC handshakes
+
+ default:
+ t.Fatalf(
+ "tc.enableQUIC.IsNone() = %v, tc.enableQUIC.UnwrapOr(false) = %v, jsonresp.QUICHandshake = %v",
+ tc.enableQUIC.IsNone(),
+ tc.enableQUIC.UnwrapOr(false),
+ jsonresp.QUICHandshake,
+ )
+ }
+ })
+ }
+}
diff --git a/pkg/oohelperd/quic.go b/pkg/oohelperd/quic.go
index 3896f0516..0f62f826b 100644
--- a/pkg/oohelperd/quic.go
+++ b/pkg/oohelperd/quic.go
@@ -81,7 +81,7 @@ func quicDo(ctx context.Context, config *quicConfig) {
// See https://github.com/ooni/probe/issues/2413 to understand
// why we're using nil to force netxlite to use the cached
// default Mozilla cert pool.
- tlsConfig := &tls.Config{
+ tlsConfig := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
NextProtos: []string{"h3"},
RootCAs: nil,
ServerName: config.URLHostname,
diff --git a/pkg/oohelperd/tcptls.go b/pkg/oohelperd/tcptls.go
index 3f93b17f3..0bfa82029 100644
--- a/pkg/oohelperd/tcptls.go
+++ b/pkg/oohelperd/tcptls.go
@@ -127,7 +127,7 @@ func tcpTLSDo(ctx context.Context, config *tcpTLSConfig) {
// See https://github.com/ooni/probe/issues/2413 to understand
// why we're using nil to force netxlite to use the cached
// default Mozilla cert pool.
- tlsConfig := &tls.Config{
+ tlsConfig := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
NextProtos: []string{"h2", "http/1.1"},
RootCAs: nil,
ServerName: config.URLHostname,
@@ -140,7 +140,7 @@ func tcpTLSDo(ctx context.Context, config *tcpTLSConfig) {
// perform the handshake
tlsConn, err := thx.Handshake(ctx, conn, tlsConfig)
- measurexlite.MaybeClose(tlsConn)
+ _ = measurexlite.MaybeClose(tlsConn)
// publish time required to handshake
tlsElapsed := time.Since(tlsT0)
diff --git a/pkg/oonirun/experiment.go b/pkg/oonirun/experiment.go
index 628171abf..2232a8d05 100644
--- a/pkg/oonirun/experiment.go
+++ b/pkg/oonirun/experiment.go
@@ -64,14 +64,14 @@ type Experiment struct {
newInputLoaderFn func(inputPolicy model.InputPolicy) inputLoader
// newSubmitterFn is OPTIONAL and used for testing.
- newSubmitterFn func(ctx context.Context) (engine.Submitter, error)
+ newSubmitterFn func(ctx context.Context) (model.Submitter, error)
// newSaverFn is OPTIONAL and used for testing.
- newSaverFn func(experiment model.Experiment) (engine.Saver, error)
+ newSaverFn func(experiment model.Experiment) (model.Saver, error)
// newInputProcessorFn is OPTIONAL and used for testing.
newInputProcessorFn func(experiment model.Experiment, inputList []model.OOAPIURLInfo,
- saver engine.Saver, submitter engine.Submitter) inputProcessor
+ saver model.Saver, submitter model.Submitter) inputProcessor
}
// Run runs the given experiment.
@@ -92,7 +92,7 @@ func (ed *Experiment) Run(ctx context.Context) error {
// 3. randomize input, if needed
if ed.Random {
- rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
+ rnd := rand.New(rand.NewSource(time.Now().UnixNano())) // #nosec G404 -- not really important
rnd.Shuffle(len(inputList), func(i, j int) {
inputList[i], inputList[j] = inputList[j], inputList[i]
})
@@ -138,47 +138,46 @@ type inputProcessor = model.ExperimentInputProcessor
// newInputProcessor creates a new inputProcessor instance.
func (ed *Experiment) newInputProcessor(experiment model.Experiment,
- inputList []model.OOAPIURLInfo, saver engine.Saver, submitter engine.Submitter) inputProcessor {
+ inputList []model.OOAPIURLInfo, saver model.Saver, submitter model.Submitter) inputProcessor {
if ed.newInputProcessorFn != nil {
return ed.newInputProcessorFn(experiment, inputList, saver, submitter)
}
- return &engine.InputProcessor{
+ return &InputProcessor{
Annotations: ed.Annotations,
Experiment: &experimentWrapper{
- child: engine.NewInputProcessorExperimentWrapper(experiment),
+ child: NewInputProcessorExperimentWrapper(experiment),
logger: ed.Session.Logger(),
total: len(inputList),
},
Inputs: inputList,
MaxRuntime: time.Duration(ed.MaxRuntime) * time.Second,
Options: experimentOptionsToStringList(ed.ExtraOptions),
- Saver: engine.NewInputProcessorSaverWrapper(saver),
+ Saver: NewInputProcessorSaverWrapper(saver),
Submitter: &experimentSubmitterWrapper{
- child: engine.NewInputProcessorSubmitterWrapper(submitter),
+ child: NewInputProcessorSubmitterWrapper(submitter),
logger: ed.Session.Logger(),
},
}
}
// newSaver creates a new engine.Saver instance.
-func (ed *Experiment) newSaver(experiment model.Experiment) (engine.Saver, error) {
+func (ed *Experiment) newSaver(experiment model.Experiment) (model.Saver, error) {
if ed.newSaverFn != nil {
return ed.newSaverFn(experiment)
}
- return engine.NewSaver(engine.SaverConfig{
- Enabled: !ed.NoJSON,
- Experiment: experiment,
- FilePath: ed.ReportFile,
- Logger: ed.Session.Logger(),
+ return NewSaver(SaverConfig{
+ Enabled: !ed.NoJSON,
+ FilePath: ed.ReportFile,
+ Logger: ed.Session.Logger(),
})
}
// newSubmitter creates a new engine.Submitter instance.
-func (ed *Experiment) newSubmitter(ctx context.Context) (engine.Submitter, error) {
+func (ed *Experiment) newSubmitter(ctx context.Context) (model.Submitter, error) {
if ed.newSubmitterFn != nil {
return ed.newSubmitterFn(ctx)
}
- return engine.NewSubmitter(ctx, engine.SubmitterConfig{
+ return NewSubmitter(ctx, SubmitterConfig{
Enabled: !ed.NoCollector,
Session: ed.Session,
Logger: ed.Session.Logger(),
@@ -234,7 +233,7 @@ func experimentOptionsToStringList(options map[string]any) (out []string) {
// experimentWrapper wraps an experiment and logs progress
type experimentWrapper struct {
// child is the child experiment wrapper
- child engine.InputProcessorExperimentWrapper
+ child InputProcessorExperimentWrapper
// logger is the logger to use
logger model.Logger
@@ -255,7 +254,7 @@ func (ew *experimentWrapper) MeasureAsync(
// fail if we cannot submit a measurement
type experimentSubmitterWrapper struct {
// child is the child submitter wrapper
- child engine.InputProcessorSubmitterWrapper
+ child InputProcessorSubmitterWrapper
// logger is the logger to use
logger model.Logger
diff --git a/pkg/oonirun/experiment_test.go b/pkg/oonirun/experiment_test.go
index f999962fa..7a068d661 100644
--- a/pkg/oonirun/experiment_test.go
+++ b/pkg/oonirun/experiment_test.go
@@ -8,7 +8,6 @@ import (
"testing"
"time"
- "github.com/ooni/probe-engine/pkg/engine"
"github.com/ooni/probe-engine/pkg/mocks"
"github.com/ooni/probe-engine/pkg/model"
"github.com/ooni/probe-engine/pkg/testingx"
@@ -81,7 +80,7 @@ func TestExperimentRunWithFailureToSubmitAndShuffle(t *testing.T) {
},
newExperimentBuilderFn: nil,
newInputLoaderFn: nil,
- newSubmitterFn: func(ctx context.Context) (engine.Submitter, error) {
+ newSubmitterFn: func(ctx context.Context) (model.Submitter, error) {
subm := &mocks.Submitter{
MockSubmit: func(ctx context.Context, m *model.Measurement) error {
failedToSubmit++
@@ -171,9 +170,9 @@ func TestExperimentRun(t *testing.T) {
Session Session
newExperimentBuilderFn func(experimentName string) (model.ExperimentBuilder, error)
newInputLoaderFn func(inputPolicy model.InputPolicy) inputLoader
- newSubmitterFn func(ctx context.Context) (engine.Submitter, error)
- newSaverFn func(experiment model.Experiment) (engine.Saver, error)
- newInputProcessorFn func(experiment model.Experiment, inputList []model.OOAPIURLInfo, saver engine.Saver, submitter engine.Submitter) inputProcessor
+ newSubmitterFn func(ctx context.Context) (model.Submitter, error)
+ newSaverFn func(experiment model.Experiment) (model.Saver, error)
+ newInputProcessorFn func(experiment model.Experiment, inputList []model.OOAPIURLInfo, saver model.Saver, submitter model.Submitter) inputProcessor
}
type args struct {
ctx context.Context
@@ -274,7 +273,7 @@ func TestExperimentRun(t *testing.T) {
},
}
},
- newSubmitterFn: func(ctx context.Context) (engine.Submitter, error) {
+ newSubmitterFn: func(ctx context.Context) (model.Submitter, error) {
return nil, errMocked
},
},
@@ -317,10 +316,10 @@ func TestExperimentRun(t *testing.T) {
},
}
},
- newSubmitterFn: func(ctx context.Context) (engine.Submitter, error) {
+ newSubmitterFn: func(ctx context.Context) (model.Submitter, error) {
return &mocks.Submitter{}, nil
},
- newSaverFn: func(experiment model.Experiment) (engine.Saver, error) {
+ newSaverFn: func(experiment model.Experiment) (model.Saver, error) {
return nil, errMocked
},
},
@@ -363,14 +362,14 @@ func TestExperimentRun(t *testing.T) {
},
}
},
- newSubmitterFn: func(ctx context.Context) (engine.Submitter, error) {
+ newSubmitterFn: func(ctx context.Context) (model.Submitter, error) {
return &mocks.Submitter{}, nil
},
- newSaverFn: func(experiment model.Experiment) (engine.Saver, error) {
+ newSaverFn: func(experiment model.Experiment) (model.Saver, error) {
return &mocks.Saver{}, nil
},
newInputProcessorFn: func(experiment model.Experiment, inputList []model.OOAPIURLInfo,
- saver engine.Saver, submitter engine.Submitter) inputProcessor {
+ saver model.Saver, submitter model.Submitter) inputProcessor {
return &mocks.ExperimentInputProcessor{
MockRun: func(ctx context.Context) error {
return errMocked
diff --git a/pkg/engine/inputprocessor.go b/pkg/oonirun/inputprocessor.go
similarity index 97%
rename from pkg/engine/inputprocessor.go
rename to pkg/oonirun/inputprocessor.go
index 79d7f4525..5fcf6707e 100644
--- a/pkg/engine/inputprocessor.go
+++ b/pkg/oonirun/inputprocessor.go
@@ -1,4 +1,4 @@
-package engine
+package oonirun
import (
"context"
@@ -76,11 +76,11 @@ type InputProcessorSaverWrapper interface {
}
type inputProcessorSaverWrapper struct {
- saver Saver
+ saver model.Saver
}
// NewInputProcessorSaverWrapper wraps a Saver for InputProcessor.
-func NewInputProcessorSaverWrapper(saver Saver) InputProcessorSaverWrapper {
+func NewInputProcessorSaverWrapper(saver model.Saver) InputProcessorSaverWrapper {
return inputProcessorSaverWrapper{saver: saver}
}
diff --git a/pkg/engine/inputprocessor_test.go b/pkg/oonirun/inputprocessor_test.go
similarity index 99%
rename from pkg/engine/inputprocessor_test.go
rename to pkg/oonirun/inputprocessor_test.go
index ed1f6d63c..a4e875121 100644
--- a/pkg/engine/inputprocessor_test.go
+++ b/pkg/oonirun/inputprocessor_test.go
@@ -1,4 +1,4 @@
-package engine
+package oonirun
import (
"context"
diff --git a/pkg/engine/saver.go b/pkg/oonirun/saver.go
similarity index 52%
rename from pkg/engine/saver.go
rename to pkg/oonirun/saver.go
index d128872d6..eaf6d424b 100644
--- a/pkg/engine/saver.go
+++ b/pkg/oonirun/saver.go
@@ -1,22 +1,17 @@
-package engine
+package oonirun
import (
"errors"
+ "github.com/ooni/probe-engine/pkg/engine"
"github.com/ooni/probe-engine/pkg/model"
)
-// Saver is an alias for model.Saver.
-type Saver = model.Saver
-
// SaverConfig is the configuration for creating a new Saver.
type SaverConfig struct {
// Enabled is true if saving is enabled.
Enabled bool
- // Experiment is the experiment we're currently running.
- Experiment SaverExperiment
-
// FilePath is the filepath where to append the measurement as a
// serialized JSON followed by a newline character.
FilePath string
@@ -25,23 +20,18 @@ type SaverConfig struct {
Logger model.Logger
}
-// SaverExperiment is an experiment according to the Saver.
-type SaverExperiment interface {
- SaveMeasurement(m *model.Measurement, filepath string) error
-}
-
// NewSaver creates a new instance of Saver.
-func NewSaver(config SaverConfig) (Saver, error) {
+func NewSaver(config SaverConfig) (model.Saver, error) {
if !config.Enabled {
return fakeSaver{}, nil
}
if config.FilePath == "" {
return nil, errors.New("saver: passed an empty filepath")
}
- return realSaver{
- Experiment: config.Experiment,
- FilePath: config.FilePath,
- Logger: config.Logger,
+ return &realSaver{
+ FilePath: config.FilePath,
+ Logger: config.Logger,
+ savefunc: engine.SaveMeasurement,
}, nil
}
@@ -51,17 +41,17 @@ func (fs fakeSaver) SaveMeasurement(m *model.Measurement) error {
return nil
}
-var _ Saver = fakeSaver{}
+var _ model.Saver = fakeSaver{}
type realSaver struct {
- Experiment SaverExperiment
- FilePath string
- Logger model.Logger
+ FilePath string
+ Logger model.Logger
+ savefunc func(measurement *model.Measurement, filePath string) error
}
-func (rs realSaver) SaveMeasurement(m *model.Measurement) error {
+func (rs *realSaver) SaveMeasurement(m *model.Measurement) error {
rs.Logger.Info("saving measurement to disk")
- return rs.Experiment.SaveMeasurement(m, rs.FilePath)
+ return rs.savefunc(m, rs.FilePath)
}
-var _ Saver = realSaver{}
+var _ model.Saver = &realSaver{}
diff --git a/pkg/engine/saver_test.go b/pkg/oonirun/saver_test.go
similarity index 67%
rename from pkg/engine/saver_test.go
rename to pkg/oonirun/saver_test.go
index fd292a330..9acac5bf9 100644
--- a/pkg/engine/saver_test.go
+++ b/pkg/oonirun/saver_test.go
@@ -1,12 +1,14 @@
-package engine
+package oonirun
import (
"errors"
+ "os"
"testing"
"github.com/apex/log"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/runtimex"
)
func TestNewSaverDisabled(t *testing.T) {
@@ -38,43 +40,39 @@ func TestNewSaverWithEmptyFilePath(t *testing.T) {
}
}
-type FakeSaverExperiment struct {
- M *model.Measurement
- Error error
- FilePath string
-}
-
-func (fse *FakeSaverExperiment) SaveMeasurement(m *model.Measurement, filepath string) error {
- fse.M = m
- fse.FilePath = filepath
- return fse.Error
-}
-
-var _ SaverExperiment = &FakeSaverExperiment{}
-
func TestNewSaverWithFailureWhenSaving(t *testing.T) {
+ filep := runtimex.Try1(os.CreateTemp("", ""))
+ filename := filep.Name()
+ filep.Close()
expected := errors.New("mocked error")
- fse := &FakeSaverExperiment{Error: expected}
saver, err := NewSaver(SaverConfig{
- Enabled: true,
- FilePath: "report.jsonl",
- Experiment: fse,
- Logger: log.Log,
+ Enabled: true,
+ FilePath: filename,
+ Logger: log.Log,
})
if err != nil {
t.Fatal(err)
}
- if _, ok := saver.(realSaver); !ok {
+ realSaver, ok := saver.(*realSaver)
+ if !ok {
t.Fatal("not the type of saver we expected")
}
+ var (
+ gotMeasurement *model.Measurement
+ gotFilePath string
+ )
+ realSaver.savefunc = func(measurement *model.Measurement, filePath string) error {
+ gotMeasurement, gotFilePath = measurement, filePath
+ return expected
+ }
m := &model.Measurement{Input: "www.kernel.org"}
if err := saver.SaveMeasurement(m); !errors.Is(err, expected) {
t.Fatalf("not the error we expected: %+v", err)
}
- if diff := cmp.Diff(fse.M, m); diff != "" {
+ if diff := cmp.Diff(m, gotMeasurement); diff != "" {
t.Fatal(diff)
}
- if fse.FilePath != "report.jsonl" {
+ if gotFilePath != filename {
t.Fatal("passed invalid filepath")
}
}
diff --git a/pkg/oonirun/session.go b/pkg/oonirun/session.go
index c1079399e..89710d522 100644
--- a/pkg/oonirun/session.go
+++ b/pkg/oonirun/session.go
@@ -20,7 +20,7 @@ type Session interface {
engine.InputLoaderSession
// A Session is also a SubmitterSession.
- engine.SubmitterSession
+ SubmitterSession
// DefaultHTTPClient returns the session's default HTTPClient.
DefaultHTTPClient() model.HTTPClient
diff --git a/pkg/engine/submitter.go b/pkg/oonirun/submitter.go
similarity index 99%
rename from pkg/engine/submitter.go
rename to pkg/oonirun/submitter.go
index 27ef11d7a..8301b13f8 100644
--- a/pkg/engine/submitter.go
+++ b/pkg/oonirun/submitter.go
@@ -1,4 +1,4 @@
-package engine
+package oonirun
import (
"context"
diff --git a/pkg/engine/submitter_test.go b/pkg/oonirun/submitter_test.go
similarity index 99%
rename from pkg/engine/submitter_test.go
rename to pkg/oonirun/submitter_test.go
index af0cd6c85..498303070 100644
--- a/pkg/engine/submitter_test.go
+++ b/pkg/oonirun/submitter_test.go
@@ -1,4 +1,4 @@
-package engine
+package oonirun
import (
"context"
diff --git a/pkg/oonirun/v2.go b/pkg/oonirun/v2.go
index 3b2865869..1dd4f53d9 100644
--- a/pkg/oonirun/v2.go
+++ b/pkg/oonirun/v2.go
@@ -14,7 +14,7 @@ import (
"github.com/hexops/gotextdiff"
"github.com/hexops/gotextdiff/myers"
"github.com/hexops/gotextdiff/span"
- "github.com/ooni/probe-engine/pkg/httpx"
+ "github.com/ooni/probe-engine/pkg/httpclientx"
"github.com/ooni/probe-engine/pkg/kvstore"
"github.com/ooni/probe-engine/pkg/model"
"github.com/ooni/probe-engine/pkg/runtimex"
@@ -60,28 +60,19 @@ type V2Nettest struct {
TestName string `json:"test_name"`
}
-// ErrHTTPRequestFailed indicates that an HTTP request failed.
-var ErrHTTPRequestFailed = errors.New("oonirun: HTTP request failed")
-
// getV2DescriptorFromHTTPSURL GETs a v2Descriptor instance from
// a static URL (e.g., from a GitHub repo or from a Gist).
func getV2DescriptorFromHTTPSURL(ctx context.Context, client model.HTTPClient,
logger model.Logger, URL string) (*V2Descriptor, error) {
- template := httpx.APIClientTemplate{
- Accept: "",
- Authorization: "",
- BaseURL: URL,
- HTTPClient: client,
- Host: "",
- LogBody: true,
- Logger: logger,
- UserAgent: model.HTTPHeaderUserAgent,
- }
- var desc V2Descriptor
- if err := template.Build().GetJSON(ctx, "", &desc); err != nil {
- return nil, err
- }
- return &desc, nil
+ return httpclientx.GetJSON[*V2Descriptor](
+ ctx,
+ httpclientx.NewEndpoint(URL),
+ &httpclientx.Config{
+ Authorization: "", // not needed
+ Client: client,
+ Logger: logger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
}
// v2DescriptorCache contains all the known v2Descriptor entries.
@@ -96,7 +87,11 @@ const v2DescriptorCacheKey = "oonirun-v2.state"
// v2DescriptorCacheLoad loads the v2DescriptorCache.
func v2DescriptorCacheLoad(fsstore model.KeyValueStore) (*v2DescriptorCache, error) {
+ // attempt to access the cache
data, err := fsstore.Get(v2DescriptorCacheKey)
+
+ // if there's a miss either create a new descriptor or return the
+ // error if it's something I/O related
if err != nil {
if errors.Is(err, kvstore.ErrNoSuchKey) {
cache := &v2DescriptorCache{
@@ -106,13 +101,19 @@ func v2DescriptorCacheLoad(fsstore model.KeyValueStore) (*v2DescriptorCache, err
}
return nil, err
}
+
+ // transform the raw descriptor into a struct
var cache v2DescriptorCache
if err := json.Unmarshal(data, &cache); err != nil {
return nil, err
}
+
+ // handle the case where there are no entries inside the on-disk cache
+ // by properly initializing to a non-nil map
if cache.Entries == nil {
cache.Entries = make(map[string]*V2Descriptor)
}
+
return &cache, nil
}
@@ -168,13 +169,18 @@ func V2MeasureDescriptor(ctx context.Context, config *LinkConfig, desc *V2Descri
// more robust in terms of the implementation.
return ErrNilDescriptor
}
+
logger := config.Session.Logger()
+
for _, nettest := range desc.Nettests {
+ // early handling of the case where the test name is empty
if nettest.TestName == "" {
logger.Warn("oonirun: nettest name cannot be empty")
v2CountEmptyNettestNames.Add(1)
continue
}
+
+ // construct an experiment from the current nettest
exp := &Experiment{
Annotations: config.Annotations,
ExtraOptions: nettest.Options,
@@ -193,12 +199,15 @@ func V2MeasureDescriptor(ctx context.Context, config *LinkConfig, desc *V2Descri
newSaverFn: nil,
newInputProcessorFn: nil,
}
+
+ // actually run the experiment
if err := exp.Run(ctx); err != nil {
logger.Warnf("cannot run experiment: %s", err.Error())
v2CountFailedExperiments.Add(1)
continue
}
}
+
return nil
}
@@ -209,14 +218,25 @@ var ErrNeedToAcceptChanges = errors.New("oonirun: need to accept changes")
// v2DescriptorDiff shows what changed between the old and the new descriptors.
func v2DescriptorDiff(oldValue, newValue *V2Descriptor, URL string) string {
+ // JSON serialize old descriptor
oldData, err := json.MarshalIndent(oldValue, "", " ")
runtimex.PanicOnError(err, "json.MarshalIndent failed unexpectedly")
+
+ // JSON serialize new descriptor
newData, err := json.MarshalIndent(newValue, "", " ")
runtimex.PanicOnError(err, "json.MarshalIndent failed unexpectedly")
+
+ // make sure the serializations are newline-terminated
oldString, newString := string(oldData)+"\n", string(newData)+"\n"
+
+ // generate names for the final diff
oldFile := "OLD " + URL
newFile := "NEW " + URL
+
+ // compute the edits to update from the old to the new descriptor
edits := myers.ComputeEdits(span.URIFromPath(oldFile), oldString, newString)
+
+ // transform the edits and obtain an unified diff
return fmt.Sprint(gotextdiff.ToUnified(oldFile, newFile, oldString, edits))
}
@@ -233,25 +253,39 @@ func v2DescriptorDiff(oldValue, newValue *V2Descriptor, URL string) string {
func v2MeasureHTTPS(ctx context.Context, config *LinkConfig, URL string) error {
logger := config.Session.Logger()
logger.Infof("oonirun/v2: running %s", URL)
+
+ // load the descriptor from the cache
cache, err := v2DescriptorCacheLoad(config.KVStore)
if err != nil {
return err
}
+
+ // pull a possibly new descriptor without updating the old descriptor
clnt := config.Session.DefaultHTTPClient()
oldValue, newValue, err := cache.PullChangesWithoutSideEffects(ctx, clnt, logger, URL)
if err != nil {
return err
}
+
+ // compare the new descriptor to the old descriptor
diff := v2DescriptorDiff(oldValue, newValue, URL)
+
+ // possibly stop if configured to ask for permission when accepting changes
if !config.AcceptChanges && diff != "" {
logger.Warnf("oonirun: %s changed as follows:\n\n%s", URL, diff)
logger.Warnf("oonirun: we are not going to run this link until you accept changes")
return ErrNeedToAcceptChanges
}
+
+ // in case there are changes, update the descriptor
if diff != "" {
if err := cache.Update(config.KVStore, URL, newValue); err != nil {
return err
}
}
- return V2MeasureDescriptor(ctx, config, newValue) // handles nil newValue gracefully
+
+ // measure using the possibly-new descriptor
+ //
+ // note: this function gracefully handles nil values
+ return V2MeasureDescriptor(ctx, config, newValue)
}
diff --git a/pkg/oonirun/v2_test.go b/pkg/oonirun/v2_test.go
index a36761e4f..5d5520922 100644
--- a/pkg/oonirun/v2_test.go
+++ b/pkg/oonirun/v2_test.go
@@ -9,13 +9,17 @@ import (
"testing"
"time"
+ "github.com/ooni/probe-engine/pkg/httpclientx"
"github.com/ooni/probe-engine/pkg/kvstore"
"github.com/ooni/probe-engine/pkg/mocks"
"github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/netxlite"
"github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/testingx"
)
func TestOONIRunV2LinkCommonCase(t *testing.T) {
+ // make a local server that returns a reasonable descriptor for the example experiment
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
descriptor := &V2Descriptor{
Name: "",
@@ -33,8 +37,10 @@ func TestOONIRunV2LinkCommonCase(t *testing.T) {
runtimex.PanicOnError(err, "json.Marshal failed")
w.Write(data)
}))
+
defer server.Close()
ctx := context.Background()
+
config := &LinkConfig{
AcceptChanges: true, // avoid "oonirun: need to accept changes" error
Annotations: map[string]string{
@@ -42,19 +48,24 @@ func TestOONIRunV2LinkCommonCase(t *testing.T) {
},
KVStore: &kvstore.Memory{},
MaxRuntime: 0,
- NoCollector: true,
+ NoCollector: true, // disable collector so we don't submit
NoJSON: true,
Random: false,
ReportFile: "",
Session: newMinimalFakeSession(),
}
+
+ // create a link runner for the local server URL
r := NewLinkRunner(config, server.URL)
+
+ // run and verify that we could run without getting errors
if err := r.Run(ctx); err != nil {
t.Fatal(err)
}
}
func TestOONIRunV2LinkCannotUpdateCache(t *testing.T) {
+ // make a server that returns a minimal descriptor for the example experiment
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
descriptor := &V2Descriptor{
Name: "",
@@ -72,8 +83,12 @@ func TestOONIRunV2LinkCannotUpdateCache(t *testing.T) {
runtimex.PanicOnError(err, "json.Marshal failed")
w.Write(data)
}))
+
defer server.Close()
ctx := context.Background()
+
+ // create with a key value store that returns an empty cache and fails to update
+ // the cache afterwards such that we can see if we detect such an error
expected := errors.New("mocked")
config := &LinkConfig{
AcceptChanges: true, // avoid "oonirun: need to accept changes" error
@@ -95,14 +110,21 @@ func TestOONIRunV2LinkCannotUpdateCache(t *testing.T) {
ReportFile: "",
Session: newMinimalFakeSession(),
}
+
+ // create new runner for the local server URL
r := NewLinkRunner(config, server.URL)
+
+ // attempt to run the link
err := r.Run(ctx)
+
+ // make sure we exactly got the cache updating error
if !errors.Is(err, expected) {
t.Fatal("unexpected err", err)
}
}
func TestOONIRunV2LinkWithoutAcceptChanges(t *testing.T) {
+ // make a local server that would return a reasonable descriptor
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
descriptor := &V2Descriptor{
Name: "",
@@ -120,8 +142,11 @@ func TestOONIRunV2LinkWithoutAcceptChanges(t *testing.T) {
runtimex.PanicOnError(err, "json.Marshal failed")
w.Write(data)
}))
+
defer server.Close()
ctx := context.Background()
+
+ // create a minimal link configuration
config := &LinkConfig{
AcceptChanges: false, // should see "oonirun: need to accept changes" error
Annotations: map[string]string{
@@ -135,19 +160,29 @@ func TestOONIRunV2LinkWithoutAcceptChanges(t *testing.T) {
ReportFile: "",
Session: newMinimalFakeSession(),
}
+
+ // create a new runner for the local server URL
r := NewLinkRunner(config, server.URL)
+
+ // attempt to run the link
err := r.Run(ctx)
+
+ // make sure the error indicates we need to accept changes
if !errors.Is(err, ErrNeedToAcceptChanges) {
t.Fatal("unexpected err", err)
}
}
func TestOONIRunV2LinkNilDescriptor(t *testing.T) {
+ // create a local server that returns a literal "null" as the JSON descriptor
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("null"))
}))
+
defer server.Close()
ctx := context.Background()
+
+ // create a minimal link configuration
config := &LinkConfig{
AcceptChanges: true, // avoid "oonirun: need to accept changes" error
Annotations: map[string]string{
@@ -161,14 +196,23 @@ func TestOONIRunV2LinkNilDescriptor(t *testing.T) {
ReportFile: "",
Session: newMinimalFakeSession(),
}
+
+ // attempt to run the link at the local server
r := NewLinkRunner(config, server.URL)
- if err := r.Run(ctx); err != nil {
- t.Fatal(err)
+
+ // make sure we correctly handled an invalid "null" descriptor
+ if err := r.Run(ctx); !errors.Is(err, httpclientx.ErrIsNil) {
+ t.Fatal("unexpected error", err)
}
}
func TestOONIRunV2LinkEmptyTestName(t *testing.T) {
+ // load the count of the number of cases where the test name was empty so we can
+ // later on check whether this count has increased due to running this test
emptyTestNamesPrev := v2CountEmptyNettestNames.Load()
+
+ // create a local server that will respond with a minimal descriptor that
+ // actually contains an empty test name, which is what we want to test
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
descriptor := &V2Descriptor{
Name: "",
@@ -186,8 +230,11 @@ func TestOONIRunV2LinkEmptyTestName(t *testing.T) {
runtimex.PanicOnError(err, "json.Marshal failed")
w.Write(data)
}))
+
defer server.Close()
ctx := context.Background()
+
+ // create a minimal link configuration
config := &LinkConfig{
AcceptChanges: true, // avoid "oonirun: need to accept changes" error
Annotations: map[string]string{
@@ -201,30 +248,116 @@ func TestOONIRunV2LinkEmptyTestName(t *testing.T) {
ReportFile: "",
Session: newMinimalFakeSession(),
}
+
+ // construct a link runner relative to the local server URL
r := NewLinkRunner(config, server.URL)
+
+ // attempt to run and verify there's no error (the code only emits a warning in this case)
if err := r.Run(ctx); err != nil {
t.Fatal(err)
}
+
+ // make sure the loop for running nettests continued where we expected it to do so
if v2CountEmptyNettestNames.Load() != emptyTestNamesPrev+1 {
t.Fatal("expected to see 1 more instance of empty nettest names")
}
}
+func TestOONIRunV2LinkConnectionResetByPeer(t *testing.T) {
+ // create a local server that will reset the connection immediately.
+ // actually contains an empty test name, which is what we want to test
+ server := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+
+ defer server.Close()
+ ctx := context.Background()
+
+ // create a minimal link configuration
+ config := &LinkConfig{
+ AcceptChanges: true, // avoid "oonirun: need to accept changes" error
+ Annotations: map[string]string{
+ "platform": "linux",
+ },
+ KVStore: &kvstore.Memory{},
+ MaxRuntime: 0,
+ NoCollector: true,
+ NoJSON: true,
+ Random: false,
+ ReportFile: "",
+ Session: newMinimalFakeSession(),
+ }
+
+ // construct a link runner relative to the local server URL
+ r := NewLinkRunner(config, server.URL)
+
+ // attempt to run and verify we got ECONNRESET
+ if err := r.Run(ctx); !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+}
+
+func TestOONIRunV2LinkNonParseableJSON(t *testing.T) {
+ // create a local server that will respond with a non-parseable JSON.
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{`))
+ }))
+
+ defer server.Close()
+ ctx := context.Background()
+
+ // create a minimal link configuration
+ config := &LinkConfig{
+ AcceptChanges: true, // avoid "oonirun: need to accept changes" error
+ Annotations: map[string]string{
+ "platform": "linux",
+ },
+ KVStore: &kvstore.Memory{},
+ MaxRuntime: 0,
+ NoCollector: true,
+ NoJSON: true,
+ Random: false,
+ ReportFile: "",
+ Session: newMinimalFakeSession(),
+ }
+
+ // construct a link runner relative to the local server URL
+ r := NewLinkRunner(config, server.URL)
+
+ // attempt to run and verify there's a JSON parsing error
+ if err := r.Run(ctx); err == nil || err.Error() != "unexpected end of JSON input" {
+ t.Fatal("unexpected error", err)
+ }
+}
+
func TestV2MeasureDescriptor(t *testing.T) {
+
t.Run("with nil descriptor", func(t *testing.T) {
ctx := context.Background()
config := &LinkConfig{}
+
+ // invoke the function with a nil descriptor and make sure the code
+ // is correctly handling this specific case by returnning error
err := V2MeasureDescriptor(ctx, config, nil)
+
if !errors.Is(err, ErrNilDescriptor) {
t.Fatal("unexpected err", err)
}
})
t.Run("with failing experiment", func(t *testing.T) {
+ // load the previous count of failed experiments so we can check that it increased later
previousFailedExperiments := v2CountFailedExperiments.Load()
+
expected := errors.New("mocked error")
+
ctx := context.Background()
sess := newMinimalFakeSession()
+
+ // create a mocked submitter that will panic in case we try to submit, such that
+ // this test fails with a panic if we go as far as attempting to submit
+ //
+ // Note: the convention is that we do not submit experiment results when the
+ // experiment measurement function returns a non-nil error, since such an error
+ // represents a fundamental failure in setting up the experiment
sess.MockNewSubmitter = func(ctx context.Context) (model.Submitter, error) {
subm := &mocks.Submitter{
MockSubmit: func(ctx context.Context, m *model.Measurement) error {
@@ -233,6 +366,9 @@ func TestV2MeasureDescriptor(t *testing.T) {
}
return subm, nil
}
+
+ // mock an experiment builder where we have the measurement function fail by returning
+ // an error, which has the meaning indicated in the previous comment
sess.MockNewExperimentBuilder = func(name string) (model.ExperimentBuilder, error) {
eb := &mocks.ExperimentBuilder{
MockInputPolicy: func() model.InputPolicy {
@@ -258,6 +394,8 @@ func TestV2MeasureDescriptor(t *testing.T) {
}
return eb, nil
}
+
+ // create a mostly empty config referring to the session
config := &LinkConfig{
AcceptChanges: false,
Annotations: map[string]string{},
@@ -269,6 +407,8 @@ func TestV2MeasureDescriptor(t *testing.T) {
ReportFile: "",
Session: sess,
}
+
+ // create a mostly empty descriptor referring to the example experiment
descr := &V2Descriptor{
Name: "",
Description: "",
@@ -279,10 +419,18 @@ func TestV2MeasureDescriptor(t *testing.T) {
TestName: "example",
}},
}
+
+ // attempt to measure this descriptor
err := V2MeasureDescriptor(ctx, config, descr)
+
+ // here we do not expect to see an error because the implementation continues
+ // until it has run all experiments and just emits warning messages
if err != nil {
t.Fatal(err)
}
+
+ // however there's also a count of the number of times we failed to load
+ // an experiment and we use that to make sure the code failed where we expected
if v2CountFailedExperiments.Load() != previousFailedExperiments+1 {
t.Fatal("expected to see a failed experiment")
}
@@ -290,9 +438,13 @@ func TestV2MeasureDescriptor(t *testing.T) {
}
func TestV2MeasureHTTPS(t *testing.T) {
+
t.Run("when we cannot load from cache", func(t *testing.T) {
expected := errors.New("mocked error")
ctx := context.Background()
+
+ // construct the link configuration with a key-value store that fails
+ // with a well-know error when attempting to load.
config := &LinkConfig{
AcceptChanges: false,
Annotations: map[string]string{},
@@ -308,15 +460,22 @@ func TestV2MeasureHTTPS(t *testing.T) {
ReportFile: "",
Session: newMinimalFakeSession(),
}
+
+ // attempt to measure with the given config (there's no need to pass an URL
+ // here because we should fail to load from the cache first)
err := v2MeasureHTTPS(ctx, config, "")
+
+ // verify that we've actually got the expected error
if !errors.Is(err, expected) {
t.Fatal("unexpected err", err)
}
})
t.Run("when we cannot pull changes", func(t *testing.T) {
+ // create and immediately cancel a context so that HTTP would fail
ctx, cancel := context.WithCancel(context.Background())
cancel() // fail immediately
+
config := &LinkConfig{
AcceptChanges: false,
Annotations: map[string]string{},
@@ -328,25 +487,39 @@ func TestV2MeasureHTTPS(t *testing.T) {
ReportFile: "",
Session: newMinimalFakeSession(),
}
- err := v2MeasureHTTPS(ctx, config, "https://example.com") // should not use URL
+
+ // attempt to measure with a random URL (which is fine since we shouldn't use it)
+ err := v2MeasureHTTPS(ctx, config, "https://example.com")
+
+ // make sure that we've actually go the expected error
if !errors.Is(err, context.Canceled) {
t.Fatal("unexpected err", err)
}
})
+
}
func TestV2DescriptorCacheLoad(t *testing.T) {
- t.Run("cannot unmarshal cache content", func(t *testing.T) {
+
+ t.Run("handle the case where we cannot unmarshal the cache content", func(t *testing.T) {
+ // write an invalid serialized JSON into the cache
fsstore := &kvstore.Memory{}
if err := fsstore.Set(v2DescriptorCacheKey, []byte("{")); err != nil {
t.Fatal(err)
}
+
+ // attempt to load descriptors
cache, err := v2DescriptorCacheLoad(fsstore)
+
+ // make sure we cannot unmarshal
if err == nil || err.Error() != "unexpected end of JSON input" {
t.Fatal("unexpected err", err)
}
+
+ // make sure the returned cache is nil
if cache != nil {
t.Fatal("expected nil cache")
}
})
+
}
diff --git a/pkg/probeservices/bouncer.go b/pkg/probeservices/bouncer.go
index 66a923c49..48bced5ff 100644
--- a/pkg/probeservices/bouncer.go
+++ b/pkg/probeservices/bouncer.go
@@ -1,14 +1,32 @@
package probeservices
+//
+// bouncer.go - GET /api/v1/test-helpers
+//
+
import (
"context"
+ "github.com/ooni/probe-engine/pkg/httpclientx"
"github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/urlx"
)
-// GetTestHelpers is like GetCollectors but for test helpers.
-func (c Client) GetTestHelpers(
- ctx context.Context) (output map[string][]model.OOAPIService, err error) {
- err = c.APIClientTemplate.WithBodyLogging().Build().GetJSON(ctx, "/api/v1/test-helpers", &output)
- return
+// GetTestHelpers queries the /api/v1/test-helpers API.
+func (c *Client) GetTestHelpers(ctx context.Context) (map[string][]model.OOAPIService, error) {
+ // construct the URL to use
+ URL, err := urlx.ResolveReference(c.BaseURL, "/api/v1/test-helpers", "")
+ if err != nil {
+ return nil, err
+ }
+
+ // get the response
+ return httpclientx.GetJSON[map[string][]model.OOAPIService](
+ ctx,
+ httpclientx.NewEndpoint(URL).WithHostOverride(c.Host),
+ &httpclientx.Config{
+ Client: c.HTTPClient,
+ Logger: c.Logger,
+ UserAgent: c.UserAgent,
+ })
}
diff --git a/pkg/probeservices/bouncer_test.go b/pkg/probeservices/bouncer_test.go
index 25f23a851..71b2574b5 100644
--- a/pkg/probeservices/bouncer_test.go
+++ b/pkg/probeservices/bouncer_test.go
@@ -2,18 +2,236 @@ package probeservices
import (
"context"
+ "errors"
+ "net/http"
+ "net/url"
"testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/mocks"
+ "github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/must"
+ "github.com/ooni/probe-engine/pkg/netxlite"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/testingx"
)
func TestGetTestHelpers(t *testing.T) {
- if testing.Short() {
- t.Skip("skip test in short mode")
- }
- testhelpers, err := newclient().GetTestHelpers(context.Background())
- if err != nil {
- t.Fatal(err)
- }
- if len(testhelpers) <= 1 {
- t.Fatal("no returned test helpers?!")
- }
+
+ // First, let's check whether we can get a response from the real OONI backend.
+ t.Run("is working as intended with the real backend", func(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skip test in short mode")
+ }
+
+ // create client
+ client := newclient()
+
+ // issue the request
+ testhelpers, err := client.GetTestHelpers(context.Background())
+
+ // we do not expect an error here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // we expect at least one TH
+ if len(testhelpers) <= 1 {
+ t.Fatal("no returned test helpers?!")
+ }
+ })
+
+ // Now let's construct a test server that returns a valid response and try
+ // to communicate with such a test server successfully and with errors
+
+ t.Run("is working as intended with a local test server", func(t *testing.T) {
+ // this is what we expect to receive
+ expect := map[string][]model.OOAPIService{
+ "web-connectivity": {{
+ Address: "https://0.th.ooni.org/",
+ Type: "https",
+ }},
+ }
+
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ runtimex.Assert(r.Method == http.MethodGet, "invalid method")
+ runtimex.Assert(r.URL.Path == "/api/v1/test-helpers", "invalid URL path")
+ w.Write(must.MarshalJSON(expect))
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // issue the GET request
+ testhelpers, err := client.GetTestHelpers(context.Background())
+
+ // we do not expect an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // we expect to see exactly what the server sent
+ if diff := cmp.Diff(expect, testhelpers); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+
+ t.Run("we can use cloudfronting", func(t *testing.T) {
+ // this is what we expect to receive
+ expect := map[string][]model.OOAPIService{
+ "web-connectivity": {{
+ Address: "https://0.th.ooni.org/",
+ Type: "https",
+ }},
+ }
+
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ runtimex.Assert(r.Host == "www.cloudfront.com", "invalid r.Host")
+ runtimex.Assert(r.Method == http.MethodGet, "invalid method")
+ runtimex.Assert(r.URL.Path == "/api/v1/test-helpers", "invalid URL path")
+ w.Write(must.MarshalJSON(expect))
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // make sure we're using cloudfronting
+ client.Host = "www.cloudfront.com"
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // issue the GET request
+ testhelpers, err := client.GetTestHelpers(context.Background())
+
+ // we do not expect an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // we expect to see exactly what the server sent
+ if diff := cmp.Diff(expect, testhelpers); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+
+ t.Run("reports an error when the connection is reset", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // issue the GET request
+ testhelpers, err := client.GetTestHelpers(context.Background())
+
+ // we do expect an error
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect to see a zero-length / nil map
+ if len(testhelpers) != 0 {
+ t.Fatal("expected result lenght to be zero")
+ }
+ })
+
+ t.Run("reports an error when the response is not JSON parsable", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{`))
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // issue the GET request
+ testhelpers, err := client.GetTestHelpers(context.Background())
+
+ // we do expect an error
+ if err == nil || err.Error() != "unexpected end of JSON input" {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect to see a zero-length / nil map
+ if len(testhelpers) != 0 {
+ t.Fatal("expected result lenght to be zero")
+ }
+ })
+
+ t.Run("correctly handles the case where the URL is unparseable", func(t *testing.T) {
+ // create a probeservices client
+ client := newclient()
+
+ // override the URL to be unparseable
+ client.BaseURL = "\t\t\t"
+
+ // issue the GET request
+ testhelpers, err := client.GetTestHelpers(context.Background())
+
+ // we do expect an error
+ if err == nil || err.Error() != `parse "\t\t\t": net/url: invalid control character in URL` {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect to see a zero-length / nil map
+ if len(testhelpers) != 0 {
+ t.Fatal("expected result lenght to be zero")
+ }
+ })
}
diff --git a/pkg/probeservices/checkin.go b/pkg/probeservices/checkin.go
index abb58847c..2fa15b275 100644
--- a/pkg/probeservices/checkin.go
+++ b/pkg/probeservices/checkin.go
@@ -4,9 +4,9 @@ import (
"context"
"github.com/ooni/probe-engine/pkg/checkincache"
- "github.com/ooni/probe-engine/pkg/httpapi"
+ "github.com/ooni/probe-engine/pkg/httpclientx"
"github.com/ooni/probe-engine/pkg/model"
- "github.com/ooni/probe-engine/pkg/ooapi"
+ "github.com/ooni/probe-engine/pkg/urlx"
)
// CheckIn function is called by probes asking if there are tests to be run
@@ -17,17 +17,33 @@ import (
// or an explanatory error, in case of failure.
func (c Client) CheckIn(
ctx context.Context, config model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) {
- // prepare endpoint and descriptor for the API call
- epnt := c.newHTTPAPIEndpoint()
- desc := ooapi.NewDescriptorCheckIn(&config)
+ // construct the URL to use
+ URL, err := urlx.ResolveReference(c.BaseURL, "/api/v1/check-in", "")
+ if err != nil {
+ return nil, err
+ }
+
+ // issue the API call
+ resp, err := httpclientx.PostJSON[*model.OOAPICheckInConfig, *model.OOAPICheckInResult](
+ ctx,
+ httpclientx.NewEndpoint(URL).WithHostOverride(c.Host),
+ &config,
+ &httpclientx.Config{
+ Authorization: "", // not needed
+ Client: c.HTTPClient,
+ Logger: c.Logger,
+ UserAgent: c.UserAgent,
+ })
- // issue the API call and handle failures
- resp, err := httpapi.Call(ctx, desc, epnt)
+ // handle the case of error
if err != nil {
return nil, err
}
- // make sure we track selected parts of the response
+ // make sure we track selected parts of the response and ignore
+ // the error because OONI Probe would also work without this caching
+ // it would only work more poorly, but it does not seem worth it
+ // crippling it entirely if we cannot write into the kvstore
_ = checkincache.Store(c.KVStore, resp)
return resp, nil
}
diff --git a/pkg/probeservices/checkin_test.go b/pkg/probeservices/checkin_test.go
index 2b50ef35c..3cb6623ce 100644
--- a/pkg/probeservices/checkin_test.go
+++ b/pkg/probeservices/checkin_test.go
@@ -2,19 +2,26 @@ package probeservices
import (
"context"
- "strings"
+ "encoding/json"
+ "errors"
+ "net/http"
+ "net/url"
"testing"
+ "time"
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/checkincache"
+ "github.com/ooni/probe-engine/pkg/kvstore"
+ "github.com/ooni/probe-engine/pkg/mocks"
"github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/must"
+ "github.com/ooni/probe-engine/pkg/netxlite"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/testingx"
)
-func TestCheckInSuccess(t *testing.T) {
- if testing.Short() {
- t.Skip("skip test in short mode")
- }
-
- client := newclient()
- client.BaseURL = "https://ams-pg-test.ooni.org"
+func TestCheckIn(t *testing.T) {
+ // define a common configuration to use across all tests
config := model.OOAPICheckInConfig{
Charging: true,
OnWiFi: true,
@@ -28,49 +35,485 @@ func TestCheckInSuccess(t *testing.T) {
CategoryCodes: []string{"NEWS", "CULTR"},
},
}
- ctx := context.Background()
- result, err := client.CheckIn(ctx, config)
- if err != nil {
- t.Fatal(err)
- }
- if result == nil || result.Tests.WebConnectivity == nil {
- t.Fatal("got nil result or WebConnectivity")
- }
- if result.Tests.WebConnectivity.ReportID == "" {
- t.Fatal("ReportID is empty")
- }
- if len(result.Tests.WebConnectivity.URLs) < 1 {
- t.Fatal("unexpected number of URLs")
- }
- for _, entry := range result.Tests.WebConnectivity.URLs {
- if entry.CategoryCode != "NEWS" && entry.CategoryCode != "CULTR" {
- t.Fatalf("unexpected category code: %+v", entry)
+
+ t.Run("with the real API server", func(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skip test in short mode")
}
- }
-}
-func TestCheckInFailure(t *testing.T) {
- client := newclient()
- client.BaseURL = "https://\t\t\t/" // cause test to fail
- config := model.OOAPICheckInConfig{
- Charging: true,
- OnWiFi: true,
- Platform: "android",
- ProbeASN: "AS12353",
- ProbeCC: "PT",
- RunType: model.RunTypeTimed,
- SoftwareName: "ooniprobe-android",
- SoftwareVersion: "2.7.1",
- WebConnectivity: model.OOAPICheckInConfigWebConnectivity{
- CategoryCodes: []string{"NEWS", "CULTR"},
- },
- }
- ctx := context.Background()
- result, err := client.CheckIn(ctx, config)
- if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
- t.Fatal("not the error we expected")
- }
- if result != nil {
- t.Fatal("results?!")
- }
+ client := newclient()
+ client.BaseURL = "https://ams-pg-test.ooni.org" // use the test infra
+
+ ctx := context.Background()
+
+ // call the API
+ result, err := client.CheckIn(ctx, config)
+
+ // we do not expect to see an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // sanity check the returned response
+ if result == nil || result.Tests.WebConnectivity == nil {
+ t.Fatal("got nil result or nil WebConnectivity")
+ }
+ if result.Tests.WebConnectivity.ReportID == "" {
+ t.Fatal("ReportID is empty")
+ }
+ if len(result.Tests.WebConnectivity.URLs) < 1 {
+ t.Fatal("unexpected number of URLs")
+ }
+
+ // ensure the category codes match our request
+ for _, entry := range result.Tests.WebConnectivity.URLs {
+ if entry.CategoryCode != "NEWS" && entry.CategoryCode != "CULTR" {
+ t.Fatalf("unexpected category code: %+v", entry)
+ }
+ }
+ })
+
+ t.Run("with a working-as-intended local server", func(t *testing.T) {
+ // define our expectations
+ expect := &model.OOAPICheckInResult{
+ Conf: model.OOAPICheckInResultConfig{
+ Features: map[string]bool{},
+ TestHelpers: map[string][]model.OOAPIService{
+ "web-connectivity": {{
+ Address: "https://0.th.ooni.org/",
+ Type: "https",
+ }},
+ },
+ },
+ ProbeASN: "AS30722",
+ ProbeCC: "US",
+ Tests: model.OOAPICheckInResultNettests{
+ WebConnectivity: &model.OOAPICheckInInfoWebConnectivity{
+ ReportID: "20240424T134700Z_webconnectivity_IT_30722_n1_q5N5YSTWEqHYDo9v",
+ URLs: []model.OOAPIURLInfo{{
+ CategoryCode: "NEWS",
+ CountryCode: "IT",
+ URL: "https://www.example.com/",
+ }},
+ },
+ },
+ UTCTime: time.Date(2022, 11, 22, 1, 2, 3, 0, time.UTC),
+ V: 1,
+ }
+
+ // create a local server that responds with the expectation
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ runtimex.Assert(r.Method == http.MethodPost, "invalid method")
+ runtimex.Assert(r.URL.Path == "/api/v1/check-in", "invalid URL path")
+ rawreqbody := runtimex.Try1(netxlite.ReadAllContext(r.Context(), r.Body))
+ var gotrequest model.OOAPICheckInConfig
+ must.UnmarshalJSON(rawreqbody, &gotrequest)
+ diff := cmp.Diff(config, gotrequest)
+ runtimex.Assert(diff == "", "request mismatch:"+diff)
+ w.Write(must.MarshalJSON(expect))
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // call the API
+ result, err := client.CheckIn(context.Background(), config)
+
+ // we do not expect to see an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // we expect to see exactly what the server sent
+ if diff := cmp.Diff(expect, result); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+
+ t.Run("we can use cloudfronting", func(t *testing.T) {
+ // define our expectations
+ expect := &model.OOAPICheckInResult{
+ Conf: model.OOAPICheckInResultConfig{
+ Features: map[string]bool{},
+ TestHelpers: map[string][]model.OOAPIService{
+ "web-connectivity": {{
+ Address: "https://0.th.ooni.org/",
+ Type: "https",
+ }},
+ },
+ },
+ ProbeASN: "AS30722",
+ ProbeCC: "US",
+ Tests: model.OOAPICheckInResultNettests{
+ WebConnectivity: &model.OOAPICheckInInfoWebConnectivity{
+ ReportID: "20240424T134700Z_webconnectivity_IT_30722_n1_q5N5YSTWEqHYDo9v",
+ URLs: []model.OOAPIURLInfo{{
+ CategoryCode: "NEWS",
+ CountryCode: "IT",
+ URL: "https://www.example.com/",
+ }},
+ },
+ },
+ UTCTime: time.Date(2022, 11, 22, 1, 2, 3, 0, time.UTC),
+ V: 1,
+ }
+
+ // create a local server that responds with the expectation
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ runtimex.Assert(r.Host == "www.cloudfront.com", "invalid r.Host")
+ runtimex.Assert(r.Method == http.MethodPost, "invalid method")
+ runtimex.Assert(r.URL.Path == "/api/v1/check-in", "invalid URL path")
+ rawreqbody := runtimex.Try1(netxlite.ReadAllContext(r.Context(), r.Body))
+ var gotrequest model.OOAPICheckInConfig
+ must.UnmarshalJSON(rawreqbody, &gotrequest)
+ diff := cmp.Diff(config, gotrequest)
+ runtimex.Assert(diff == "", "request mismatch:"+diff)
+ w.Write(must.MarshalJSON(expect))
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // make sure we're using cloudfronting
+ client.Host = "www.cloudfront.com"
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // call the API
+ result, err := client.CheckIn(context.Background(), config)
+
+ // we do not expect to see an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // we expect to see exactly what the server sent
+ if diff := cmp.Diff(expect, result); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+
+ t.Run("reports an error when the connection is reset", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // call the API
+ result, err := client.CheckIn(context.Background(), config)
+
+ // we do expect an error
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect to see a nil pointer
+ if result != nil {
+ t.Fatal("expected result to be nil")
+ }
+ })
+
+ t.Run("reports an error when the response is not JSON parsable", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{`))
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // call the API
+ result, err := client.CheckIn(context.Background(), config)
+
+ // we do expect an error
+ if err == nil || err.Error() != "unexpected end of JSON input" {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect to see a nil pointer
+ if result != nil {
+ t.Fatal("expected result to be nil")
+ }
+ })
+
+ t.Run("correctly handles the case where the URL is unparseable", func(t *testing.T) {
+ // create a probeservices client
+ client := newclient()
+
+ // override the URL to be unparseable
+ client.BaseURL = "\t\t\t"
+
+ // call the API
+ result, err := client.CheckIn(context.Background(), config)
+
+ // we do expect an error
+ if err == nil || err.Error() != `parse "\t\t\t": net/url: invalid control character in URL` {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect to see a nil pointer
+ if result != nil {
+ t.Fatal("expected result to be nil")
+ }
+ })
+
+ t.Run("we store feature flags coming from the check-in API", func(t *testing.T) {
+ // define our expectations
+ //
+ // note: when calling the checkincache.GetFeatureFlag function, we will
+ // get a false result if there's no corresponding information into the
+ // key-value store. So, we're setting true here to check we can read from
+ // the key-value store values that are different from the default.
+ expect := &model.OOAPICheckInResult{
+ Conf: model.OOAPICheckInResultConfig{
+ Features: map[string]bool{
+ "torsf_enabled": true,
+ "vanilla_tor_enabled": true,
+ },
+ TestHelpers: map[string][]model.OOAPIService{
+ "web-connectivity": {{
+ Address: "https://0.th.ooni.org/",
+ Type: "https",
+ }},
+ },
+ },
+ ProbeASN: "AS30722",
+ ProbeCC: "US",
+ Tests: model.OOAPICheckInResultNettests{
+ WebConnectivity: &model.OOAPICheckInInfoWebConnectivity{
+ ReportID: "20240424T134700Z_webconnectivity_IT_30722_n1_q5N5YSTWEqHYDo9v",
+ URLs: []model.OOAPIURLInfo{{
+ CategoryCode: "NEWS",
+ CountryCode: "IT",
+ URL: "https://www.example.com/",
+ }},
+ },
+ },
+ UTCTime: time.Date(2022, 11, 22, 1, 2, 3, 0, time.UTC),
+ V: 1,
+ }
+
+ // create a local server that responds with the expectation
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ runtimex.Assert(r.Method == http.MethodPost, "invalid method")
+ runtimex.Assert(r.URL.Path == "/api/v1/check-in", "invalid URL path")
+ rawreqbody := runtimex.Try1(netxlite.ReadAllContext(r.Context(), r.Body))
+ var gotrequest model.OOAPICheckInConfig
+ must.UnmarshalJSON(rawreqbody, &gotrequest)
+ diff := cmp.Diff(config, gotrequest)
+ runtimex.Assert(diff == "", "request mismatch:"+diff)
+ w.Write(must.MarshalJSON(expect))
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // call the API
+ result, err := client.CheckIn(context.Background(), config)
+
+ // we do not expect to see an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // we expect to see exactly what the server sent
+ if diff := cmp.Diff(expect, result); diff != "" {
+ t.Fatal(diff)
+ }
+
+ // make sure we have written non-default values into the key-value store
+ if !checkincache.GetFeatureFlag(client.KVStore, "torsf_enabled") {
+ t.Fatal("expected to see true here")
+ }
+ if !checkincache.GetFeatureFlag(client.KVStore, "vanilla_tor_enabled") {
+ t.Fatal("expected to see true here")
+ }
+ checkinrawdata, err := client.KVStore.Get(checkincache.CheckInFlagsState)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log(string(checkinrawdata))
+ var checkindata map[string]any
+ if err := json.Unmarshal(checkinrawdata, &checkindata); err != nil {
+ t.Fatal(err)
+ }
+ t.Log(checkindata)
+ })
+
+ t.Run("does not fail if the key-value store fails", func(t *testing.T) {
+ // define our expectations
+ //
+ // note: when calling the checkincache.GetFeatureFlag function, we will
+ // get a false result if there's no corresponding information into the
+ // key-value store. So, we're setting true here to check we can read from
+ // the key-value store values that are different from the default.
+ expect := &model.OOAPICheckInResult{
+ Conf: model.OOAPICheckInResultConfig{
+ Features: map[string]bool{
+ "torsf_enabled": true,
+ "vanilla_tor_enabled": true,
+ },
+ TestHelpers: map[string][]model.OOAPIService{
+ "web-connectivity": {{
+ Address: "https://0.th.ooni.org/",
+ Type: "https",
+ }},
+ },
+ },
+ ProbeASN: "AS30722",
+ ProbeCC: "US",
+ Tests: model.OOAPICheckInResultNettests{
+ WebConnectivity: &model.OOAPICheckInInfoWebConnectivity{
+ ReportID: "20240424T134700Z_webconnectivity_IT_30722_n1_q5N5YSTWEqHYDo9v",
+ URLs: []model.OOAPIURLInfo{{
+ CategoryCode: "NEWS",
+ CountryCode: "IT",
+ URL: "https://www.example.com/",
+ }},
+ },
+ },
+ UTCTime: time.Date(2022, 11, 22, 1, 2, 3, 0, time.UTC),
+ V: 1,
+ }
+
+ // create a local server that responds with the expectation
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ runtimex.Assert(r.Method == http.MethodPost, "invalid method")
+ runtimex.Assert(r.URL.Path == "/api/v1/check-in", "invalid URL path")
+ rawreqbody := runtimex.Try1(netxlite.ReadAllContext(r.Context(), r.Body))
+ var gotrequest model.OOAPICheckInConfig
+ must.UnmarshalJSON(rawreqbody, &gotrequest)
+ diff := cmp.Diff(config, gotrequest)
+ runtimex.Assert(diff == "", "request mismatch:"+diff)
+ w.Write(must.MarshalJSON(expect))
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // override the key-value store
+ //
+ // note: we create a key-value store that cannot store and we want to check
+ // that (1) still the check-in call succeeds and (2) however, we have not
+ // store anything interesting inside the check-in call.
+ goodKvStore := client.KVStore
+ brokenKvStore := &mocks.KeyValueStore{
+ MockGet: goodKvStore.Get,
+ MockSet: func(key string, value []byte) error {
+ return errors.New("mocked error")
+ },
+ }
+ client.KVStore = brokenKvStore
+
+ // call the API
+ result, err := client.CheckIn(context.Background(), config)
+
+ // we do not expect to see an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // we expect to see exactly what the server sent
+ if diff := cmp.Diff(expect, result); diff != "" {
+ t.Fatal(diff)
+ }
+
+ // make sure we are still getting the default values here
+ if checkincache.GetFeatureFlag(client.KVStore, "torsf_enabled") {
+ t.Fatal("expected to see false here")
+ }
+ if checkincache.GetFeatureFlag(client.KVStore, "vanilla_tor_enabled") {
+ t.Fatal("expected to see false here")
+ }
+ checkinrawdata, err := client.KVStore.Get(checkincache.CheckInFlagsState)
+ if !errors.Is(err, kvstore.ErrNoSuchKey) {
+ t.Fatal("unexpected error", err)
+ }
+ if len(checkinrawdata) != 0 {
+ t.Fatal("expected zero-length data here")
+ }
+ })
}
diff --git a/pkg/probeservices/collector.go b/pkg/probeservices/collector.go
index 787ae8c60..a75b5eeaf 100644
--- a/pkg/probeservices/collector.go
+++ b/pkg/probeservices/collector.go
@@ -8,7 +8,9 @@ import (
"reflect"
"sync"
+ "github.com/ooni/probe-engine/pkg/httpclientx"
"github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/urlx"
)
var (
@@ -58,10 +60,27 @@ func (c Client) OpenReport(ctx context.Context, rt model.OOAPIReportTemplate) (R
if rt.Format != model.OOAPIReportDefaultFormat {
return nil, ErrUnsupportedFormat
}
- var cor model.OOAPICollectorOpenResponse
- if err := c.APIClientTemplate.WithBodyLogging().Build().PostJSON(ctx, "/report", rt, &cor); err != nil {
+
+ URL, err := urlx.ResolveReference(c.BaseURL, "/report", "")
+ if err != nil {
+ return nil, err
+ }
+
+ cor, err := httpclientx.PostJSON[model.OOAPIReportTemplate, *model.OOAPICollectorOpenResponse](
+ ctx,
+ httpclientx.NewEndpoint(URL).WithHostOverride(c.Host),
+ rt,
+ &httpclientx.Config{
+ Client: c.HTTPClient,
+ Logger: c.Logger,
+ UserAgent: c.UserAgent,
+ },
+ )
+
+ if err != nil {
return nil, err
}
+
for _, format := range cor.SupportedFormats {
if format == "json" {
return &reportChan{ID: cor.ReportID, client: c, tmpl: rt}, nil
@@ -83,18 +102,38 @@ func (r reportChan) CanSubmit(m *model.Measurement) bool {
// submitted. Otherwise, we'll set the report ID to the empty
// string, so that you know which measurements weren't submitted.
func (r reportChan) SubmitMeasurement(ctx context.Context, m *model.Measurement) error {
- var updateResponse model.OOAPICollectorUpdateResponse
+ // TODO(bassosimone): do we need to prevent measurement submission
+ // if the measurement isn't consistent with the orig template?
+
m.ReportID = r.ID
- err := r.client.APIClientTemplate.WithBodyLogging().Build().PostJSON(
- ctx, fmt.Sprintf("/report/%s", r.ID), model.OOAPICollectorUpdateRequest{
- Format: "json",
- Content: m,
- }, &updateResponse,
+
+ URL, err := urlx.ResolveReference(r.client.BaseURL, fmt.Sprintf("/report/%s", r.ID), "")
+ if err != nil {
+ return err
+ }
+
+ apiReq := model.OOAPICollectorUpdateRequest{
+ Format: "json",
+ Content: m,
+ }
+
+ updateResponse, err := httpclientx.PostJSON[
+ model.OOAPICollectorUpdateRequest, *model.OOAPICollectorUpdateResponse](
+ ctx,
+ httpclientx.NewEndpoint(URL).WithHostOverride(r.client.Host),
+ apiReq,
+ &httpclientx.Config{
+ Client: r.client.HTTPClient,
+ Logger: r.client.Logger,
+ UserAgent: r.client.UserAgent,
+ },
)
+
if err != nil {
m.ReportID = ""
return err
}
+
// TODO(bassosimone): we should use the session logger here but for now this stopgap
// solution will allow observing the measurement URL for CLI users.
log.Printf("Measurement URL: https://explorer.ooni.org/m/%s", updateResponse.MeasurementUID)
diff --git a/pkg/probeservices/collector_test.go b/pkg/probeservices/collector_test.go
index 2ab684a1e..e23ba6363 100644
--- a/pkg/probeservices/collector_test.go
+++ b/pkg/probeservices/collector_test.go
@@ -4,23 +4,20 @@ import (
"context"
"errors"
"net/http"
- "net/http/httptest"
- "os"
+ "net/url"
"reflect"
- "strings"
"sync"
"testing"
"github.com/apex/log"
"github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/mocks"
"github.com/ooni/probe-engine/pkg/model"
"github.com/ooni/probe-engine/pkg/netxlite"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/testingx"
)
-type fakeTestKeys struct {
- Failure *string `json:"failure"`
-}
-
func makeMeasurement(rt model.OOAPIReportTemplate, ID string) model.Measurement {
return model.Measurement{
DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion,
@@ -36,14 +33,29 @@ func makeMeasurement(rt model.OOAPIReportTemplate, ID string) model.Measurement
ResolverNetworkName: "Google LLC",
SoftwareName: rt.SoftwareName,
SoftwareVersion: rt.SoftwareVersion,
- TestKeys: fakeTestKeys{Failure: nil},
+ TestKeys: map[string]any{"failure": nil},
TestName: rt.TestName,
TestStartTime: rt.TestStartTime,
TestVersion: rt.TestVersion,
}
}
+func newReportTemplateForTesting() model.OOAPIReportTemplate {
+ return model.OOAPIReportTemplate{
+ DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion,
+ Format: model.OOAPIReportDefaultFormat,
+ ProbeASN: "AS117",
+ ProbeCC: "IT",
+ SoftwareName: "ooniprobe-engine",
+ SoftwareVersion: "0.1.0",
+ TestName: "dummy",
+ TestStartTime: "2019-10-28 12:51:06",
+ TestVersion: "0.1.0",
+ }
+}
+
func TestNewReportTemplate(t *testing.T) {
+ // create a measurement with minimal fields
m := &model.Measurement{
ProbeASN: "AS117",
ProbeCC: "IT",
@@ -53,238 +65,616 @@ func TestNewReportTemplate(t *testing.T) {
TestStartTime: "2019-10-28 12:51:06",
TestVersion: "0.1.0",
}
+
+ // convert the measurement to a report template
rt := NewReportTemplate(m)
- expect := model.OOAPIReportTemplate{
- DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion,
- Format: model.OOAPIReportDefaultFormat,
- ProbeASN: "AS117",
- ProbeCC: "IT",
- SoftwareName: "ooniprobe-engine",
- SoftwareVersion: "0.1.0",
- TestName: "dummy",
- TestStartTime: "2019-10-28 12:51:06",
- TestVersion: "0.1.0",
- }
+
+ // define expectations
+ expect := newReportTemplateForTesting()
+
+ // make sure they are equal
if diff := cmp.Diff(expect, rt); diff != "" {
t.Fatal(diff)
}
}
func TestReportLifecycle(t *testing.T) {
- if testing.Short() {
- t.Skip("skip test in short mode")
- }
+ // First, let's check whether we can get a response from the real OONI backend.
+ t.Run("is working as intended with the real backend", func(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skip test in short mode")
+ }
- ctx := context.Background()
- template := model.OOAPIReportTemplate{
- DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion,
- Format: model.OOAPIReportDefaultFormat,
- ProbeASN: "AS0",
- ProbeCC: "ZZ",
- SoftwareName: "ooniprobe-engine",
- SoftwareVersion: "0.1.0",
- TestName: "dummy",
- TestStartTime: "2019-10-28 12:51:06",
- TestVersion: "0.1.0",
- }
- client := newclient()
- report, err := client.OpenReport(ctx, template)
- if err != nil {
- t.Fatal(err)
- }
- measurement := makeMeasurement(template, report.ReportID())
- if report.CanSubmit(&measurement) != true {
- t.Fatal("report should be able to submit this measurement")
- }
- if err = report.SubmitMeasurement(ctx, &measurement); err != nil {
- t.Fatal(err)
- }
- if measurement.ReportID != report.ReportID() {
- t.Fatal("report ID mismatch")
- }
-}
+ // create the client and report template for testing
+ client := newclient()
+ template := newReportTemplateForTesting()
-func TestReportLifecycleWrongExperiment(t *testing.T) {
- if testing.Short() {
- t.Skip("skip test in short mode")
- }
+ // open the report
+ report, err := client.OpenReport(context.Background(), template)
- ctx := context.Background()
- template := model.OOAPIReportTemplate{
- DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion,
- Format: model.OOAPIReportDefaultFormat,
- ProbeASN: "AS0",
- ProbeCC: "ZZ",
- SoftwareName: "ooniprobe-engine",
- SoftwareVersion: "0.1.0",
- TestName: "dummy",
- TestStartTime: "2019-10-28 12:51:06",
- TestVersion: "0.1.0",
- }
- client := newclient()
- report, err := client.OpenReport(ctx, template)
- if err != nil {
- t.Fatal(err)
- }
- measurement := makeMeasurement(template, report.ReportID())
- measurement.TestName = "antani"
- if report.CanSubmit(&measurement) != false {
- t.Fatal("report should not be able to submit this measurement")
- }
-}
+ // we expect to be able to open the report
+ if err != nil {
+ t.Fatal(err)
+ }
-func TestOpenReportInvalidDataFormatVersion(t *testing.T) {
- ctx := context.Background()
- template := model.OOAPIReportTemplate{
- DataFormatVersion: "0.1.0",
- Format: model.OOAPIReportDefaultFormat,
- ProbeASN: "AS0",
- ProbeCC: "ZZ",
- SoftwareName: "ooniprobe-engine",
- SoftwareVersion: "0.1.0",
- TestName: "dummy",
- TestStartTime: "2019-10-28 12:51:06",
- TestVersion: "0.1.0",
- }
- client := newclient()
- report, err := client.OpenReport(ctx, template)
- if !errors.Is(err, ErrUnsupportedDataFormatVersion) {
- t.Fatal("not the error we expected")
- }
- if report != nil {
- t.Fatal("expected a nil report here")
- }
-}
+ // make a measurement out of the report template
+ measurement := makeMeasurement(template, report.ReportID())
-func TestOpenReportInvalidFormat(t *testing.T) {
- ctx := context.Background()
- template := model.OOAPIReportTemplate{
- DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion,
- Format: "yaml",
- ProbeASN: "AS0",
- ProbeCC: "ZZ",
- SoftwareName: "ooniprobe-engine",
- SoftwareVersion: "0.1.0",
- TestName: "dummy",
- TestStartTime: "2019-10-28 12:51:06",
- TestVersion: "0.1.0",
- }
- client := newclient()
- report, err := client.OpenReport(ctx, template)
- if !errors.Is(err, ErrUnsupportedFormat) {
- t.Fatal("not the error we expected")
- }
- if report != nil {
- t.Fatal("expected a nil report here")
- }
-}
+ // make sure we can submit this measurement within the report, which we really
+ // expect to succeed since we created the measurement from the template
+ if report.CanSubmit(&measurement) != true {
+ t.Fatal("report should be able to submit this measurement")
+ }
-func TestJSONAPIClientCreateFailure(t *testing.T) {
- ctx := context.Background()
- template := model.OOAPIReportTemplate{
- DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion,
- Format: model.OOAPIReportDefaultFormat,
- ProbeASN: "AS0",
- ProbeCC: "ZZ",
- SoftwareName: "ooniprobe-engine",
- SoftwareVersion: "0.1.0",
- TestName: "dummy",
- TestStartTime: "2019-10-28 12:51:06",
- TestVersion: "0.1.0",
- }
- client := newclient()
- client.BaseURL = "\t" // breaks the URL parser
- report, err := client.OpenReport(ctx, template)
- if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
- t.Fatal("not the error we expected")
- }
- if report != nil {
- t.Fatal("expected a nil report here")
- }
-}
+ // attempt to submit the measurement to the backend, which should succeed
+ // since we've just opened a report for it
+ if err = report.SubmitMeasurement(context.Background(), &measurement); err != nil {
+ t.Fatal(err)
+ }
-func TestOpenResponseNoJSONSupport(t *testing.T) {
- server := httptest.NewServer(
- http.HandlerFunc(func(writer http.ResponseWriter, req *http.Request) {
- writer.Write([]byte(`{"ID":"abc","supported_formats":["yaml"]}`))
- }),
- )
- defer server.Close()
- ctx := context.Background()
- template := model.OOAPIReportTemplate{
- DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion,
- Format: model.OOAPIReportDefaultFormat,
- ProbeASN: "AS0",
- ProbeCC: "ZZ",
- SoftwareName: "ooniprobe-engine",
- SoftwareVersion: "0.1.0",
- TestName: "dummy",
- TestStartTime: "2019-10-28 12:51:06",
- TestVersion: "0.1.0",
- }
- client := newclient()
- client.BaseURL = server.URL
- report, err := client.OpenReport(ctx, template)
- if !errors.Is(err, ErrJSONFormatNotSupported) {
- t.Fatal("expected an error here")
- }
- if report != nil {
- t.Fatal("expected a nil report here")
- }
-}
+ // additionally make sure we edited the measurement report ID to
+ // contain the correct report ID used to submit
+ if measurement.ReportID != report.ReportID() {
+ t.Fatal("report ID mismatch")
+ }
+ })
-func TestEndToEnd(t *testing.T) {
- server := httptest.NewServer(
- http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if r.RequestURI == "/report" {
- w.Write([]byte(`{"report_id":"_id","supported_formats":["json"]}`))
- return
- }
- if r.RequestURI == "/report/_id" {
- data, err := netxlite.ReadAllContext(r.Context(), r.Body)
- if err != nil {
- panic(err)
- }
- sdata, err := os.ReadFile("testdata/collector-expected.jsonl")
- if err != nil {
- panic(err)
- }
- if diff := cmp.Diff(data, sdata); diff != "" {
- panic(diff)
- }
- w.Write([]byte(`{"measurement_id":"e00c584e6e9e5326"}`))
- return
+ // Now let's construct a test server that returns a valid response and try
+ // to communicate with such a test server successfully and with errors
+
+ t.Run("is working as intended with a local test server", func(t *testing.T) {
+ // create state for emulating the OONI backend
+ state := &testingx.OONICollector{}
+
+ // expose the state via HTTP
+ srv := testingx.MustNewHTTPServer(state)
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client so we speak with our local server rather than the true backend
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // create the report template used for testing
+ template := newReportTemplateForTesting()
+
+ // open the report
+ report, err := client.OpenReport(context.Background(), template)
+
+ // we expect to be able to open the report
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make a measurement out of the report template
+ measurement := makeMeasurement(template, report.ReportID())
+
+ // make sure we can submit this measurement within the report, which we really
+ // expect to succeed since we created the measurement from the template
+ if report.CanSubmit(&measurement) != true {
+ t.Fatal("report should be able to submit this measurement")
+ }
+
+ // attempt to submit the measurement to the backend, which should succeed
+ // since we've just opened a report for it
+ if err = report.SubmitMeasurement(context.Background(), &measurement); err != nil {
+ t.Fatal(err)
+ }
+
+ // additionally make sure we edited the measurement report ID to
+ // contain the correct report ID used to submit
+ if measurement.ReportID != report.ReportID() {
+ t.Fatal("report ID mismatch")
+ }
+ })
+
+ t.Run("we can use cloudfronting", func(t *testing.T) {
+ // create state for emulating the OONI backend
+ state := &testingx.OONICollector{}
+
+ // expose the state via HTTP
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ runtimex.Assert(r.Host == "www.cloudfront.com", "invalid r.Host")
+ state.ServeHTTP(w, r)
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // make sure we're using cloudfronting
+ client.Host = "www.cloudfront.com"
+
+ // override the HTTP client so we speak with our local server rather than the true backend
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // create the report template used for testing
+ template := newReportTemplateForTesting()
+
+ // open the report
+ report, err := client.OpenReport(context.Background(), template)
+
+ // we expect to be able to open the report
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make a measurement out of the report template
+ measurement := makeMeasurement(template, report.ReportID())
+
+ // make sure we can submit this measurement within the report, which we really
+ // expect to succeed since we created the measurement from the template
+ if report.CanSubmit(&measurement) != true {
+ t.Fatal("report should be able to submit this measurement")
+ }
+
+ // attempt to submit the measurement to the backend, which should succeed
+ // since we've just opened a report for it
+ if err = report.SubmitMeasurement(context.Background(), &measurement); err != nil {
+ t.Fatal(err)
+ }
+
+ // additionally make sure we edited the measurement report ID to
+ // contain the correct report ID used to submit
+ if measurement.ReportID != report.ReportID() {
+ t.Fatal("report ID mismatch")
+ }
+ })
+
+ t.Run("opening a report fails with an error when the connection is reset", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // create the report template used for testing
+ template := newReportTemplateForTesting()
+
+ // open the report
+ report, err := client.OpenReport(context.Background(), template)
+
+ // we do expect an error
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect a nil report here
+ if report != nil {
+ t.Fatal("expected a nil report here")
+ }
+ })
+
+ t.Run("opening a report fails with an error when the response is not JSON parsable", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{`))
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // create the report template used for testing
+ template := newReportTemplateForTesting()
+
+ // open the report
+ report, err := client.OpenReport(context.Background(), template)
+
+ // we do expect an error
+ if err == nil || err.Error() != "unexpected end of JSON input" {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect a nil report here
+ if report != nil {
+ t.Fatal("expected a nil report here")
+ }
+ })
+
+ t.Run("opening a report correctly handles the case where the URL is unparseable", func(t *testing.T) {
+ // create a probeservices client
+ client := newclient()
+
+ // override the URL to be unparseable
+ client.BaseURL = "\t\t\t"
+
+ // create the report template used for testing
+ template := newReportTemplateForTesting()
+
+ // open the report
+ report, err := client.OpenReport(context.Background(), template)
+
+ // we do expect an error
+ if err == nil || err.Error() != `parse "\t\t\t": net/url: invalid control character in URL` {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect a nil report here
+ if report != nil {
+ t.Fatal("expected a nil report here")
+ }
+ })
+
+ t.Run("updating a report fails with an error when the connection is reset", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // create the report template used for testing
+ template := newReportTemplateForTesting()
+
+ // create the reportChan
+ rc := reportChan{
+ ID: "xxx-xxx-xxx-xxx",
+ client: *client,
+ tmpl: template,
+ }
+
+ // update the report
+ err := rc.SubmitMeasurement(context.Background(), &model.Measurement{})
+
+ // we do expect an error
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+ })
+
+ t.Run("updating a report fails with an error when the response is not JSON parsable", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{`))
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // create the report template used for testing
+ template := newReportTemplateForTesting()
+
+ // create the reportChan
+ rc := reportChan{
+ ID: "xxx-xxx-xxx-xxx",
+ client: *client,
+ tmpl: template,
+ }
+
+ // update the report
+ err := rc.SubmitMeasurement(context.Background(), &model.Measurement{})
+
+ // we do expect an error
+ if err == nil || err.Error() != "unexpected end of JSON input" {
+ t.Fatal("unexpected error", err)
+ }
+ })
+
+ t.Run("updating a report correctly handles the case where the URL is unparseable", func(t *testing.T) {
+ // create a probeservices client
+ client := newclient()
+
+ // override the URL to be unparseable
+ client.BaseURL = "\t\t\t"
+
+ // create the report template used for testing
+ template := newReportTemplateForTesting()
+
+ // create the reportChan
+ rc := reportChan{
+ ID: "xxx-xxx-xxx-xxx",
+ client: *client,
+ tmpl: template,
+ }
+
+ // update the report
+ err := rc.SubmitMeasurement(context.Background(), &model.Measurement{})
+
+ // we do expect an error
+ if err == nil || err.Error() != `parse "\t\t\t": net/url: invalid control character in URL` {
+ t.Fatal("unexpected error", err)
+ }
+ })
+
+ t.Run("we cannot open a report with invalid data format version", func(t *testing.T) {
+ // create client and default template
+ client := newclient()
+ template := newReportTemplateForTesting()
+
+ // set a wrong data format version to test whether OpenReport would fail.
+ template.DataFormatVersion = "0.1.0"
+
+ // attempt to open the report
+ report, err := client.OpenReport(context.Background(), template)
+
+ // we expect the error to indicate the data format version is wrong
+ if !errors.Is(err, ErrUnsupportedDataFormatVersion) {
+ t.Fatal("not the error we expected", err)
+ }
+
+ // ancillary check: make sure report is nil
+ if report != nil {
+ t.Fatal("expected a nil report here")
+ }
+ })
+
+ t.Run("we cannot open a report with invalid data serialization format", func(t *testing.T) {
+ // create client and default template
+ client := newclient()
+ template := newReportTemplateForTesting()
+
+ // set a wrong data serialization format to test whether OpenReport would fail.
+ template.Format = "yaml"
+
+ // attempt to open the report
+ report, err := client.OpenReport(context.Background(), template)
+
+ // we expect the error to indicate the data format version is wrong
+ if !errors.Is(err, ErrUnsupportedFormat) {
+ t.Fatal("not the error we expected", err)
+ }
+
+ // ancillary check: make sure report is nil
+ if report != nil {
+ t.Fatal("expected a nil report here")
+ }
+ })
+
+ t.Run("we cannot open a report if the server doesn't support JSON", func(t *testing.T) {
+ // create state for emulating the OONI backend
+ state := &testingx.OONICollector{}
+
+ // override the open report response to claim we only support YAML
+ state.EditOpenReportResponse = func(resp *model.OOAPICollectorOpenResponse) {
+ resp.SupportedFormats = []string{"yaml"}
+ }
+
+ // expose the state via HTTP
+ srv := testingx.MustNewHTTPServer(state)
+ defer srv.Close()
+
+ // create template and client
+ template := newReportTemplateForTesting()
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // attempt to open a report
+ report, err := client.OpenReport(context.Background(), template)
+
+ if !errors.Is(err, ErrJSONFormatNotSupported) {
+ t.Fatal("expected an error here")
+ }
+ if report != nil {
+ t.Fatal("expected a nil report here")
+ }
+ })
+
+ t.Run("we cannot submit using the wrong experiment name", func(t *testing.T) {
+ // create state for emulating the OONI backend
+ state := &testingx.OONICollector{}
+
+ // expose the state via HTTP
+ srv := testingx.MustNewHTTPServer(state)
+ defer srv.Close()
+
+ // create template and client
+ template := newReportTemplateForTesting()
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // attempt to open a report
+ report, err := client.OpenReport(context.Background(), template)
+
+ // we expect to see a success here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // create a measurement to submit
+ measurement := makeMeasurement(template, report.ReportID())
+
+ // set the wrong test name to see if we can actually submit it
+ measurement.TestName = "antani"
+
+ // we expect to not be able to submit the measurement
+ if report.CanSubmit(&measurement) != false {
+ t.Fatal("report should not be able to submit this measurement")
+ }
+ })
+
+ t.Run("end-to-end test where we verify requests and responses", func(t *testing.T) {
+ // create the template
+ template := newReportTemplateForTesting()
+
+ // define the reportID we'll force
+ reportID := "xxx-xx-xx-xxx"
+
+ // create the measurement
+ measurement := makeMeasurement(template, reportID)
+
+ // create state for emulating the OONI backend
+ state := &testingx.OONICollector{}
+
+ // make sure we receive the exact report template we're sending
+ state.ValidateReportTemplate = func(rt *model.OOAPIReportTemplate) error {
+ if diff := cmp.Diff(&template, rt); diff != "" {
+ return errors.New(diff)
}
- if r.RequestURI == "/report/_id/close" {
- w.Write([]byte(`{}`))
- return
+ return nil
+ }
+
+ // make sure we override the report ID
+ state.EditOpenReportResponse = func(resp *model.OOAPICollectorOpenResponse) {
+ resp.ReportID = reportID
+ }
+
+ // make sure we receive the exact measurement we're sending
+ state.ValidateMeasurement = func(meas *model.Measurement) error {
+ if diff := cmp.Diff(&measurement, meas); diff != "" {
+ return errors.New(diff)
}
- panic(r.RequestURI)
- }),
- )
- defer server.Close()
- ctx := context.Background()
- template := model.OOAPIReportTemplate{
- DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion,
- Format: model.OOAPIReportDefaultFormat,
- ProbeASN: "AS0",
- ProbeCC: "ZZ",
- SoftwareName: "ooniprobe-engine",
- SoftwareVersion: "0.1.0",
- TestName: "dummy",
- TestStartTime: "2018-11-01 15:33:17",
- TestVersion: "0.1.0",
- }
- client := newclient()
- client.BaseURL = server.URL
- report, err := client.OpenReport(ctx, template)
- if err != nil {
- t.Fatal(err)
- }
- measurement := makeMeasurement(template, report.ReportID())
- if err = report.SubmitMeasurement(ctx, &measurement); err != nil {
- t.Fatal(err)
- }
+ return nil
+ }
+
+ // define the measurement UID to expect
+ measurementUID := "x-y-z-a-b-c"
+
+ // make sure we override the measurement UID
+ state.EditUpdateResponse = func(resp *model.OOAPICollectorUpdateResponse) {
+ resp.MeasurementUID = measurementUID
+ }
+
+ // expose the state via HTTP
+ srv := testingx.MustNewHTTPServer(state)
+ defer srv.Close()
+
+ // create client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // attempt to open a report
+ report, err := client.OpenReport(context.Background(), template)
+
+ // we expect to be successful here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure the report ID is correct
+ if report.ReportID() != reportID {
+ t.Fatal("got unexpected reportID value", report.ReportID())
+ }
+
+ // make sure we can submit this measurement within the report, which we really
+ // expect to succeed since we created the measurement from the template
+ if report.CanSubmit(&measurement) != true {
+ t.Fatal("report should be able to submit this measurement")
+ }
+
+ // attempt to submit the measurement to the backend, which should succeed
+ // since we've just opened a report for it
+ if err = report.SubmitMeasurement(context.Background(), &measurement); err != nil {
+ t.Fatal(err)
+ }
+
+ // additionally make sure we edited the measurement report ID to
+ // contain the correct report ID used to submit
+ if measurement.ReportID != report.ReportID() {
+ t.Fatal("report ID mismatch")
+ }
+ })
}
type RecordingReportChannel struct {
@@ -338,68 +728,7 @@ func (rro *RecordingReportOpener) OpenReport(
return rrc, nil
}
-func TestOpenReportCancelledContext(t *testing.T) {
- ctx, cancel := context.WithCancel(context.Background())
- cancel() // immediately abort
- template := model.OOAPIReportTemplate{
- DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion,
- Format: model.OOAPIReportDefaultFormat,
- ProbeASN: "AS0",
- ProbeCC: "ZZ",
- SoftwareName: "ooniprobe-engine",
- SoftwareVersion: "0.1.0",
- TestName: "dummy",
- TestStartTime: "2019-10-28 12:51:06",
- TestVersion: "0.1.0",
- }
- client := newclient()
- report, err := client.OpenReport(ctx, template)
- if !errors.Is(err, context.Canceled) {
- t.Fatal("not the error we expected")
- }
- if report != nil {
- t.Fatal("expected nil report here")
- }
-}
-
-func TestSubmitMeasurementCancelledContext(t *testing.T) {
- if testing.Short() {
- t.Skip("skip test in short mode")
- }
-
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
- template := model.OOAPIReportTemplate{
- DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion,
- Format: model.OOAPIReportDefaultFormat,
- ProbeASN: "AS0",
- ProbeCC: "ZZ",
- SoftwareName: "ooniprobe-engine",
- SoftwareVersion: "0.1.0",
- TestName: "dummy",
- TestStartTime: "2019-10-28 12:51:06",
- TestVersion: "0.1.0",
- }
- client := newclient()
- report, err := client.OpenReport(ctx, template)
- if err != nil {
- t.Fatal(err)
- }
- measurement := makeMeasurement(template, report.ReportID())
- if report.CanSubmit(&measurement) != true {
- t.Fatal("report should be able to submit this measurement")
- }
- cancel() // cause submission to fail
- err = report.SubmitMeasurement(ctx, &measurement)
- if !errors.Is(err, context.Canceled) {
- t.Fatalf("not the error we expected: %+v", err)
- }
- if measurement.ReportID != "" {
- t.Fatal("report ID should be empty here")
- }
-}
-
-func makeMeasurementWithoutTemplate(failure, testName string) *model.Measurement {
+func makeMeasurementWithoutTemplate(testName string) *model.Measurement {
return &model.Measurement{
DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion,
ID: "bdd20d7a-bba5-40dd-a111-9863d7908572",
@@ -414,7 +743,7 @@ func makeMeasurementWithoutTemplate(failure, testName string) *model.Measurement
ResolverNetworkName: "Google LLC",
SoftwareName: "miniooni",
SoftwareVersion: "0.1.0-dev",
- TestKeys: fakeTestKeys{Failure: &failure},
+ TestKeys: map[string]any{"failure": nil},
TestName: testName,
TestStartTime: "2018-11-01 15:33:17",
TestVersion: "0.1.0",
@@ -425,15 +754,15 @@ func TestSubmitterLifecyle(t *testing.T) {
rro := &RecordingReportOpener{}
submitter := NewSubmitter(rro, log.Log)
ctx := context.Background()
- m1 := makeMeasurementWithoutTemplate("antani", "example")
+ m1 := makeMeasurementWithoutTemplate("example")
if err := submitter.Submit(ctx, m1); err != nil {
t.Fatal(err)
}
- m2 := makeMeasurementWithoutTemplate("mascetti", "example")
+ m2 := makeMeasurementWithoutTemplate("example")
if err := submitter.Submit(ctx, m2); err != nil {
t.Fatal(err)
}
- m3 := makeMeasurementWithoutTemplate("antani", "example_extended")
+ m3 := makeMeasurementWithoutTemplate("example_extended")
if err := submitter.Submit(ctx, m3); err != nil {
t.Fatal(err)
}
@@ -453,15 +782,15 @@ func TestSubmitterCannotOpenNewChannel(t *testing.T) {
submitter := NewSubmitter(rro, log.Log)
ctx, cancel := context.WithCancel(context.Background())
cancel() // fail immediately
- m1 := makeMeasurementWithoutTemplate("antani", "example")
+ m1 := makeMeasurementWithoutTemplate("example")
if err := submitter.Submit(ctx, m1); !errors.Is(err, context.Canceled) {
t.Fatal("not the error we expected")
}
- m2 := makeMeasurementWithoutTemplate("mascetti", "example")
+ m2 := makeMeasurementWithoutTemplate("example")
if err := submitter.Submit(ctx, m2); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
- m3 := makeMeasurementWithoutTemplate("antani", "example_extended")
+ m3 := makeMeasurementWithoutTemplate("example_extended")
if err := submitter.Submit(ctx, m3); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
diff --git a/pkg/probeservices/login.go b/pkg/probeservices/login.go
index e5068c313..fd9c3dc75 100644
--- a/pkg/probeservices/login.go
+++ b/pkg/probeservices/login.go
@@ -3,7 +3,9 @@ package probeservices
import (
"context"
+ "github.com/ooni/probe-engine/pkg/httpclientx"
"github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/urlx"
)
// MaybeLogin performs login if necessary
@@ -17,11 +19,27 @@ func (c Client) MaybeLogin(ctx context.Context) error {
return ErrNotRegistered
}
c.LoginCalls.Add(1)
- var auth model.OOAPILoginAuth
- if err := c.APIClientTemplate.Build().PostJSON(
- ctx, "/api/v1/login", *creds, &auth); err != nil {
+
+ URL, err := urlx.ResolveReference(c.BaseURL, "/api/v1/login", "")
+ if err != nil {
+ return err
+ }
+
+ auth, err := httpclientx.PostJSON[*model.OOAPILoginCredentials, *model.OOAPILoginAuth](
+ ctx,
+ httpclientx.NewEndpoint(URL).WithHostOverride(c.Host),
+ creds,
+ &httpclientx.Config{
+ Client: c.HTTPClient,
+ Logger: model.DiscardLogger,
+ UserAgent: c.UserAgent,
+ },
+ )
+
+ if err != nil {
return err
}
+
state.Expire = auth.Expire
state.Token = auth.Token
return c.StateFile.Set(state)
diff --git a/pkg/probeservices/login_test.go b/pkg/probeservices/login_test.go
index a2bf3345f..00465bf82 100644
--- a/pkg/probeservices/login_test.go
+++ b/pkg/probeservices/login_test.go
@@ -2,75 +2,320 @@ package probeservices
import (
"context"
+ "errors"
+ "net/http"
+ "net/url"
"testing"
"time"
+
+ "github.com/ooni/probe-engine/pkg/mocks"
+ "github.com/ooni/probe-engine/pkg/netxlite"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/testingx"
)
func TestMaybeLogin(t *testing.T) {
- t.Run("when we already have a token", func(t *testing.T) {
+ // First, let's check whether we can get a response from the real OONI backend.
+ t.Run("is working as intended with the real OONI backend", func(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skip test in short mode")
+ }
+
+ // create client
clnt := newclient()
- state := State{
- Expire: time.Now().Add(time.Hour),
- Token: "xx-xxx-x-xxxx",
+
+ // we need to register first because we don't have state yet
+ if err := clnt.MaybeRegister(context.Background(), MetadataFixture()); err != nil {
+ t.Fatal(err)
}
- if err := clnt.StateFile.Set(state); err != nil {
+
+ // now we try to login and get a token
+ if err := clnt.MaybeLogin(context.Background()); err != nil {
t.Fatal(err)
}
- ctx := context.Background()
- if err := clnt.MaybeLogin(ctx); err != nil {
+
+ // do this again, and later on we'll verify that we
+ // did actually issue just a single login call
+ if err := clnt.MaybeLogin(context.Background()); err != nil {
t.Fatal(err)
}
+
+ // make sure we did call login just once: the second call
+ // should not invoke login because we have good state
+ if clnt.LoginCalls.Load() != 1 {
+ t.Fatal("called login API too many times")
+ }
})
- t.Run("when we have already registered", func(t *testing.T) {
+ // Now let's construct a test server that returns a valid response and try
+ // to communicate with such a test server successfully and with errors
+
+ t.Run("is working as intended with a local test server", func(t *testing.T) {
+ // create state for emulating the OONI backend
+ state := &testingx.OONIBackendWithLoginFlow{}
+
+ // expose the state via HTTP
+ srv := testingx.MustNewHTTPServer(state.NewMux())
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client so we speak with our local server rather than the true backend
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // we need to register first because we don't have state yet
+ if err := client.MaybeRegister(context.Background(), MetadataFixture()); err != nil {
+ t.Fatal(err)
+ }
+
+ // now we try to login and get a token
+ if err := client.MaybeLogin(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+
+ // do this again, and later on we'll verify that we
+ // did actually issue just a single login call
+ if err := client.MaybeLogin(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we did call login just once: the second call
+ // should not invoke login because we have good state
+ if client.LoginCalls.Load() != 1 {
+ t.Fatal("called login API too many times")
+ }
+ })
+
+ t.Run("we can use cloudfronting", func(t *testing.T) {
+ // create state for emulating the OONI backend
+ state := &testingx.OONIBackendWithLoginFlow{}
+ mux := state.NewMux()
+
+ // expose the state via HTTP
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ runtimex.Assert(r.Host == "www.cloudfront.com", "invalid r.Host")
+ mux.ServeHTTP(w, r)
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // make sure we're using cloudfronting
+ client.Host = "www.cloudfront.com"
+
+ // override the HTTP client so we speak with our local server rather than the true backend
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // we need to register first because we don't have state yet
+ if err := client.MaybeRegister(context.Background(), MetadataFixture()); err != nil {
+ t.Fatal(err)
+ }
+
+ // now we try to login and get a token
+ if err := client.MaybeLogin(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+
+ // do this again, and later on we'll verify that we
+ // did actually issue just a single login call
+ if err := client.MaybeLogin(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we did call login just once: the second call
+ // should not invoke login because we have good state
+ if client.LoginCalls.Load() != 1 {
+ t.Fatal("called login API too many times")
+ }
+ })
+
+ t.Run("reports an error when the connection is reset", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // we need to convince the client that we're registered first otherwise it will
+ // refuse to send a request to the server and we won't be testing networking
+ runtimex.Try0(client.StateFile.Set(State{
+ ClientID: "ttt-uuu-iii",
+ Expire: time.Time{}, // explicitly empty
+ Password: "xxx-xxx-xxx",
+ Token: "", // explicitly empty
+ }))
+
+ // now we try to login and get a token
+ err := client.MaybeLogin(context.Background())
+
+ // we do expect an error
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure we did call login
+ if client.LoginCalls.Load() != 1 {
+ t.Fatal("called login API the wrong number of times")
+ }
+ })
+
+ t.Run("reports an error when the response is not JSON parsable", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{`))
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // we need to convince the client that we're registered first otherwise it will
+ // refuse to send a request to the server and we won't be testing networking
+ runtimex.Try0(client.StateFile.Set(State{
+ ClientID: "ttt-uuu-iii",
+ Expire: time.Time{}, // explicitly empty
+ Password: "xxx-xxx-xxx",
+ Token: "", // explicitly empty
+ }))
+
+ // now we try to login and get a token
+ err := client.MaybeLogin(context.Background())
+
+ // we do expect an error
+ if err == nil || err.Error() != "unexpected end of JSON input" {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure we did call login
+ if client.LoginCalls.Load() != 1 {
+ t.Fatal("called login API the wrong number of times")
+ }
+ })
+
+ t.Run("when we already have a token", func(t *testing.T) {
clnt := newclient()
+
+ // create a state with valid expire and token
state := State{
- // Explicitly empty to clarify what this test does
+ Expire: time.Now().Add(time.Hour),
+ Token: "xx-xxx-x-xxxx",
}
+
+ // synchronize the state
if err := clnt.StateFile.Set(state); err != nil {
t.Fatal(err)
}
- ctx := context.Background()
- if err := clnt.MaybeLogin(ctx); err == nil {
- t.Fatal("expected an error here")
+
+ // now call login and we expect no error because we should
+ // already have what we need to perform a login
+ if err := clnt.MaybeLogin(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we did not call login
+ if clnt.LoginCalls.Load() != 0 {
+ t.Fatal("called login API the wrong number of times")
}
})
- t.Run("when the API call fails", func(t *testing.T) {
+ t.Run("when we have not registered yet", func(t *testing.T) {
clnt := newclient()
- clnt.BaseURL = "\t\t\t" // causes the code to fail
- state := State{
- ClientID: "xx-xxx-x-xxxx",
- Password: "xx",
- }
+
+ // With explicitly empty state so it's pretty obvioust there's no state
+ state := State{}
+
+ // synchronize the state
if err := clnt.StateFile.Set(state); err != nil {
t.Fatal(err)
}
- ctx := context.Background()
- if err := clnt.MaybeLogin(ctx); err == nil {
- t.Fatal("expected an error here")
+
+ // now try to login and expect to see we've not registered yet
+ if err := clnt.MaybeLogin(context.Background()); !errors.Is(err, ErrNotRegistered) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure we did not call login
+ if clnt.LoginCalls.Load() != 0 {
+ t.Fatal("called login API the wrong number of times")
}
})
-}
-func TestMaybeLoginIdempotent(t *testing.T) {
- if testing.Short() {
- t.Skip("skip test in short mode")
- }
-
- clnt := newclient()
- ctx := context.Background()
- metadata := MetadataFixture()
- if err := clnt.MaybeRegister(ctx, metadata); err != nil {
- t.Fatal(err)
- }
- if err := clnt.MaybeLogin(ctx); err != nil {
- t.Fatal(err)
- }
- if err := clnt.MaybeLogin(ctx); err != nil {
- t.Fatal(err)
- }
- if clnt.LoginCalls.Load() != 1 {
- t.Fatal("called login API too many times")
- }
+ t.Run("correctly handles the case where the URL is unparseable", func(t *testing.T) {
+ // create a probeservices client
+ client := newclient()
+
+ // override the URL to be unparseable
+ client.BaseURL = "\t\t\t"
+
+ // we need to convince the client that we're registered first otherwise it will
+ // refuse to send a request to the server and we won't be testing networking
+ runtimex.Try0(client.StateFile.Set(State{
+ ClientID: "ttt-uuu-iii",
+ Expire: time.Time{}, // explicitly empty
+ Password: "xxx-xxx-xxx",
+ Token: "", // explicitly empty
+ }))
+
+ // now we try to login and get a token
+ err := client.MaybeLogin(context.Background())
+
+ // we do expect an error
+ if err == nil || err.Error() != `parse "\t\t\t": net/url: invalid control character in URL` {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure we did call login
+ if client.LoginCalls.Load() != 1 {
+ t.Fatal("called login API the wrong number of times")
+ }
+ })
}
diff --git a/pkg/probeservices/measurementmeta.go b/pkg/probeservices/measurementmeta.go
index 322dad42e..314feaf92 100644
--- a/pkg/probeservices/measurementmeta.go
+++ b/pkg/probeservices/measurementmeta.go
@@ -1,16 +1,22 @@
package probeservices
+//
+// measurementmeta.go - GET /api/v1/measurement_meta
+//
+
import (
"context"
"net/url"
- "github.com/ooni/probe-engine/pkg/httpx"
+ "github.com/ooni/probe-engine/pkg/httpclientx"
"github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/urlx"
)
// GetMeasurementMeta returns meta information about a measurement.
func (c Client) GetMeasurementMeta(
ctx context.Context, config model.OOAPIMeasurementMetaConfig) (*model.OOAPIMeasurementMeta, error) {
+ // construct the query to use
query := url.Values{}
query.Add("report_id", config.ReportID)
if config.Input != "" {
@@ -19,15 +25,20 @@ func (c Client) GetMeasurementMeta(
if config.Full {
query.Add("full", "true")
}
- var response model.OOAPIMeasurementMeta
- err := (&httpx.APIClientTemplate{
- BaseURL: c.BaseURL,
- HTTPClient: c.HTTPClient,
- Logger: c.Logger,
- UserAgent: c.UserAgent,
- }).WithBodyLogging().Build().GetJSONWithQuery(ctx, "/api/v1/measurement_meta", query, &response)
+
+ // construct the URL to use
+ URL, err := urlx.ResolveReference(c.BaseURL, "/api/v1/measurement_meta", query.Encode())
if err != nil {
return nil, err
}
- return &response, nil
+
+ // get the response
+ return httpclientx.GetJSON[*model.OOAPIMeasurementMeta](
+ ctx,
+ httpclientx.NewEndpoint(URL).WithHostOverride(c.Host),
+ &httpclientx.Config{
+ Client: c.HTTPClient,
+ Logger: c.Logger,
+ UserAgent: c.UserAgent,
+ })
}
diff --git a/pkg/probeservices/measurementmeta_test.go b/pkg/probeservices/measurementmeta_test.go
index 79d0b2e17..b98b5bb58 100644
--- a/pkg/probeservices/measurementmeta_test.go
+++ b/pkg/probeservices/measurementmeta_test.go
@@ -2,116 +2,266 @@ package probeservices
import (
"context"
- "encoding/json"
"errors"
"net/http"
- "sync/atomic"
+ "net/url"
"testing"
+ "time"
- "github.com/apex/log"
- "github.com/ooni/probe-engine/pkg/httpx"
- "github.com/ooni/probe-engine/pkg/kvstore"
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/mocks"
"github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/must"
+ "github.com/ooni/probe-engine/pkg/netxlite"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/testingx"
)
-func TestGetMeasurementMetaWorkingAsIntended(t *testing.T) {
- if testing.Short() {
- t.Skip("skip test in short mode")
- }
+func TestGetMeasurementMeta(t *testing.T) {
- client := Client{
- APIClientTemplate: httpx.APIClientTemplate{
- BaseURL: "https://api.ooni.io/",
- HTTPClient: http.DefaultClient,
- Logger: log.Log,
- UserAgent: "miniooni/0.1.0-dev",
- },
- KVStore: &kvstore.Memory{},
- LoginCalls: &atomic.Int64{},
- RegisterCalls: &atomic.Int64{},
- StateFile: NewStateFile(&kvstore.Memory{}),
- }
+ // This is the configuration we use for both testing with the real API server
+ // and for testing with a local HTTP server for -short tests.
config := model.OOAPIMeasurementMetaConfig{
ReportID: `20201209T052225Z_urlgetter_IT_30722_n1_E1VUhMz08SEkgYFU`,
Full: true,
Input: `https://www.example.org`,
}
- ctx := context.Background()
- mmeta, err := client.GetMeasurementMeta(ctx, config)
- if err != nil {
- t.Fatal(err)
- }
- if mmeta.Anomaly != false {
- t.Fatal("unexpected anomaly value")
- }
- if mmeta.CategoryCode != "" {
- t.Fatal("unexpected category code value")
- }
- if mmeta.Confirmed != false {
- t.Fatal("unexpected confirmed value")
- }
- if mmeta.Failure != true {
- // TODO(bassosimone): this field seems wrong
- t.Fatal("unexpected failure value")
- }
- if mmeta.Input == nil || *mmeta.Input != config.Input {
- t.Fatal("unexpected input value")
- }
- if mmeta.MeasurementStartTime.String() != "2020-12-09 05:22:25 +0000 UTC" {
- t.Fatal("unexpected measurement start time value")
- }
- if mmeta.ProbeASN != 30722 {
- t.Fatal("unexpected probe asn value")
- }
- if mmeta.ProbeCC != "IT" {
- t.Fatal("unexpected probe cc value")
- }
- if mmeta.ReportID != config.ReportID {
- t.Fatal("unexpected report id value")
- }
- // TODO(bassosimone): we could better this check
- var scores interface{}
- if err := json.Unmarshal([]byte(mmeta.Scores), &scores); err != nil {
- t.Fatalf("cannot parse scores value: %+v", err)
- }
- if mmeta.TestName != "urlgetter" {
- t.Fatal("unexpected test name value")
- }
- if mmeta.TestStartTime.String() != "2020-12-09 05:22:25 +0000 UTC" {
- t.Fatal("unexpected test start time value")
- }
- // TODO(bassosimone): we could better this check
- var rawmeas interface{}
- if err := json.Unmarshal([]byte(mmeta.RawMeasurement), &rawmeas); err != nil {
- t.Fatalf("cannot parse raw measurement: %+v", err)
- }
-}
-func TestGetMeasurementMetaWorkingWithCancelledContext(t *testing.T) {
- client := Client{
- APIClientTemplate: httpx.APIClientTemplate{
- BaseURL: "https://api.ooni.io/",
- HTTPClient: http.DefaultClient,
- Logger: log.Log,
- UserAgent: "miniooni/0.1.0-dev",
- },
- KVStore: &kvstore.Memory{},
- LoginCalls: &atomic.Int64{},
- RegisterCalls: &atomic.Int64{},
- StateFile: NewStateFile(&kvstore.Memory{}),
- }
- config := model.OOAPIMeasurementMetaConfig{
- ReportID: `20201209T052225Z_urlgetter_IT_30722_n1_E1VUhMz08SEkgYFU`,
- Full: true,
- Input: `https://www.example.org`,
- }
- ctx, cancel := context.WithCancel(context.Background())
- cancel() // fail immediately
- mmeta, err := client.GetMeasurementMeta(ctx, config)
- if !errors.Is(err, context.Canceled) {
- t.Fatalf("not the error we expected: %+v", err)
- }
- if mmeta != nil {
- t.Fatal("we expected a nil mmeta here")
+ // This is what we expectMmeta the API to send us. We share this struct
+ // because we're using it for testing with the real backend as well as
+ // for testing with a local test server.
+ //
+ // The measurement is marked as "failed". This feels wrong but it may
+ // be that the fastpah marks all urlgetter measurements as failed.
+ //
+ // We are not including the raw measurement for simplicity (also, there are
+ // not tests for the ooni/backend API anyway, and so it's fine).
+ expectMmeta := &model.OOAPIMeasurementMeta{
+ Anomaly: false,
+ CategoryCode: "",
+ Confirmed: false,
+ Failure: true,
+ Input: &config.Input,
+ MeasurementStartTime: time.Date(2020, 12, 9, 5, 22, 25, 0, time.UTC),
+ ProbeASN: 30722,
+ ProbeCC: "IT",
+ ReportID: "20201209T052225Z_urlgetter_IT_30722_n1_E1VUhMz08SEkgYFU",
+ Scores: `{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}`,
+ TestName: "urlgetter",
+ TestStartTime: time.Date(2020, 12, 9, 5, 22, 25, 0, time.UTC),
+ RawMeasurement: "",
}
+
+ // First, let's check whether we can get a response from the real OONI backend.
+ t.Run("is working as intended with the real backend", func(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skip test in short mode")
+ }
+
+ // construct a client and override the URL to be the production backend
+ // instead of the testing backend, so we have stable measurements
+ client := newclient()
+ client.BaseURL = "https://api.ooni.io/"
+
+ // issue the API call proper
+ mmeta, err := client.GetMeasurementMeta(context.Background(), config)
+
+ // we do not expect to see errors obviously
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // the raw measurement must not be empty and must parse as JSON
+ //
+ // once we know that, clear it for the subsequent cmp.Diff
+ if mmeta.RawMeasurement == "" {
+ t.Fatal("mmeta.RawMeasurement should not be empty")
+ }
+ var rawmeas any
+ must.UnmarshalJSON([]byte(mmeta.RawMeasurement), &rawmeas)
+ mmeta.RawMeasurement = ""
+
+ // compare with the expectation
+ if diff := cmp.Diff(expectMmeta, mmeta); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+
+ // Now let's construct a test server that returns a valid response and try
+ // to communicate with such a test server successfully and with errors
+
+ t.Run("is working as intended with a local test server", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ runtimex.Assert(r.Method == http.MethodGet, "invalid method")
+ runtimex.Assert(r.URL.Path == "/api/v1/measurement_meta", "invalid URL path")
+ w.Write(must.MarshalJSON(expectMmeta))
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // issue the API call proper
+ mmeta, err := client.GetMeasurementMeta(context.Background(), config)
+
+ // we do not expect to see errors obviously
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // compare with the expectation
+ if diff := cmp.Diff(expectMmeta, mmeta); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+
+ t.Run("we can use cloudfronting", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ runtimex.Assert(r.Host == "www.cloudfront.com", "invalid r.Host")
+ runtimex.Assert(r.Method == http.MethodGet, "invalid method")
+ runtimex.Assert(r.URL.Path == "/api/v1/measurement_meta", "invalid URL path")
+ w.Write(must.MarshalJSON(expectMmeta))
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // make sure we're using cloudfronting
+ client.Host = "www.cloudfront.com"
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // issue the API call proper
+ mmeta, err := client.GetMeasurementMeta(context.Background(), config)
+
+ // we do not expect to see errors obviously
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // compare with the expectation
+ if diff := cmp.Diff(expectMmeta, mmeta); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+
+ t.Run("reports an error when the connection is reset", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // issue the API call proper
+ mmeta, err := client.GetMeasurementMeta(context.Background(), config)
+
+ // we do expect an error
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect mmeta to be nil
+ if mmeta != nil {
+ t.Fatal("expected nil meta")
+ }
+ })
+
+ t.Run("reports an error when the response is not JSON parsable", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{`))
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // issue the API call proper
+ mmeta, err := client.GetMeasurementMeta(context.Background(), config)
+
+ // we do expect an error
+ if err == nil || err.Error() != "unexpected end of JSON input" {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect mmeta to be nil
+ if mmeta != nil {
+ t.Fatal("expected nil meta")
+ }
+ })
+
+ t.Run("correctly handles the case where the URL is unparseable", func(t *testing.T) {
+ // create a probeservices client
+ client := newclient()
+
+ // override the URL to be unparseable
+ client.BaseURL = "\t\t\t"
+
+ // issue the API call proper
+ mmeta, err := client.GetMeasurementMeta(context.Background(), config)
+
+ // we do expect an error
+ if err == nil || err.Error() != `parse "\t\t\t": net/url: invalid control character in URL` {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect mmeta to be nil
+ if mmeta != nil {
+ t.Fatal("expected nil meta")
+ }
+ })
}
diff --git a/pkg/probeservices/probeservices.go b/pkg/probeservices/probeservices.go
index 143c4b498..18d711f9f 100644
--- a/pkg/probeservices/probeservices.go
+++ b/pkg/probeservices/probeservices.go
@@ -28,8 +28,6 @@ import (
"net/url"
"sync/atomic"
- "github.com/ooni/probe-engine/pkg/httpapi"
- "github.com/ooni/probe-engine/pkg/httpx"
"github.com/ooni/probe-engine/pkg/model"
)
@@ -65,11 +63,15 @@ type Session interface {
// Client is a client for the OONI probe services API.
type Client struct {
- httpx.APIClientTemplate
+ BaseURL string
+ HTTPClient model.HTTPClient
+ Host string
KVStore model.KeyValueStore
+ Logger model.Logger
LoginCalls *atomic.Int64
RegisterCalls *atomic.Int64
StateFile StateFile
+ UserAgent string
}
// GetCredsAndAuth is an utility function that returns the credentials with
@@ -92,16 +94,15 @@ func (c Client) GetCredsAndAuth() (*model.OOAPILoginCredentials, *model.OOAPILog
// function fails, e.g., we don't support the specified endpoint.
func NewClient(sess Session, endpoint model.OOAPIService) (*Client, error) {
client := &Client{
- APIClientTemplate: httpx.APIClientTemplate{
- BaseURL: endpoint.Address,
- HTTPClient: sess.DefaultHTTPClient(),
- Logger: sess.Logger(),
- UserAgent: sess.UserAgent(),
- },
+ BaseURL: endpoint.Address,
+ HTTPClient: sess.DefaultHTTPClient(),
+ Host: "",
KVStore: sess.KeyValueStore(),
+ Logger: sess.Logger(),
LoginCalls: &atomic.Int64{},
RegisterCalls: &atomic.Int64{},
StateFile: NewStateFile(sess.KeyValueStore()),
+ UserAgent: sess.UserAgent(),
}
switch endpoint.Type {
case "https":
@@ -117,7 +118,7 @@ func NewClient(sess Session, endpoint model.OOAPIService) (*Client, error) {
if URL.Scheme != "https" || URL.Host != URL.Hostname() {
return nil, ErrUnsupportedCloudFrontAddress
}
- client.APIClientTemplate.Host = URL.Hostname()
+ client.Host = URL.Hostname()
URL.Host = endpoint.Front
client.BaseURL = URL.String()
if _, err := url.Parse(client.BaseURL); err != nil {
@@ -128,17 +129,3 @@ func NewClient(sess Session, endpoint model.OOAPIService) (*Client, error) {
return nil, ErrUnsupportedEndpoint
}
}
-
-// newHTTPAPIEndpoint is a convenience function for constructing a new
-// instance of *httpapi.Endpoint based on the content of Client
-func (c Client) newHTTPAPIEndpoint() *httpapi.Endpoint {
- // TODO(https://github.com/ooni/probe/issues/2362): we should migrate all APIs to use
- // httpapi, which supports fallback, while httpx does not support fallback.
- return &httpapi.Endpoint{
- BaseURL: c.BaseURL,
- HTTPClient: c.HTTPClient,
- Logger: c.Logger,
- Host: c.Host,
- UserAgent: c.UserAgent,
- }
-}
diff --git a/pkg/probeservices/psiphon.go b/pkg/probeservices/psiphon.go
index a01421d05..95db20bf2 100644
--- a/pkg/probeservices/psiphon.go
+++ b/pkg/probeservices/psiphon.go
@@ -3,15 +3,39 @@ package probeservices
import (
"context"
"fmt"
+
+ "github.com/ooni/probe-engine/pkg/httpclientx"
+ "github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/urlx"
)
// FetchPsiphonConfig fetches psiphon config from authenticated OONI orchestra.
func (c Client) FetchPsiphonConfig(ctx context.Context) ([]byte, error) {
+ // get credentials and authentication token
_, auth, err := c.GetCredsAndAuth()
if err != nil {
return nil, err
}
+
+ // format Authorization header value
s := fmt.Sprintf("Bearer %s", auth.Token)
- client := c.APIClientTemplate.BuildWithAuthorization(s)
- return client.FetchResource(ctx, "/api/v1/test-list/psiphon-config")
+
+ // construct the URL to use
+ URL, err := urlx.ResolveReference(c.BaseURL, "/api/v1/test-list/psiphon-config", "")
+ if err != nil {
+ return nil, err
+ }
+
+ // get response
+ //
+ // use a model.DiscardLogger to avoid logging config
+ return httpclientx.GetRaw(
+ ctx,
+ httpclientx.NewEndpoint(URL).WithHostOverride(c.Host),
+ &httpclientx.Config{
+ Authorization: s,
+ Client: c.HTTPClient,
+ Logger: model.DiscardLogger,
+ UserAgent: c.UserAgent,
+ })
}
diff --git a/pkg/probeservices/psiphon_test.go b/pkg/probeservices/psiphon_test.go
index 3bfacd2de..f92490b27 100644
--- a/pkg/probeservices/psiphon_test.go
+++ b/pkg/probeservices/psiphon_test.go
@@ -4,44 +4,301 @@ import (
"context"
"encoding/json"
"errors"
+ "net/http"
+ "net/url"
"testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/mocks"
+ "github.com/ooni/probe-engine/pkg/netxlite"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/testingx"
)
func TestFetchPsiphonConfig(t *testing.T) {
- if testing.Short() {
- t.Skip("skip test in short mode")
- }
- clnt := newclient()
- if err := clnt.MaybeRegister(context.Background(), MetadataFixture()); err != nil {
- t.Fatal(err)
- }
- if err := clnt.MaybeLogin(context.Background()); err != nil {
- t.Fatal(err)
- }
- data, err := clnt.FetchPsiphonConfig(context.Background())
- if err != nil {
- t.Fatal(err)
- }
- var config interface{}
- if err := json.Unmarshal(data, &config); err != nil {
- t.Fatal(err)
- }
-}
+ // psiphonflow is the flow with which we invoke the psiphon API
+ psiphonflow := func(t *testing.T, client *Client) ([]byte, error) {
+ // we need to make sure we're registered and logged in
+ if err := client.MaybeRegister(context.Background(), MetadataFixture()); err != nil {
+ t.Fatal(err)
+ }
+ if err := client.MaybeLogin(context.Background()); err != nil {
+ t.Fatal(err)
+ }
-func TestFetchPsiphonConfigNotRegistered(t *testing.T) {
- clnt := newclient()
- state := State{
- // Explicitly empty so the test is more clear
- }
- if err := clnt.StateFile.Set(state); err != nil {
- t.Fatal(err)
- }
- data, err := clnt.FetchPsiphonConfig(context.Background())
- if !errors.Is(err, ErrNotRegistered) {
- t.Fatal("expected an error here")
- }
- if data != nil {
- t.Fatal("expected nil data here")
+ // then we can try to fetch the config
+ return client.FetchPsiphonConfig(context.Background())
}
+
+ // First, let's check whether we can get a response from the real OONI backend.
+ t.Run("is working as intended with the real backend", func(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skip test in short mode")
+ }
+
+ clnt := newclient()
+
+ // run the psiphon flow
+ data, err := psiphonflow(t, clnt)
+
+ // we do not expect an error here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // the config is bytes but we want to make sure we can parse it
+ var config interface{}
+ if err := json.Unmarshal(data, &config); err != nil {
+ t.Fatal(err)
+ }
+ })
+
+ // Now let's construct a test server that returns a valid response and try
+ // to communicate with such a test server successfully and with errors
+
+ t.Run("is working as intended with a local test server", func(t *testing.T) {
+ // create state for emulating the OONI backend
+ state := &testingx.OONIBackendWithLoginFlow{}
+
+ // make sure we return something that is JSON parseable
+ state.SetPsiphonConfig([]byte(`{}`))
+
+ // expose the state via HTTP
+ srv := testingx.MustNewHTTPServer(state.NewMux())
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client so we speak with our local server rather than the true backend
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // then we can try to fetch the config
+ data, err := psiphonflow(t, client)
+
+ // we do not expect an error here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // the config is bytes but we want to make sure we can parse it
+ var config interface{}
+ if err := json.Unmarshal(data, &config); err != nil {
+ t.Fatal(err)
+ }
+ })
+
+ t.Run("we can use cloudfronting", func(t *testing.T) {
+ // create state for emulating the OONI backend
+ state := &testingx.OONIBackendWithLoginFlow{}
+ mux := state.NewMux()
+
+ // make sure we return something that is JSON parseable
+ state.SetPsiphonConfig([]byte(`{}`))
+
+ // expose the state via HTTP
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ runtimex.Assert(r.Host == "www.cloudfront.com", "invalid r.Host")
+ mux.ServeHTTP(w, r)
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // make sure we're using cloudfronting
+ client.Host = "www.cloudfront.com"
+
+ // override the HTTP client so we speak with our local server rather than the true backend
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // then we can try to fetch the config
+ data, err := psiphonflow(t, client)
+
+ // we do not expect an error here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // the config is bytes but we want to make sure we can parse it
+ var config interface{}
+ if err := json.Unmarshal(data, &config); err != nil {
+ t.Fatal(err)
+ }
+ })
+
+ t.Run("reports an error when the connection is reset", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // we need to convince the client that we're logged in first otherwise it will
+ // refuse to send a request to the server and we won't be testing networking
+ runtimex.Try0(client.StateFile.Set(State{
+ ClientID: "ttt-uuu-iii",
+ Expire: time.Now().Add(30 * time.Hour),
+ Password: "xxx-xxx-xxx",
+ Token: "abc-yyy-zzz",
+ }))
+
+ // issue the call directly: no register or login otherwise we're testing
+ // the register or login implementation LOL
+ data, err := client.FetchPsiphonConfig(context.Background())
+
+ // we do expect an error
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect to see zero-length data
+ if len(data) != 0 {
+ t.Fatal("expected result lenght to be zero")
+ }
+ })
+
+ t.Run("when we're not registered", func(t *testing.T) {
+ clnt := newclient()
+
+ // With explicitly empty state so it's pretty obvioust there's no state
+ state := State{}
+
+ // force the state to be empty
+ if err := clnt.StateFile.Set(state); err != nil {
+ t.Fatal(err)
+ }
+
+ // attempt to fetch the config
+ data, err := clnt.FetchPsiphonConfig(context.Background())
+
+ // ensure that the error says we're not registered
+ if !errors.Is(err, ErrNotRegistered) {
+ t.Fatal("expected an error here")
+ }
+
+ // obviously the data should be empty as well
+ if len(data) != 0 {
+ t.Fatal("expected nil data here")
+ }
+ })
+
+ t.Run("correctly handles the case where the URL is unparseable", func(t *testing.T) {
+ // create a probeservices client
+ client := newclient()
+
+ // override the URL to be unparseable
+ client.BaseURL = "\t\t\t"
+
+ // we need to convince the client that we're logged in first otherwise it will
+ // refuse to send a request to the server and we won't be testing networking
+ runtimex.Try0(client.StateFile.Set(State{
+ ClientID: "ttt-uuu-iii",
+ Expire: time.Now().Add(30 * time.Hour),
+ Password: "xxx-xxx-xxx",
+ Token: "abc-yyy-zzz",
+ }))
+
+ // issue the API call proper
+ data, err := client.FetchPsiphonConfig(context.Background())
+
+ // we do expect an error
+ if err == nil || err.Error() != `parse "\t\t\t": net/url: invalid control character in URL` {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect data to be zero length
+ if len(data) != 0 {
+ t.Fatal("expected zero length data")
+ }
+ })
+
+ t.Run("is not logging the response body", func(t *testing.T) {
+ // create state for emulating the OONI backend
+ state := &testingx.OONIBackendWithLoginFlow{}
+
+ // make sure we return something that is JSON parseable
+ state.SetPsiphonConfig([]byte(`{}`))
+
+ // expose the state via HTTP
+ srv := testingx.MustNewHTTPServer(state.NewMux())
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // create and use a logger for collecting logs
+ logger := &testingx.Logger{}
+ client.Logger = logger
+
+ // override the HTTP client so we speak with our local server rather than the true backend
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // then we can try to fetch the config
+ data, err := psiphonflow(t, client)
+
+ // we do not expect an error here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // the config is bytes but we want to make sure we can parse it
+ var config interface{}
+ if err := json.Unmarshal(data, &config); err != nil {
+ t.Fatal(err)
+ }
+
+ // assert that there are no logs
+ //
+ // the register, login, and psiphon API should not log their bodies
+ if diff := cmp.Diff([]string{}, logger.AllLines()); diff != "" {
+ t.Fatal(diff)
+ }
+ })
}
diff --git a/pkg/probeservices/register.go b/pkg/probeservices/register.go
index 4447cb058..37a1a129e 100644
--- a/pkg/probeservices/register.go
+++ b/pkg/probeservices/register.go
@@ -1,10 +1,16 @@
package probeservices
+//
+// register.go - POST /api/v1/register
+//
+
import (
"context"
+ "github.com/ooni/probe-engine/pkg/httpclientx"
"github.com/ooni/probe-engine/pkg/model"
"github.com/ooni/probe-engine/pkg/randx"
+ "github.com/ooni/probe-engine/pkg/urlx"
)
// MaybeRegister registers this client if not already registered
@@ -24,11 +30,27 @@ func (c Client) MaybeRegister(ctx context.Context, metadata model.OOAPIProbeMeta
OOAPIProbeMetadata: metadata,
Password: pwd,
}
- var resp model.OOAPIRegisterResponse
- if err := c.APIClientTemplate.Build().PostJSON(
- ctx, "/api/v1/register", req, &resp); err != nil {
+
+ // construct the URL to use
+ URL, err := urlx.ResolveReference(c.BaseURL, "/api/v1/register", "")
+ if err != nil {
+ return err
+ }
+
+ resp, err := httpclientx.PostJSON[*model.OOAPIRegisterRequest, *model.OOAPIRegisterResponse](
+ ctx,
+ httpclientx.NewEndpoint(URL).WithHostOverride(c.Host),
+ req,
+ &httpclientx.Config{
+ Client: c.HTTPClient,
+ Logger: model.DiscardLogger,
+ UserAgent: c.UserAgent,
+ },
+ )
+ if err != nil {
return err
}
+
state.ClientID = resp.ClientID
state.Password = pwd
return c.StateFile.Set(state)
diff --git a/pkg/probeservices/register_test.go b/pkg/probeservices/register_test.go
index 6e81a1782..13082b1b0 100644
--- a/pkg/probeservices/register_test.go
+++ b/pkg/probeservices/register_test.go
@@ -3,37 +3,239 @@ package probeservices
import (
"context"
"errors"
+ "net/http"
+ "net/url"
"strings"
"testing"
+ "github.com/ooni/probe-engine/pkg/mocks"
"github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/netxlite"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/testingx"
)
func TestMaybeRegister(t *testing.T) {
+ t.Run("is working as intended with the real OONI backend", func(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skip test in short mode")
+ }
+
+ // create client
+ clnt := newclient()
+
+ // attempt to register once
+ if err := clnt.MaybeRegister(context.Background(), MetadataFixture()); err != nil {
+ t.Fatal(err)
+ }
+
+ // try again (we want to make sure it's idempotent once we've registered)
+ if err := clnt.MaybeRegister(context.Background(), MetadataFixture()); err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we indeed only called it once
+ if clnt.RegisterCalls.Load() != 1 {
+ t.Fatal("called register API too many times")
+ }
+ })
+
+ // Now let's construct a test server that returns a valid response and try
+ // to communicate with such a test server successfully and with errors
+
+ t.Run("is working as intended with a local test server", func(t *testing.T) {
+ // create state for emulating the OONI backend
+ state := &testingx.OONIBackendWithLoginFlow{}
+
+ // expose the state via HTTP
+ srv := testingx.MustNewHTTPServer(state.NewMux())
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client so we speak with our local server rather than the true backend
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // attempt to register once
+ if err := client.MaybeRegister(context.Background(), MetadataFixture()); err != nil {
+ t.Fatal(err)
+ }
+
+ // try again (we want to make sure it's idempotent once we've registered)
+ if err := client.MaybeRegister(context.Background(), MetadataFixture()); err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we indeed only called it once
+ if client.RegisterCalls.Load() != 1 {
+ t.Fatal("called register API too many times")
+ }
+ })
+
+ t.Run("we can use cloudfronting", func(t *testing.T) {
+ // create state for emulating the OONI backend
+ state := &testingx.OONIBackendWithLoginFlow{}
+ mux := state.NewMux()
+
+ // expose the state via HTTP
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ runtimex.Assert(r.Host == "www.cloudfront.com", "invalid r.Host")
+ mux.ServeHTTP(w, r)
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // make sure we're using cloudfronting
+ client.Host = "www.cloudfront.com"
+
+ // override the HTTP client so we speak with our local server rather than the true backend
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // attempt to register once
+ if err := client.MaybeRegister(context.Background(), MetadataFixture()); err != nil {
+ t.Fatal(err)
+ }
+
+ // try again (we want to make sure it's idempotent once we've registered)
+ if err := client.MaybeRegister(context.Background(), MetadataFixture()); err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we indeed only called it once
+ if client.RegisterCalls.Load() != 1 {
+ t.Fatal("called register API too many times")
+ }
+ })
+
+ t.Run("reports an error when the connection is reset", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // attempt to register
+ err := client.MaybeRegister(context.Background(), MetadataFixture())
+
+ // we do expect an error
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure we did call register
+ if client.RegisterCalls.Load() != 1 {
+ t.Fatal("called register API the wrong number of times")
+ }
+ })
+
+ t.Run("reports an error when the response is not JSON parsable", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{`))
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // attempt to register
+ err := client.MaybeRegister(context.Background(), MetadataFixture())
+
+ // we do expect an error
+ if err == nil || err.Error() != "unexpected end of JSON input" {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure we did call register
+ if client.RegisterCalls.Load() != 1 {
+ t.Fatal("called register API the wrong number of times")
+ }
+ })
+
t.Run("when metadata is not valid", func(t *testing.T) {
+ // we expect ErrInvalidMetadata when metadata is empty
clnt := newclient()
- ctx := context.Background()
- var metadata model.OOAPIProbeMetadata
- err := clnt.MaybeRegister(ctx, metadata)
+ err := clnt.MaybeRegister(context.Background(), model.OOAPIProbeMetadata{})
if !errors.Is(err, ErrInvalidMetadata) {
t.Fatal("expected an error here")
}
})
+
t.Run("when we have already registered", func(t *testing.T) {
clnt := newclient()
+
+ // create a state with valid credentials
state := State{
ClientID: "xx-xxx-x-xxxx",
Password: "xx",
}
+
+ // synchronize the state
if err := clnt.StateFile.Set(state); err != nil {
t.Fatal(err)
}
- ctx := context.Background()
- metadata := MetadataFixture()
- if err := clnt.MaybeRegister(ctx, metadata); err != nil {
+
+ // attempt to register, which should immediately succeed
+ if err := clnt.MaybeRegister(context.Background(), MetadataFixture()); err != nil {
t.Fatal(err)
}
+
+ // make sure we did not call register
+ if clnt.RegisterCalls.Load() != 0 {
+ t.Fatal("called register API the wrong number of times")
+ }
})
+
t.Run("when the API call fails", func(t *testing.T) {
clnt := newclient()
clnt.BaseURL = "\t\t\t" // makes it fail
@@ -44,23 +246,25 @@ func TestMaybeRegister(t *testing.T) {
t.Fatal("expected an error here")
}
})
-}
-func TestMaybeRegisterIdempotent(t *testing.T) {
- if testing.Short() {
- t.Skip("skip test in short mode")
- }
-
- clnt := newclient()
- ctx := context.Background()
- metadata := MetadataFixture()
- if err := clnt.MaybeRegister(ctx, metadata); err != nil {
- t.Fatal(err)
- }
- if err := clnt.MaybeRegister(ctx, metadata); err != nil {
- t.Fatal(err)
- }
- if clnt.RegisterCalls.Load() != 1 {
- t.Fatal("called register API too many times")
- }
+ t.Run("correctly handles the case where the URL is unparseable", func(t *testing.T) {
+ // create a probeservices client
+ client := newclient()
+
+ // override the URL to be unparseable
+ client.BaseURL = "\t\t\t"
+
+ // attempt to register
+ err := client.MaybeRegister(context.Background(), MetadataFixture())
+
+ // we do expect an error
+ if err == nil || err.Error() != `parse "\t\t\t": net/url: invalid control character in URL` {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure we did call register
+ if client.RegisterCalls.Load() != 1 {
+ t.Fatal("called register API the wrong number of times")
+ }
+ })
}
diff --git a/pkg/probeservices/testdata/collector-expected.jsonl b/pkg/probeservices/testdata/collector-expected.jsonl
deleted file mode 100644
index 7f9af5f38..000000000
--- a/pkg/probeservices/testdata/collector-expected.jsonl
+++ /dev/null
@@ -1 +0,0 @@
-{"format":"json","content":{"data_format_version":"0.2.0","id":"bdd20d7a-bba5-40dd-a111-9863d7908572","input":null,"measurement_start_time":"2018-11-01 15:33:20","probe_asn":"AS0","probe_cc":"ZZ","probe_ip":"1.2.3.4","probe_network_name":"","report_id":"_id","resolver_asn":"AS15169","resolver_ip":"8.8.8.8","resolver_network_name":"Google LLC","software_name":"ooniprobe-engine","software_version":"0.1.0","test_keys":{"failure":null},"test_name":"dummy","test_runtime":5.0565230846405,"test_start_time":"2018-11-01 15:33:17","test_version":"0.1.0"}}
\ No newline at end of file
diff --git a/pkg/probeservices/tor.go b/pkg/probeservices/tor.go
index ed5d27f2c..00e5eccbf 100644
--- a/pkg/probeservices/tor.go
+++ b/pkg/probeservices/tor.go
@@ -5,20 +5,42 @@ import (
"fmt"
"net/url"
+ "github.com/ooni/probe-engine/pkg/httpclientx"
"github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/urlx"
)
// FetchTorTargets returns the targets for the tor experiment.
-func (c Client) FetchTorTargets(ctx context.Context, cc string) (result map[string]model.OOAPITorTarget, err error) {
+func (c Client) FetchTorTargets(ctx context.Context, cc string) (map[string]model.OOAPITorTarget, error) {
+ // get credentials and authentication token
_, auth, err := c.GetCredsAndAuth()
if err != nil {
return nil, err
}
+
+ // format Authorization header value
s := fmt.Sprintf("Bearer %s", auth.Token)
- client := c.APIClientTemplate.BuildWithAuthorization(s)
+
+ // create query string
query := url.Values{}
query.Add("country_code", cc)
- err = client.GetJSONWithQuery(
- ctx, "/api/v1/test-list/tor-targets", query, &result)
- return
+
+ // construct the URL to use
+ URL, err := urlx.ResolveReference(c.BaseURL, "/api/v1/test-list/tor-targets", query.Encode())
+ if err != nil {
+ return nil, err
+ }
+
+ // get response
+ //
+ // use a model.DiscardLogger to avoid logging bridges
+ return httpclientx.GetJSON[map[string]model.OOAPITorTarget](
+ ctx,
+ httpclientx.NewEndpoint(URL).WithHostOverride(c.Host),
+ &httpclientx.Config{
+ Authorization: s,
+ Client: c.HTTPClient,
+ Logger: model.DiscardLogger,
+ UserAgent: c.UserAgent,
+ })
}
diff --git a/pkg/probeservices/tor_test.go b/pkg/probeservices/tor_test.go
index 62f706514..149fb8f85 100644
--- a/pkg/probeservices/tor_test.go
+++ b/pkg/probeservices/tor_test.go
@@ -2,85 +2,346 @@ package probeservices
import (
"context"
+ "errors"
"net/http"
+ "net/url"
"testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/mocks"
+ "github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/netxlite"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/testingx"
)
func TestFetchTorTargets(t *testing.T) {
- if testing.Short() {
- t.Skip("skip test in short mode")
- }
- clnt := newclient()
- if err := clnt.MaybeRegister(context.Background(), MetadataFixture()); err != nil {
- t.Fatal(err)
- }
- if err := clnt.MaybeLogin(context.Background()); err != nil {
- t.Fatal(err)
- }
- data, err := clnt.FetchTorTargets(context.Background(), "ZZ")
- if err != nil {
- t.Fatal(err)
- }
- if data == nil || len(data) <= 0 {
- t.Fatal("invalid data")
- }
-}
+ // Implementation note: OONIBackendWithLoginFlow ensures that tor sends a query
+ // string and there are also tests making sure of that. We use to have a test that
+ // checked for the query string here, but now it seems unnecessary.
-func TestFetchTorTargetsNotRegistered(t *testing.T) {
- clnt := newclient()
- state := State{
- // Explicitly empty so the test is more clear
- }
- if err := clnt.StateFile.Set(state); err != nil {
- t.Fatal(err)
- }
- data, err := clnt.FetchTorTargets(context.Background(), "ZZ")
- if err == nil {
- t.Fatal("expected an error here")
- }
- if data != nil {
- t.Fatal("expected nil data here")
+ // torflow is the flow with which we invoke the tor API
+ torflow := func(t *testing.T, client *Client) (map[string]model.OOAPITorTarget, error) {
+ // we need to make sure we're registered and logged in
+ if err := client.MaybeRegister(context.Background(), MetadataFixture()); err != nil {
+ t.Fatal(err)
+ }
+ if err := client.MaybeLogin(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+
+ // then we can try to fetch the config
+ return client.FetchTorTargets(context.Background(), "ZZ")
}
-}
-type FetchTorTargetsHTTPTransport struct {
- Response *http.Response
-}
+ // First, let's check whether we can get a response from the real OONI backend.
+ t.Run("is working as intended with the real backend", func(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skip test in short mode")
+ }
-func (clnt *FetchTorTargetsHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
- resp, err := http.DefaultTransport.RoundTrip(req)
- if err != nil {
- return nil, err
- }
- if req.URL.Path == "/api/v1/test-list/tor-targets" {
- clnt.Response = resp
- }
- return resp, err
-}
+ clnt := newclient()
-func TestFetchTorTargetsSetsQueryString(t *testing.T) {
- if testing.Short() {
- t.Skip("skip test in short mode")
- }
+ // run the tor flow
+ targets, err := torflow(t, clnt)
- clnt := newclient()
- txp := new(FetchTorTargetsHTTPTransport)
- clnt.HTTPClient = &http.Client{Transport: txp}
- if err := clnt.MaybeRegister(context.Background(), MetadataFixture()); err != nil {
- t.Fatal(err)
- }
- if err := clnt.MaybeLogin(context.Background()); err != nil {
- t.Fatal(err)
- }
- data, err := clnt.FetchTorTargets(context.Background(), "ZZ")
- if err != nil {
- t.Fatal(err)
- }
- if data == nil || len(data) <= 0 {
- t.Fatal("invalid data")
- }
- requestURL := txp.Response.Request.URL
- if requestURL.Query().Get("country_code") != "ZZ" {
- t.Fatal("invalid country code")
- }
+ // we do not expect an error here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // we expect non-zero length targets
+ if len(targets) <= 0 {
+ t.Fatal("expected non-zero-length targets")
+ }
+ })
+
+ // Now let's construct a test server that returns a valid response and try
+ // to communicate with such a test server successfully and with errors
+
+ t.Run("is working as intended with a local test server", func(t *testing.T) {
+ // create state for emulating the OONI backend
+ state := &testingx.OONIBackendWithLoginFlow{}
+
+ // make sure we return something that is JSON parseable and non-zero-length
+ state.SetTorTargets([]byte(`{"foo": {}}`))
+
+ // expose the state via HTTP
+ srv := testingx.MustNewHTTPServer(state.NewMux())
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client so we speak with our local server rather than the true backend
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // run the tor flow
+ targets, err := torflow(t, client)
+
+ // we do not expect an error here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // we expect non-zero length targets
+ if len(targets) <= 0 {
+ t.Fatal("expected non-zero-length targets")
+ }
+ })
+
+ t.Run("we can use cloudfronting", func(t *testing.T) {
+ // create state for emulating the OONI backend
+ state := &testingx.OONIBackendWithLoginFlow{}
+ mux := state.NewMux()
+
+ // make sure we return something that is JSON parseable and non-zero-length
+ state.SetTorTargets([]byte(`{"foo": {}}`))
+
+ // expose the state via HTTP
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ runtimex.Assert(r.Host == "www.cloudfront.com", "invalid r.Host")
+ mux.ServeHTTP(w, r)
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // make sure we're using cloudfronting
+ client.Host = "www.cloudfront.com"
+
+ // override the HTTP client so we speak with our local server rather than the true backend
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // run the tor flow
+ targets, err := torflow(t, client)
+
+ // we do not expect an error here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // we expect non-zero length targets
+ if len(targets) <= 0 {
+ t.Fatal("expected non-zero-length targets")
+ }
+ })
+
+ t.Run("reports an error when the connection is reset", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // we need to convince the client that we're logged in first otherwise it will
+ // refuse to send a request to the server and we won't be testing networking
+ runtimex.Try0(client.StateFile.Set(State{
+ ClientID: "ttt-uuu-iii",
+ Expire: time.Now().Add(30 * time.Hour),
+ Password: "xxx-xxx-xxx",
+ Token: "abc-yyy-zzz",
+ }))
+
+ // run the tor flow
+ targets, err := torflow(t, client)
+
+ // we do expect an error
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect to see zero-length targets
+ if len(targets) != 0 {
+ t.Fatal("expected targets to be zero length")
+ }
+ })
+
+ t.Run("reports an error when the response is not JSON parsable", func(t *testing.T) {
+ // create quick and dirty server to serve the response
+ srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{`))
+ }))
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // override the HTTP client
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // we need to convince the client that we're logged in first otherwise it will
+ // refuse to send a request to the server and we won't be testing networking
+ runtimex.Try0(client.StateFile.Set(State{
+ ClientID: "ttt-uuu-iii",
+ Expire: time.Now().Add(30 * time.Hour),
+ Password: "xxx-xxx-xxx",
+ Token: "abc-yyy-zzz",
+ }))
+
+ // run the tor flow
+ targets, err := torflow(t, client)
+
+ // we do expect an error
+ if err == nil || err.Error() != "unexpected end of JSON input" {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect to see zero-length targets
+ if len(targets) != 0 {
+ t.Fatal("expected targets to be zero length")
+ }
+ })
+
+ t.Run("when we're not registered", func(t *testing.T) {
+ clnt := newclient()
+
+ // With explicitly empty state so it's pretty obvioust there's no state
+ state := State{}
+
+ // force the state to be empty
+ if err := clnt.StateFile.Set(state); err != nil {
+ t.Fatal(err)
+ }
+
+ // run the tor flow
+ targets, err := clnt.FetchTorTargets(context.Background(), "ZZ")
+
+ // ensure that the error says we're not registered
+ if !errors.Is(err, ErrNotRegistered) {
+ t.Fatal("expected an error here")
+ }
+
+ // we expect zero length targets
+ if len(targets) != 0 {
+ t.Fatal("expected zero-length targets")
+ }
+ })
+
+ t.Run("correctly handles the case where the URL is unparseable", func(t *testing.T) {
+ // create a probeservices client
+ client := newclient()
+
+ // override the URL to be unparseable
+ client.BaseURL = "\t\t\t"
+
+ // we need to convince the client that we're logged in first otherwise it will
+ // refuse to send a request to the server and we won't be testing networking
+ runtimex.Try0(client.StateFile.Set(State{
+ ClientID: "ttt-uuu-iii",
+ Expire: time.Now().Add(30 * time.Hour),
+ Password: "xxx-xxx-xxx",
+ Token: "abc-yyy-zzz",
+ }))
+
+ targets, err := client.FetchTorTargets(context.Background(), "ZZ")
+
+ // we do expect an error
+ if err == nil || err.Error() != `parse "\t\t\t": net/url: invalid control character in URL` {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect zero length targets
+ if len(targets) != 0 {
+ t.Fatal("expected zero-length targets")
+ }
+ })
+
+ t.Run("is not logging the response body", func(t *testing.T) {
+ // create state for emulating the OONI backend
+ state := &testingx.OONIBackendWithLoginFlow{}
+
+ // make sure we return something that is JSON parseable
+ state.SetTorTargets([]byte(`{}`))
+
+ // expose the state via HTTP
+ srv := testingx.MustNewHTTPServer(state.NewMux())
+ defer srv.Close()
+
+ // create a probeservices client
+ client := newclient()
+
+ // create and use a logger for collecting logs
+ logger := &testingx.Logger{}
+ client.Logger = logger
+
+ // override the HTTP client so we speak with our local server rather than the true backend
+ client.HTTPClient = &mocks.HTTPClient{
+ MockDo: func(req *http.Request) (*http.Response, error) {
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ req.URL.Scheme = URL.Scheme
+ req.URL.Host = URL.Host
+ return http.DefaultClient.Do(req)
+ },
+ MockCloseIdleConnections: func() {
+ http.DefaultClient.CloseIdleConnections()
+ },
+ }
+
+ // then we can try to fetch the targets
+ targets, err := torflow(t, client)
+
+ // we do not expect an error here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // we expect to see zero-length targets
+ if len(targets) != 0 {
+ t.Fatal("expected targets to be zero length")
+ }
+
+ // assert that there are no logs
+ //
+ // the register, login, and tor API should not log their bodies
+ if diff := cmp.Diff([]string{}, logger.AllLines()); diff != "" {
+ t.Fatal(diff)
+ }
+ })
}
diff --git a/pkg/ptx/obfs4.go b/pkg/ptx/obfs4.go
index d3a8707d1..ea09cbe7a 100644
--- a/pkg/ptx/obfs4.go
+++ b/pkg/ptx/obfs4.go
@@ -138,7 +138,7 @@ func (d *obfs4CancellableDialer) dial(
select {
case connch <- conn:
default:
- conn.Close() // context won the race
+ _ = conn.Close() // context won the race
}
}()
select {
diff --git a/pkg/ptx/ptx.go b/pkg/ptx/ptx.go
index 464b7222b..50104a0bf 100644
--- a/pkg/ptx/ptx.go
+++ b/pkg/ptx/ptx.go
@@ -122,11 +122,11 @@ func (lst *Listener) forward(ctx context.Context, left, right net.Conn, done cha
wg.Add(2)
go func() {
defer wg.Done()
- netxlite.CopyContext(ctx, left, right)
+ _, _ = netxlite.CopyContext(ctx, left, right)
}()
go func() {
defer wg.Done()
- netxlite.CopyContext(ctx, right, left)
+ _, _ = netxlite.CopyContext(ctx, right, left)
}()
wg.Wait()
}
@@ -157,7 +157,7 @@ func (lst *Listener) handleSocksConn(ctx context.Context, socksConn SocksConn) e
}
ptConn, err := lst.PTDialer.DialContext(ctx)
if err != nil {
- socksConn.Close() // we own it
+ _ = socksConn.Close() // we own it
lst.logger().Warnf("ptx: ContextDialer.DialContext error: %s", err)
return err // used for testing
}
@@ -296,7 +296,7 @@ func (lst *Listener) Stop() {
lst.cancel() // cancel is idempotent
}
if lst.listener != nil {
- lst.listener.Close() // should be idempotent
+ _ = lst.listener.Close() // should be idempotent
}
}
diff --git a/pkg/ptx/snowflake.go b/pkg/ptx/snowflake.go
index 792718970..b649facee 100644
--- a/pkg/ptx/snowflake.go
+++ b/pkg/ptx/snowflake.go
@@ -163,7 +163,7 @@ func (d *SnowflakeDialer) dialContext(
select {
case connch <- conn:
default:
- conn.Close() // context won the race
+ _ = conn.Close() // context won the race
}
}()
select {
diff --git a/pkg/shellx/shellx.go b/pkg/shellx/shellx.go
index 48685f537..a45a40bcc 100644
--- a/pkg/shellx/shellx.go
+++ b/pkg/shellx/shellx.go
@@ -128,7 +128,7 @@ func cmd(config *Config, argv *Argv, envp *Envp) *execabs.Cmd {
// hence the choice to keep using x/sys/execabs everywhere.
//
// See for more information.
- cmd := execabs.Command(argv.P, argv.V...)
+ cmd := execabs.Command(argv.P, argv.V...) // #nosec G204 - this is working as intended
cmd.Env = os.Environ()
for _, entry := range envp.V {
if config.Logger != nil {
@@ -314,7 +314,7 @@ func CopyFile(source, dest string, perms fs.FileMode) error {
return err
}
if _, err := ioCopy(destfp, sourcefp); err != nil {
- destfp.Close()
+ _ = destfp.Close()
return err
}
return destfp.Close()
diff --git a/pkg/testingproxy/hosthttps.go b/pkg/testingproxy/hosthttps.go
index 1944ccb89..bc8a06416 100644
--- a/pkg/testingproxy/hosthttps.go
+++ b/pkg/testingproxy/hosthttps.go
@@ -60,7 +60,7 @@ func (tc *hostNetworkTestCaseWithHTTPWithTLS) Run(t *testing.T) {
// extend the default cert pool with the proxy's own CA
pool := netxlite.NewMozillaCertPool()
pool.AddCert(proxyServer.CACert)
- tlsConfig := &tls.Config{RootCAs: pool}
+ tlsConfig := &tls.Config{RootCAs: pool} // #nosec G402 - code used for testing
// create an HTTP client configured to use the given proxy
//
diff --git a/pkg/testingproxy/netemhttp.go b/pkg/testingproxy/netemhttp.go
index 494a45d93..f37eca0da 100644
--- a/pkg/testingproxy/netemhttp.go
+++ b/pkg/testingproxy/netemhttp.go
@@ -67,7 +67,7 @@ func (tc *netemTestCaseWithHTTP) Run(t *testing.T) {
// configure the wwwStack as the DNS resolver with proper configuration
dnsConfig := netem.NewDNSConfig()
- dnsConfig.AddRecord("www.example.com.", "", wwwIPAddr)
+ runtimex.Try0(dnsConfig.AddRecord("www.example.com.", "", wwwIPAddr))
dnsServer := runtimex.Try1(netem.NewDNSServer(log.Log, wwwStack, wwwIPAddr, dnsConfig))
defer dnsServer.Close()
@@ -76,7 +76,7 @@ func (tc *netemTestCaseWithHTTP) Run(t *testing.T) {
&net.TCPAddr{IP: net.ParseIP(wwwIPAddr), Port: 80},
wwwStack,
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Write([]byte("Bonsoir, Elliot!\r\n"))
+ _, _ = w.Write([]byte("Bonsoir, Elliot!\r\n"))
}),
)
defer wwwServer80.Close()
@@ -86,7 +86,7 @@ func (tc *netemTestCaseWithHTTP) Run(t *testing.T) {
&net.TCPAddr{IP: net.ParseIP(wwwIPAddr), Port: 443},
wwwStack,
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Write([]byte("Bonsoir, Elliot!\r\n"))
+ _, _ = w.Write([]byte("Bonsoir, Elliot!\r\n"))
}),
wwwStack,
"www.example.com",
@@ -118,7 +118,7 @@ func (tc *netemTestCaseWithHTTP) Run(t *testing.T) {
netxlite.HTTPTransportOptionProxyURL(runtimex.Try1(url.Parse(proxyServer.URL))),
// TODO(https://github.com/ooni/probe/issues/2536)
- netxlite.HTTPTransportOptionTLSClientConfig(&tls.Config{
+ netxlite.HTTPTransportOptionTLSClientConfig(&tls.Config{ // #nosec G402 - code used for testing
RootCAs: clientStack.DefaultCertPool(),
}),
)
diff --git a/pkg/testingproxy/netemhttps.go b/pkg/testingproxy/netemhttps.go
index 25a5a8a39..dbd03c37d 100644
--- a/pkg/testingproxy/netemhttps.go
+++ b/pkg/testingproxy/netemhttps.go
@@ -67,7 +67,7 @@ func (tc *netemTestCaseWithHTTPWithTLS) Run(t *testing.T) {
// configure the wwwStack as the DNS resolver with proper configuration
dnsConfig := netem.NewDNSConfig()
- dnsConfig.AddRecord("www.example.com.", "", wwwIPAddr)
+ runtimex.Try0(dnsConfig.AddRecord("www.example.com.", "", wwwIPAddr))
dnsServer := runtimex.Try1(netem.NewDNSServer(log.Log, wwwStack, wwwIPAddr, dnsConfig))
defer dnsServer.Close()
@@ -76,7 +76,7 @@ func (tc *netemTestCaseWithHTTPWithTLS) Run(t *testing.T) {
&net.TCPAddr{IP: net.ParseIP(wwwIPAddr), Port: 80},
wwwStack,
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Write([]byte("Bonsoir, Elliot!\r\n"))
+ _, _ = w.Write([]byte("Bonsoir, Elliot!\r\n"))
}),
)
defer wwwServer80.Close()
@@ -86,7 +86,7 @@ func (tc *netemTestCaseWithHTTPWithTLS) Run(t *testing.T) {
&net.TCPAddr{IP: net.ParseIP(wwwIPAddr), Port: 443},
wwwStack,
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Write([]byte("Bonsoir, Elliot!\r\n"))
+ _, _ = w.Write([]byte("Bonsoir, Elliot!\r\n"))
}),
wwwStack,
"www.example.com",
@@ -120,7 +120,7 @@ func (tc *netemTestCaseWithHTTPWithTLS) Run(t *testing.T) {
netxlite.HTTPTransportOptionProxyURL(runtimex.Try1(url.Parse(proxyServer.URL))),
// TODO(https://github.com/ooni/probe/issues/2536)
- netxlite.HTTPTransportOptionTLSClientConfig(&tls.Config{
+ netxlite.HTTPTransportOptionTLSClientConfig(&tls.Config{ // #nosec G402 - code used for testing
RootCAs: clientStack.DefaultCertPool(),
}),
)
diff --git a/pkg/testingproxy/socksnetem.go b/pkg/testingproxy/socksnetem.go
index 017d23d53..98b7eafc5 100644
--- a/pkg/testingproxy/socksnetem.go
+++ b/pkg/testingproxy/socksnetem.go
@@ -67,7 +67,7 @@ func (tc *netemTestCaseWithSOCKS) Run(t *testing.T) {
// configure the wwwStack as the DNS resolver with proper configuration
dnsConfig := netem.NewDNSConfig()
- dnsConfig.AddRecord("www.example.com.", "", wwwIPAddr)
+ runtimex.Try0(dnsConfig.AddRecord("www.example.com.", "", wwwIPAddr))
dnsServer := runtimex.Try1(netem.NewDNSServer(log.Log, wwwStack, wwwIPAddr, dnsConfig))
defer dnsServer.Close()
@@ -76,7 +76,7 @@ func (tc *netemTestCaseWithSOCKS) Run(t *testing.T) {
&net.TCPAddr{IP: net.ParseIP(wwwIPAddr), Port: 80},
wwwStack,
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Write([]byte("Bonsoir, Elliot!\r\n"))
+ _, _ = w.Write([]byte("Bonsoir, Elliot!\r\n"))
}),
)
defer wwwServer80.Close()
@@ -86,7 +86,7 @@ func (tc *netemTestCaseWithSOCKS) Run(t *testing.T) {
&net.TCPAddr{IP: net.ParseIP(wwwIPAddr), Port: 443},
wwwStack,
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Write([]byte("Bonsoir, Elliot!\r\n"))
+ _, _ = w.Write([]byte("Bonsoir, Elliot!\r\n"))
}),
wwwStack,
"www.example.com",
@@ -118,7 +118,7 @@ func (tc *netemTestCaseWithSOCKS) Run(t *testing.T) {
netxlite.HTTPTransportOptionProxyURL(proxyServer.URL()),
// TODO(https://github.com/ooni/probe/issues/2536)
- netxlite.HTTPTransportOptionTLSClientConfig(&tls.Config{
+ netxlite.HTTPTransportOptionTLSClientConfig(&tls.Config{ // #nosec G402 - code used for testing
RootCAs: clientStack.DefaultCertPool(),
}),
)
diff --git a/pkg/testingsocks5/auth.go b/pkg/testingsocks5/auth.go
index 63f8200bb..cf0eaf9f6 100644
--- a/pkg/testingsocks5/auth.go
+++ b/pkg/testingsocks5/auth.go
@@ -65,7 +65,7 @@ func (s *Server) authenticate(cconn net.Conn) (*authContext, error) {
// noAcceptableAuth is used to handle when we have no eligible authentication mechanism
func noAcceptableAuth(conn net.Conn) error {
- conn.Write([]byte{socks5Version, noAcceptable})
+ _, _ = conn.Write([]byte{socks5Version, noAcceptable})
return errNoSupportedAuth
}
diff --git a/pkg/testingx/dnsoverhttps.go b/pkg/testingx/dnsoverhttps.go
index d179de80b..d32a8961a 100644
--- a/pkg/testingx/dnsoverhttps.go
+++ b/pkg/testingx/dnsoverhttps.go
@@ -21,7 +21,7 @@ func (p *DNSOverHTTPSHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
rawQuery := runtimex.Try1(io.ReadAll(r.Body))
rawResponse := runtimex.Try1(p.RoundTripper.RoundTrip(r.Context(), rawQuery))
w.Header().Add("content-type", "application/dns-message")
- w.Write(rawResponse)
+ _, _ = w.Write(rawResponse)
}
func (p *DNSOverHTTPSHandler) handlePanic(w http.ResponseWriter) {
diff --git a/pkg/testingx/fakefill.go b/pkg/testingx/fakefill.go
index 1adea3e98..fdd7261d0 100644
--- a/pkg/testingx/fakefill.go
+++ b/pkg/testingx/fakefill.go
@@ -42,7 +42,7 @@ func (ff *FakeFiller) getRandLocked() *rand.Rand {
if ff.Now != nil {
now = ff.Now
}
- ff.rnd = rand.New(rand.NewSource(now().UnixNano()))
+ ff.rnd = rand.New(rand.NewSource(now().UnixNano())) // #nosec G404 -- used for testing
}
return ff.rnd
}
diff --git a/pkg/testingx/geoip.go b/pkg/testingx/geoip.go
index 0a084dff5..d1bcdeee6 100644
--- a/pkg/testingx/geoip.go
+++ b/pkg/testingx/geoip.go
@@ -20,5 +20,5 @@ func (p *GeoIPHandlerUbuntu) ServeHTTP(w http.ResponseWriter, r *http.Request) {
p.ProbeIP,
)
w.Header().Add("Content-Type", "text/xml")
- w.Write([]byte(resp))
+ _, _ = w.Write([]byte(resp))
}
diff --git a/pkg/testingx/httptestx.go b/pkg/testingx/httptestx.go
index 80bbc345a..6fe357d0f 100644
--- a/pkg/testingx/httptestx.go
+++ b/pkg/testingx/httptestx.go
@@ -6,8 +6,10 @@ import (
"net"
"net/http"
"net/url"
+ "time"
"github.com/ooni/netem"
+ "github.com/ooni/probe-engine/pkg/randx"
"github.com/ooni/probe-engine/pkg/runtimex"
)
@@ -66,7 +68,7 @@ func MustNewHTTPServerEx(addr *net.TCPAddr, httpListener TCPListener, handler ht
Path: "/",
}
srv := &HTTPServer{
- Config: &http.Server{Handler: handler},
+ Config: &http.Server{Handler: handler}, // #nosec G112 - just a testing server
Listener: listener,
TLS: nil,
URL: baseURL.String(),
@@ -111,7 +113,7 @@ func MustNewHTTPServerTLSEx(
otherNames = append(otherNames, extraSNIs...)
srv := &HTTPServer{
- Config: &http.Server{Handler: handler},
+ Config: &http.Server{Handler: handler}, // #nosec G112 - just a testing server
Listener: listener,
TLS: ca.MustNewServerTLSConfig(commonName, otherNames...),
URL: baseURL.String(),
@@ -143,7 +145,7 @@ var HTTPBlockpage451 = []byte(`
func HTTPHandlerBlockpage451() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusUnavailableForLegalReasons)
- w.Write(HTTPBlockpage451)
+ _, _ = w.Write(HTTPBlockpage451)
})
}
@@ -173,14 +175,7 @@ func HTTPHandlerTimeout() http.Handler {
}
func httpHandlerHijack(w http.ResponseWriter, r *http.Request, policy string) {
- // Note:
- //
- // 1. we assume we can hihack the connection
- //
- // 2. Hijack won't fail the first time it's invoked
- hijacker := w.(http.Hijacker)
- conn, _ := runtimex.Try2(hijacker.Hijack())
-
+ conn := httpHijack(w)
defer conn.Close()
switch policy {
@@ -194,3 +189,40 @@ func httpHandlerHijack(w http.ResponseWriter, r *http.Request, policy string) {
// nothing
}
}
+
+// HTTPHandlerResetWhileReadingBody returns a handler that sends a
+// connection reset by peer while the client is reading the body.
+func HTTPHandlerResetWhileReadingBody() http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn := httpHijack(w)
+ defer conn.Close()
+
+ // write the HTTP response headers
+ _, _ = conn.Write([]byte("HTTP/1.1 200 Ok\r\n"))
+ _, _ = conn.Write([]byte("Content-Type: text/html\r\n"))
+ _, _ = conn.Write([]byte("Content-Length: 65535\r\n"))
+ _, _ = conn.Write([]byte("\r\n"))
+
+ // start writing the response
+ content := randx.Letters(32768)
+ _, _ = conn.Write([]byte(content))
+
+ // sleep for half a second simulating something wrong
+ time.Sleep(500 * time.Millisecond)
+
+ // finally issue reset for the conn
+ tcpMaybeResetNetConn(conn)
+ })
+}
+
+// httpHijack is a convenience function to hijack the underlying connection.
+func httpHijack(w http.ResponseWriter) net.Conn {
+ // Note:
+ //
+ // 1. we assume we can hihack the connection
+ //
+ // 2. Hijack won't fail the first time it's invoked
+ hijacker := w.(http.Hijacker)
+ conn, _ := runtimex.Try2(hijacker.Hijack())
+ return conn
+}
diff --git a/pkg/testingx/httptestx_test.go b/pkg/testingx/httptestx_test.go
index 43a71bb22..480e4277e 100644
--- a/pkg/testingx/httptestx_test.go
+++ b/pkg/testingx/httptestx_test.go
@@ -498,3 +498,43 @@ func TestHTTPTestxWithNetem(t *testing.T) {
})
}
}
+
+func TestHTTPHandlerResetWhileReadingBody(t *testing.T) {
+ // create a server for testing the given handler
+ server := testingx.MustNewHTTPServer(testingx.HTTPHandlerResetWhileReadingBody())
+ defer server.Close()
+
+ // create a suitable HTTP transport using netxlite
+ netx := &netxlite.Netx{Underlying: nil}
+ dialer := netx.NewDialerWithoutResolver(log.Log)
+ handshaker := netx.NewTLSHandshakerStdlib(log.Log)
+ tlsDialer := netxlite.NewTLSDialer(dialer, handshaker)
+ txp := netxlite.NewHTTPTransportWithOptions(log.Log, dialer, tlsDialer)
+
+ // create the request
+ req := runtimex.Try1(http.NewRequest("GET", server.URL, nil))
+
+ // perform the round trip
+ resp, err := txp.RoundTrip(req)
+
+ // we do not expect an error during the round trip
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // start reading the response where we expect to see a RST
+ respbody, err := netxlite.ReadAllContext(req.Context(), resp.Body)
+
+ // verify we received a connection reset
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("expected ECONNRESET, got", err)
+ }
+
+ // make sure we've got no bytes
+ if len(respbody) != 0 {
+ t.Fatal("expected to see zero bytes here")
+ }
+}
diff --git a/pkg/testingx/logger.go b/pkg/testingx/logger.go
new file mode 100644
index 000000000..fe214cddb
--- /dev/null
+++ b/pkg/testingx/logger.go
@@ -0,0 +1,101 @@
+package testingx
+
+import (
+ "fmt"
+ "sync"
+
+ "github.com/ooni/probe-engine/pkg/logmodel"
+)
+
+// Logger implements [logmodel.Logger] and collects all the log lines.
+//
+// The zero value of this struct is ready to use.
+type Logger struct {
+ // debug contains debug lines.
+ debug []string
+
+ // info contains info lines.
+ info []string
+
+ // mu provides mutual exclusion.
+ mu sync.Mutex
+
+ // warning contains warning lines.
+ warning []string
+}
+
+var _ logmodel.Logger = &Logger{}
+
+// Debug implements logmodel.Logger.
+func (l *Logger) Debug(msg string) {
+ l.mu.Lock()
+ l.debug = append(l.debug, msg)
+ l.mu.Unlock()
+}
+
+// Debugf implements logmodel.Logger.
+func (l *Logger) Debugf(format string, v ...interface{}) {
+ l.Debug(fmt.Sprintf(format, v...))
+}
+
+// Info implements logmodel.Logger.
+func (l *Logger) Info(msg string) {
+ l.mu.Lock()
+ l.info = append(l.info, msg)
+ l.mu.Unlock()
+}
+
+// Infof implements logmodel.Logger.
+func (l *Logger) Infof(format string, v ...interface{}) {
+ l.Info(fmt.Sprintf(format, v...))
+}
+
+// Warn implements logmodel.Logger.
+func (l *Logger) Warn(msg string) {
+ l.mu.Lock()
+ l.warning = append(l.warning, msg)
+ l.mu.Unlock()
+}
+
+// Warnf implements logmodel.Logger.
+func (l *Logger) Warnf(format string, v ...interface{}) {
+ l.Warn(fmt.Sprintf(format, v...))
+}
+
+// DebugLines returns a copy of the observed debug lines.
+func (l *Logger) DebugLines() []string {
+ l.mu.Lock()
+ out := append([]string{}, l.debug...)
+ l.mu.Unlock()
+ return out
+}
+
+// InfoLines returns a copy of the observed info lines.
+func (l *Logger) InfoLines() []string {
+ l.mu.Lock()
+ out := append([]string{}, l.info...)
+ l.mu.Unlock()
+ return out
+}
+
+// WarnLines returns a copy of the observed warn lines.
+func (l *Logger) WarnLines() []string {
+ l.mu.Lock()
+ out := append([]string{}, l.warning...)
+ l.mu.Unlock()
+ return out
+}
+
+// ClearAll removes all the log lines collected so far.
+func (l *Logger) ClearAll() {
+ l.mu.Lock()
+ l.debug = []string{}
+ l.info = []string{}
+ l.warning = []string{}
+ l.mu.Unlock()
+}
+
+// AllLines returns all the collected lines.
+func (l *Logger) AllLines() []string {
+ return append(append(append([]string{}, l.DebugLines()...), l.InfoLines()...), l.WarnLines()...)
+}
diff --git a/pkg/testingx/logger_test.go b/pkg/testingx/logger_test.go
new file mode 100644
index 000000000..1e44b4341
--- /dev/null
+++ b/pkg/testingx/logger_test.go
@@ -0,0 +1,46 @@
+package testingx
+
+import (
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+)
+
+func TestLogger(t *testing.T) {
+ logger := &Logger{}
+
+ logger.Debug("foobar")
+ logger.Debugf("foo%s", "baz")
+ expectDebug := []string{"foobar", "foobaz"}
+
+ logger.Info("barfoo")
+ logger.Infof("bar%s", "baz")
+ expectInfo := []string{"barfoo", "barbaz"}
+
+ logger.Warn("jarjar")
+ logger.Warnf("jar%s", "baz")
+ expectWarn := []string{"jarjar", "jarbaz"}
+
+ // make sure we can get individual lines
+ if diff := cmp.Diff(expectDebug, logger.DebugLines()); diff != "" {
+ t.Fatal(diff)
+ }
+ if diff := cmp.Diff(expectInfo, logger.InfoLines()); diff != "" {
+ t.Fatal(diff)
+ }
+ if diff := cmp.Diff(expectWarn, logger.WarnLines()); diff != "" {
+ t.Fatal(diff)
+ }
+
+ // make sure we can get combines lines
+ expectCombined := append(append(append([]string{}, expectDebug...), expectInfo...), expectWarn...)
+ if diff := cmp.Diff(expectCombined, logger.AllLines()); diff != "" {
+ t.Fatal(diff)
+ }
+
+ // make sure clear works
+ logger.ClearAll()
+ if diff := cmp.Diff([]string{}, logger.AllLines()); diff != "" {
+ t.Fatal(diff)
+ }
+}
diff --git a/pkg/testingx/oonibackendwithlogin.go b/pkg/testingx/oonibackendwithlogin.go
new file mode 100644
index 000000000..ba39d486c
--- /dev/null
+++ b/pkg/testingx/oonibackendwithlogin.go
@@ -0,0 +1,289 @@
+package testingx
+
+//
+// Code for testing the OONI backend login flow.
+//
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "sync"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/must"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+)
+
+// OONIBackendWithLoginFlowUserRecord is a user record used by [OONIBackendWithLoginFlow].
+type OONIBackendWithLoginFlowUserRecord struct {
+ Expire time.Time
+ Password string
+ Token string
+}
+
+// OONIBackendWithLoginFlow implements the register and login workflows
+// and serves the psiphon config and tor targets.
+//
+// The zero value is ready to use.
+//
+// This struct methods panics for several errors. Only use for testing purposes!
+type OONIBackendWithLoginFlow struct {
+ // logins maps the existing login names to the corresponding record.
+ logins map[string]*OONIBackendWithLoginFlowUserRecord
+
+ // mu provides mutual exclusion.
+ mu sync.Mutex
+
+ // psiphonConfig is the serialized psiphon config to send to authenticated clients.
+ psiphonConfig []byte
+
+ // tokens maps a token to a user record.
+ tokens map[string]*OONIBackendWithLoginFlowUserRecord
+
+ // torTargets is the serialized tor config to send to authenticated clients.
+ torTargets []byte
+}
+
+// SetPsiphonConfig sets psiphon configuration to use.
+//
+// This method is safe to call concurrently with incoming HTTP requests.
+func (h *OONIBackendWithLoginFlow) SetPsiphonConfig(config []byte) {
+ defer h.mu.Unlock()
+ h.mu.Lock()
+ h.psiphonConfig = config
+}
+
+// SetTorTargets sets tor targets to use.
+//
+// This method is safe to call concurrently with incoming HTTP requests.
+func (h *OONIBackendWithLoginFlow) SetTorTargets(config []byte) {
+ defer h.mu.Unlock()
+ h.mu.Lock()
+ h.torTargets = config
+}
+
+// DoWithLockedUserRecord performs an action with the given user record. The action will
+// run while we're holding the [*OONIBackendWithLoginFlow] mutex.
+func (h *OONIBackendWithLoginFlow) DoWithLockedUserRecord(
+ username string, fx func(rec *OONIBackendWithLoginFlowUserRecord) error) error {
+ defer h.mu.Unlock()
+ h.mu.Lock()
+ rec := h.logins[username]
+ if rec == nil {
+ return errors.New("no such record")
+ }
+ return fx(rec)
+}
+
+// NewMux constructs an [*http.ServeMux] configured with the correct routing.
+func (h *OONIBackendWithLoginFlow) NewMux() *http.ServeMux {
+ mux := http.NewServeMux()
+ mux.Handle("/api/v1/register", h.handleRegister())
+ mux.Handle("/api/v1/login", h.handleLogin())
+ mux.Handle("/api/v1/test-list/psiphon-config", h.withAuthentication(h.handlePsiphonConfig()))
+ mux.Handle("/api/v1/test-list/tor-targets", h.withAuthentication(h.handleTorTargets()))
+ return mux
+}
+
+func (h *OONIBackendWithLoginFlow) handleRegister() http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // make sure the method is OK
+ if r.Method != http.MethodPost {
+ w.WriteHeader(501)
+ return
+ }
+
+ // read the raw request body
+ rawreqbody := runtimex.Try1(io.ReadAll(r.Body))
+
+ // unmarshal the request
+ var request model.OOAPIRegisterRequest
+ must.UnmarshalJSON(rawreqbody, &request)
+
+ // lock the users database
+ h.mu.Lock()
+
+ // make sure the map is usable
+ if h.logins == nil {
+ h.logins = make(map[string]*OONIBackendWithLoginFlowUserRecord)
+ }
+
+ // create new login
+ userID := uuid.Must(uuid.NewRandom()).String()
+
+ // save login
+ h.logins[userID] = &OONIBackendWithLoginFlowUserRecord{
+ Expire: time.Time{},
+ Password: request.Password,
+ Token: "",
+ }
+
+ // unlock the users database
+ h.mu.Unlock()
+
+ // prepare response
+ response := &model.OOAPIRegisterResponse{
+ ClientID: userID,
+ }
+
+ // send response
+ _, _ = w.Write(must.MarshalJSON(response))
+ })
+}
+
+func (h *OONIBackendWithLoginFlow) handleLogin() http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // make sure the method is OK
+ if r.Method != http.MethodPost {
+ w.WriteHeader(501)
+ return
+ }
+
+ // read the raw request body
+ rawreqbody := runtimex.Try1(io.ReadAll(r.Body))
+
+ // unmarshal the request
+ var request model.OOAPILoginCredentials
+ must.UnmarshalJSON(rawreqbody, &request)
+
+ // lock the users database
+ h.mu.Lock()
+
+ // attempt to access user record
+ record := h.logins[request.Username]
+
+ // handle the case where the user does not exist
+ if record == nil {
+ // unlock the users database
+ h.mu.Unlock()
+
+ // return 401
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+
+ // handle the case where the password is invalid
+ if request.Password != record.Password {
+ // unlock the users database
+ h.mu.Unlock()
+
+ // return 401
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+
+ // create token
+ token := uuid.Must(uuid.NewRandom()).String()
+
+ // create expiry date
+ expirydate := time.Now().Add(10 * time.Minute)
+
+ // update record
+ record.Token = token
+ record.Expire = expirydate
+
+ // create the token bearer header
+ bearer := fmt.Sprintf("Bearer %s", token)
+
+ // make sure the tokens map is okay
+ if h.tokens == nil {
+ h.tokens = make(map[string]*OONIBackendWithLoginFlowUserRecord)
+ }
+
+ // update the tokens map
+ h.tokens[bearer] = record
+
+ // unlock the users database
+ h.mu.Unlock()
+
+ // prepare response
+ response := &model.OOAPILoginAuth{
+ Expire: expirydate,
+ Token: token,
+ }
+
+ // send response
+ _, _ = w.Write(must.MarshalJSON(response))
+ })
+}
+
+func (h *OONIBackendWithLoginFlow) handlePsiphonConfig() http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // make sure the method is OK
+ if r.Method != http.MethodGet {
+ w.WriteHeader(501)
+ return
+ }
+
+ // we must lock because of SetPsiphonConfig
+ h.mu.Lock()
+ _, _ = w.Write(h.psiphonConfig)
+ h.mu.Unlock()
+ })
+}
+
+func (h *OONIBackendWithLoginFlow) handleTorTargets() http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // make sure the method is OK
+ if r.Method != http.MethodGet {
+ w.WriteHeader(501)
+ return
+ }
+
+ // make sure the client has provided the right query string
+ cc := r.URL.Query().Get("country_code")
+ if cc == "" {
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+
+ // we must lock because of SetTorTargets
+ h.mu.Lock()
+ _, _ = w.Write(h.torTargets)
+ h.mu.Unlock()
+ })
+
+}
+
+func (h *OONIBackendWithLoginFlow) withAuthentication(child http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // get the authorization header
+ authorization := r.Header.Get("Authorization")
+
+ // lock the users database
+ h.mu.Lock()
+
+ // check whether we have state
+ record := h.tokens[authorization]
+
+ // handle the case of nonexisting state
+ if record == nil {
+ // unlock the users database
+ h.mu.Unlock()
+
+ // return 401
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+
+ // handle the case of expired state
+ if time.Until(record.Expire) <= 0 {
+ // unlock the users database
+ h.mu.Unlock()
+
+ // return 401
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+
+ // unlock the users database
+ h.mu.Unlock()
+
+ // defer to the child handler
+ child.ServeHTTP(w, r)
+ })
+}
diff --git a/pkg/testingx/oonibackendwithlogin_test.go b/pkg/testingx/oonibackendwithlogin_test.go
new file mode 100644
index 000000000..6aac81eb3
--- /dev/null
+++ b/pkg/testingx/oonibackendwithlogin_test.go
@@ -0,0 +1,501 @@
+package testingx
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "net/http"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/uuid"
+ "github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/must"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+ "github.com/ooni/probe-engine/pkg/urlx"
+)
+
+func TestOONIBackendWithLoginFlow(t *testing.T) {
+ // create state
+ state := &OONIBackendWithLoginFlow{}
+
+ // create local testing server
+ server := MustNewHTTPServer(state.NewMux())
+ defer server.Close()
+
+ // create a fake filler
+ ff := &FakeFiller{}
+
+ t.Run("it may be that there's no user record", func(t *testing.T) {
+ err := state.DoWithLockedUserRecord("foobar", func(rec *OONIBackendWithLoginFlowUserRecord) error {
+ panic("should not be called")
+ })
+ if err == nil || err.Error() != "no such record" {
+ t.Fatal("unexpected error", err)
+ }
+ })
+
+ t.Run("attempt login with invalid method", func(t *testing.T) {
+ // create HTTP request
+ req := runtimex.Try1(http.NewRequest(
+ "GET",
+ runtimex.Try1(urlx.ResolveReference(server.URL, "/api/v1/login", "")),
+ nil,
+ ))
+
+ // perform the round trip
+ resp, err := http.DefaultClient.Do(req)
+
+ // we do not expect an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we eventually close the body
+ defer resp.Body.Close()
+
+ // we expect to see not implemented
+ if resp.StatusCode != http.StatusNotImplemented {
+ t.Fatal("unexpected status code", resp.StatusCode)
+ }
+ })
+
+ t.Run("attempt login with invalid credentials", func(t *testing.T) {
+ // create fake login request
+ request := &model.OOAPILoginCredentials{}
+
+ // fill it with random data
+ ff.Fill(&request)
+
+ // create HTTP request
+ req := runtimex.Try1(http.NewRequest(
+ "POST",
+ runtimex.Try1(urlx.ResolveReference(server.URL, "/api/v1/login", "")),
+ bytes.NewReader(must.MarshalJSON(request)),
+ ))
+
+ // perform the round trip
+ resp, err := http.DefaultClient.Do(req)
+
+ // we do not expect an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we eventually close the body
+ defer resp.Body.Close()
+
+ // we expect to be unauthorized
+ if resp.StatusCode != http.StatusUnauthorized {
+ t.Fatal("unexpected status code", resp.StatusCode)
+ }
+ })
+
+ t.Run("attempt register with invalid method", func(t *testing.T) {
+ // create HTTP request
+ req := runtimex.Try1(http.NewRequest(
+ "GET",
+ runtimex.Try1(urlx.ResolveReference(server.URL, "/api/v1/register", "")),
+ nil,
+ ))
+
+ // perform the round trip
+ resp, err := http.DefaultClient.Do(req)
+
+ // we do not expect an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we eventually close the body
+ defer resp.Body.Close()
+
+ // we expect to see not implemented
+ if resp.StatusCode != http.StatusNotImplemented {
+ t.Fatal("unexpected status code", resp.StatusCode)
+ }
+ })
+
+ // registerflow attempts to register and returns the username and password
+ registerflow := func(t *testing.T) (string, string) {
+ // create register request
+ //
+ // we ignore the metadata because we're testing
+ request := &model.OOAPIRegisterRequest{
+ OOAPIProbeMetadata: model.OOAPIProbeMetadata{},
+ Password: uuid.Must(uuid.NewRandom()).String(),
+ }
+
+ // create HTTP request
+ req := runtimex.Try1(http.NewRequest(
+ "POST",
+ runtimex.Try1(urlx.ResolveReference(server.URL, "/api/v1/register", "")),
+ bytes.NewReader(must.MarshalJSON(request)),
+ ))
+
+ // perform the round trip
+ resp, err := http.DefaultClient.Do(req)
+
+ // we do not expect an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we eventually close the body
+ defer resp.Body.Close()
+
+ // we expect to be authorized
+ if resp.StatusCode != http.StatusOK {
+ t.Fatal("unexpected status code", resp.StatusCode)
+ }
+
+ // read response body
+ rawrespbody := runtimex.Try1(io.ReadAll(resp.Body))
+
+ // parse the response body
+ var response model.OOAPIRegisterResponse
+ must.UnmarshalJSON(rawrespbody, &response)
+
+ // return username and password
+ return response.ClientID, request.Password
+ }
+
+ t.Run("successful register", func(t *testing.T) {
+ _, _ = registerflow(t)
+ })
+
+ loginrequest := func(username, password string) *http.Response {
+ // create login request
+ request := &model.OOAPILoginCredentials{
+ Username: username,
+ Password: password,
+ }
+
+ // create HTTP request
+ req := runtimex.Try1(http.NewRequest(
+ "POST",
+ runtimex.Try1(urlx.ResolveReference(server.URL, "/api/v1/login", "")),
+ bytes.NewReader(must.MarshalJSON(request)),
+ ))
+
+ // perform the round trip
+ resp, err := http.DefaultClient.Do(req)
+
+ // we do not expect an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ return resp
+ }
+
+ loginflow := func(username, password string) (string, time.Time) {
+ // get the response
+ resp := loginrequest(username, password)
+
+ // make sure we eventually close the body
+ defer resp.Body.Close()
+
+ // we expect to be authorized
+ if resp.StatusCode != http.StatusOK {
+ t.Fatal("unexpected status code", resp.StatusCode)
+ }
+
+ // read response body
+ rawrespbody := runtimex.Try1(io.ReadAll(resp.Body))
+
+ // parse the response body
+ var response model.OOAPILoginAuth
+ must.UnmarshalJSON(rawrespbody, &response)
+
+ // return token and expiry date
+ return response.Token, response.Expire
+ }
+
+ t.Run("successful login", func(t *testing.T) {
+ _, _ = loginflow(registerflow(t))
+ })
+
+ t.Run("login with invalid password", func(t *testing.T) {
+ // obtain the credentials
+ username, _ := registerflow(t)
+
+ // obtain the response using a completely different password
+ resp := loginrequest(username, "antani")
+
+ // make sure we eventually close the body
+ defer resp.Body.Close()
+
+ // we expect to see 401
+ if resp.StatusCode != http.StatusUnauthorized {
+ t.Fatal("unexpected status code", resp.StatusCode)
+ }
+ })
+
+ t.Run("get psiphon config with invalid method", func(t *testing.T) {
+ // obtain the token
+ token, _ := loginflow(registerflow(t))
+
+ // create HTTP request
+ req := runtimex.Try1(http.NewRequest(
+ "DELETE",
+ runtimex.Try1(urlx.ResolveReference(server.URL, "/api/v1/test-list/psiphon-config", "")),
+ nil,
+ ))
+
+ // create the authorization token
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+
+ // perform the round trip
+ resp, err := http.DefaultClient.Do(req)
+
+ // we do not expect an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we eventually close the body
+ defer resp.Body.Close()
+
+ // we expect to see not implemented
+ if resp.StatusCode != http.StatusNotImplemented {
+ t.Fatal("unexpected status code", resp.StatusCode)
+ }
+ })
+
+ t.Run("get tor targets with invalid method", func(t *testing.T) {
+ // obtain the token
+ token, _ := loginflow(registerflow(t))
+
+ // create HTTP request
+ req := runtimex.Try1(http.NewRequest(
+ "DELETE",
+ runtimex.Try1(urlx.ResolveReference(server.URL, "/api/v1/test-list/tor-targets", "")),
+ nil,
+ ))
+
+ // create the authorization token
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+
+ // perform the round trip
+ resp, err := http.DefaultClient.Do(req)
+
+ // we do not expect an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we eventually close the body
+ defer resp.Body.Close()
+
+ // we expect to see not implemented
+ if resp.StatusCode != http.StatusNotImplemented {
+ t.Fatal("unexpected status code", resp.StatusCode)
+ }
+ })
+
+ t.Run("get psiphon config with invalid token", func(t *testing.T) {
+ // create HTTP request
+ req := runtimex.Try1(http.NewRequest(
+ "GET",
+ runtimex.Try1(urlx.ResolveReference(server.URL, "/api/v1/test-list/psiphon-config", "")),
+ nil,
+ ))
+
+ // create the authorization token
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", "antani"))
+
+ // perform the round trip
+ resp, err := http.DefaultClient.Do(req)
+
+ // we do not expect an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we eventually close the body
+ defer resp.Body.Close()
+
+ // we expect to see 401
+ if resp.StatusCode != http.StatusUnauthorized {
+ t.Fatal("unexpected status code", resp.StatusCode)
+ }
+ })
+
+ t.Run("get psiphon config with expired token", func(t *testing.T) {
+ // obtain the credentials
+ username, password := registerflow(t)
+
+ // obtain the token
+ token, _ := loginflow(username, password)
+
+ // modify the token expiry time so that it's expired
+ state.DoWithLockedUserRecord(username, func(rec *OONIBackendWithLoginFlowUserRecord) error {
+ rec.Expire = time.Now().Add(-1 * time.Hour)
+ return nil
+ })
+
+ // create HTTP request
+ req := runtimex.Try1(http.NewRequest(
+ "GET",
+ runtimex.Try1(urlx.ResolveReference(server.URL, "/api/v1/test-list/psiphon-config", "")),
+ nil,
+ ))
+
+ // create the authorization token
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+
+ // perform the round trip
+ resp, err := http.DefaultClient.Do(req)
+
+ // we do not expect an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we eventually close the body
+ defer resp.Body.Close()
+
+ // we expect to see 401
+ if resp.StatusCode != http.StatusUnauthorized {
+ t.Fatal("unexpected status code", resp.StatusCode)
+ }
+ })
+
+ t.Run("we can get psiphon config", func(t *testing.T) {
+ // define the expected body
+ expectedbody := []byte(`bonsoir elliot`)
+
+ // set the config
+ state.SetPsiphonConfig(expectedbody)
+
+ // obtain the credentials
+ username, password := registerflow(t)
+
+ // obtain the token
+ token, _ := loginflow(username, password)
+
+ // create HTTP request
+ req := runtimex.Try1(http.NewRequest(
+ "GET",
+ runtimex.Try1(urlx.ResolveReference(server.URL, "/api/v1/test-list/psiphon-config", "")),
+ nil,
+ ))
+
+ // create the authorization token
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+
+ // perform the round trip
+ resp, err := http.DefaultClient.Do(req)
+
+ // we do not expect an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we eventually close the body
+ defer resp.Body.Close()
+
+ // we expect to see 200
+ if resp.StatusCode != http.StatusOK {
+ t.Fatal("unexpected status code", resp.StatusCode)
+ }
+
+ // read the full body
+ rawrespbody := runtimex.Try1(io.ReadAll(resp.Body))
+
+ // make sure we've got the expected body
+ if diff := cmp.Diff(expectedbody, rawrespbody); err != nil {
+ t.Fatal(diff)
+ }
+ })
+
+ t.Run("we can get tor targets", func(t *testing.T) {
+ // define the expected body
+ expectedbody := []byte(`bonsoir elliot`)
+
+ // set the targets
+ state.SetTorTargets(expectedbody)
+
+ // obtain the credentials
+ username, password := registerflow(t)
+
+ // obtain the token
+ token, _ := loginflow(username, password)
+
+ // create HTTP request
+ req := runtimex.Try1(http.NewRequest(
+ "GET",
+ runtimex.Try1(urlx.ResolveReference(server.URL, "/api/v1/test-list/tor-targets", "country_code=IT")),
+ nil,
+ ))
+
+ // create the authorization token
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+
+ // perform the round trip
+ resp, err := http.DefaultClient.Do(req)
+
+ // we do not expect an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we eventually close the body
+ defer resp.Body.Close()
+
+ // we expect to see 200
+ if resp.StatusCode != http.StatusOK {
+ t.Fatal("unexpected status code", resp.StatusCode)
+ }
+
+ // read the full body
+ rawrespbody := runtimex.Try1(io.ReadAll(resp.Body))
+
+ // make sure we've got the expected body
+ if diff := cmp.Diff(expectedbody, rawrespbody); err != nil {
+ t.Fatal(diff)
+ }
+ })
+
+ t.Run("we need query string to get tor targets", func(t *testing.T) {
+ // define the expected body
+ expectedbody := []byte(`bonsoir elliot`)
+
+ // set the targets
+ state.SetTorTargets(expectedbody)
+
+ // obtain the credentials
+ username, password := registerflow(t)
+
+ // obtain the token
+ token, _ := loginflow(username, password)
+
+ // create HTTP request
+ req := runtimex.Try1(http.NewRequest(
+ "GET",
+ runtimex.Try1(urlx.ResolveReference(server.URL, "/api/v1/test-list/tor-targets", "")),
+ nil,
+ ))
+
+ // create the authorization token
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+
+ // perform the round trip
+ resp, err := http.DefaultClient.Do(req)
+
+ // we do not expect an error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we eventually close the body
+ defer resp.Body.Close()
+
+ // we expect to see 400
+ if resp.StatusCode != http.StatusBadRequest {
+ t.Fatal("unexpected status code", resp.StatusCode)
+ }
+ })
+}
diff --git a/pkg/testingx/oonicollector.go b/pkg/testingx/oonicollector.go
new file mode 100644
index 000000000..499778e96
--- /dev/null
+++ b/pkg/testingx/oonicollector.go
@@ -0,0 +1,245 @@
+package testingx
+
+import (
+ "encoding/json"
+ "io"
+ "log"
+ "net/http"
+ "strings"
+ "sync"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/uuid"
+ "github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/must"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+)
+
+// OONICollector implements the OONI collector for testing.
+//
+// The zero value is ready to use.
+//
+// This struct methods panics for several errors. Only use for testing purposes!
+type OONICollector struct {
+ // EditOpenReportResponse is an OPTIONAL callback to edit the response
+ // before the server actually sends it to the client.
+ EditOpenReportResponse func(resp *model.OOAPICollectorOpenResponse)
+
+ // EditUpdateResponse is an OPTIONAL callback to edit the response
+ // before the server actually sends it to the client.
+ EditUpdateResponse func(resp *model.OOAPICollectorUpdateResponse)
+
+ // ValidateMeasurement is an OPTIONAL callback to validate the incoming measurement
+ // beyond checks that ensure it is consistent with the original template.
+ ValidateMeasurement func(meas *model.Measurement) error
+
+ // ValidateReportTemplate is an OPTIONAL callback to validate the incoming report
+ // template beyond the data format version and format fields values.
+ ValidateReportTemplate func(rt *model.OOAPIReportTemplate) error
+
+ // mu provides mutual exclusion.
+ mu sync.Mutex
+
+ // reports contains the open reports.
+ reports map[string]*model.OOAPIReportTemplate
+}
+
+// OpenReport opens a report for the given report ID and template.
+//
+// This method is safe to call concurrently with other methods.
+func (oc *OONICollector) OpenReport(reportID string, template *model.OOAPIReportTemplate) {
+ oc.mu.Lock()
+ if oc.reports == nil {
+ oc.reports = make(map[string]*model.OOAPIReportTemplate)
+ }
+ oc.reports[reportID] = template
+ oc.mu.Unlock()
+}
+
+// ServeHTTP implements [http.Handler].
+//
+// This method is safe to call concurrently with other methods.
+func (oc *OONICollector) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ // make sure that the method is POST
+ if r.Method != "POST" {
+ log.Printf("OONICollector: invalid method")
+ w.WriteHeader(http.StatusNotImplemented)
+ return
+ }
+
+ // make sure the URL path starts with /report
+ if !strings.HasPrefix(r.URL.Path, "/report") {
+ log.Printf("OONICollector: invalid URL path prefix")
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+
+ // make sure that the content-type is application/json
+ if r.Header.Get("Content-Type") != "application/json" {
+ log.Printf("OONICollector: missing content-type header")
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+
+ // read the raw request body or panic if we cannot read it
+ body := runtimex.Try1(io.ReadAll(r.Body))
+
+ log.Printf("OONICollector: URLPath %+v", r.URL.Path)
+ log.Printf("OONICollector: request body %s", string(body))
+
+ // handle the case where the user wants to open a new report
+ if r.URL.Path == "/report" {
+ log.Printf("OONICollector: opening new report")
+ oc.openReport(w, body)
+ return
+ }
+
+ // handle the case where the user wants to append to an existing report
+ log.Printf("OONICollector: updating existing report")
+ oc.updateReport(w, r.URL.Path, body)
+}
+
+// openReport handles opening a new OONI report.
+func (oc *OONICollector) openReport(w http.ResponseWriter, body []byte) {
+ // make sure we can parse the incoming request
+ var template model.OOAPIReportTemplate
+ if err := json.Unmarshal(body, &template); err != nil {
+ log.Printf("OONICollector: cannot unmarshal JSON: %s", err.Error())
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+
+ // make sure the data format version is OK
+ if template.DataFormatVersion != model.OOAPIReportDefaultDataFormatVersion {
+ log.Printf("OONICollector: invalid data format version")
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+
+ // make sure the format is also OK
+ if template.Format != model.OOAPIReportDefaultFormat {
+ log.Printf("OONICollector: invalid format")
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+
+ // optionally allow the user to validate the report template
+ if oc.ValidateReportTemplate != nil {
+ if err := oc.ValidateReportTemplate(&template); err != nil {
+ log.Printf("OONICollector: invalid report template: %s", err.Error())
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ }
+
+ // create the response
+ response := &model.OOAPICollectorOpenResponse{
+ BackendVersion: "1.3.0",
+ ReportID: uuid.Must(uuid.NewRandom()).String(),
+ SupportedFormats: []string{
+ model.OOAPIReportDefaultFormat,
+ },
+ }
+
+ // optionally allow the user to modify the response
+ if oc.EditOpenReportResponse != nil {
+ oc.EditOpenReportResponse(response)
+ }
+
+ // make sure we know that this report ID now exists - note that this must
+ // happen after the client code has edited the response
+ oc.OpenReport(response.ReportID, &template)
+
+ // set the content-type header
+ w.Header().Set("Content-Type", "application/json")
+
+ // serialize and send
+ _, _ = w.Write(must.MarshalJSON(response))
+}
+
+// updateReport handles updating an existing OONI report.
+func (oc *OONICollector) updateReport(w http.ResponseWriter, urlpath string, body []byte) {
+ // get the report ID
+ reportID := strings.TrimPrefix(urlpath, "/report/")
+
+ // obtain the report template
+ oc.mu.Lock()
+ template := oc.reports[reportID]
+ oc.mu.Unlock()
+
+ // handle the case of missing template
+ if template == nil {
+ log.Printf("OONICollector: the report does not exist: %s", reportID)
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+
+ // make sure we can parse the incoming request
+ var request model.OOAPICollectorUpdateRequest
+ if err := json.Unmarshal(body, &request); err != nil {
+ log.Printf("OONICollector: cannot unmarshal JSON: %s", err.Error())
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+
+ // make sure the measurement is encoded as JSON
+ if request.Format != "json" {
+ log.Printf("OONICollector: invalid request format: %s", request.Format)
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+
+ // make sure we can parse the content
+ //
+ // note: we unmarshaled into a map[string]any so we need to marshal
+ // and unmarshal again to get a measurement structure
+ var measurement model.Measurement
+ if err := json.Unmarshal(must.MarshalJSON(request.Content), &measurement); err != nil {
+ log.Printf("OONICollector: cannot unmarshal JSON: %s", err.Error())
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+
+ // make sure all the required fields match
+ mt := &model.OOAPIReportTemplate{
+ DataFormatVersion: measurement.DataFormatVersion,
+ Format: request.Format,
+ ProbeASN: measurement.ProbeASN,
+ ProbeCC: measurement.ProbeCC,
+ SoftwareName: measurement.SoftwareName,
+ SoftwareVersion: measurement.SoftwareVersion,
+ TestName: measurement.TestName,
+ TestStartTime: measurement.TestStartTime,
+ TestVersion: measurement.TestVersion,
+ }
+ if diff := cmp.Diff(template, mt); diff != "" {
+ log.Printf("OONICollector: measurement differs from template %s", diff)
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+
+ // give the user a chance to validate the measurement
+ if oc.ValidateMeasurement != nil {
+ if err := oc.ValidateMeasurement(&measurement); err != nil {
+ log.Printf("OONICollector: invalid measurement: %s", err.Error())
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ }
+
+ // create the response
+ response := &model.OOAPICollectorUpdateResponse{
+ MeasurementUID: uuid.Must(uuid.NewRandom()).String(),
+ }
+
+ // optionally allow the user to modify the response
+ if oc.EditUpdateResponse != nil {
+ oc.EditUpdateResponse(response)
+ }
+
+ // set the content-type header
+ w.Header().Set("Content-Type", "application/json")
+
+ // serialize and send
+ _, _ = w.Write(must.MarshalJSON(response))
+}
diff --git a/pkg/testingx/oonicollector_test.go b/pkg/testingx/oonicollector_test.go
new file mode 100644
index 000000000..6b1303ad6
--- /dev/null
+++ b/pkg/testingx/oonicollector_test.go
@@ -0,0 +1,891 @@
+package testingx
+
+import (
+ "bytes"
+ "errors"
+ "io"
+ "net/http"
+ "net/url"
+ "slices"
+ "sync"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/must"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+)
+
+// This function tests the OONICollector type.
+func TestOONICollector(t *testing.T) {
+ t.Run("common: when method is not POST", func(t *testing.T) {
+ // create and expose the testing collector
+ collector := &OONICollector{}
+ srv := MustNewHTTPServer(collector)
+ defer srv.Close()
+
+ // create request
+ req := runtimex.Try1(http.NewRequest("GET", srv.URL, nil))
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 501
+ if resp.StatusCode != http.StatusNotImplemented {
+ t.Fatal("unexpected status code")
+ }
+ })
+
+ t.Run("common: when the URL path does not start with report", func(t *testing.T) {
+ // create and expose the testing collector
+ collector := &OONICollector{}
+ srv := MustNewHTTPServer(collector)
+ defer srv.Close()
+
+ // create request
+ //
+ // note: the server URL has / as its path so this URL is good to go
+ req := runtimex.Try1(http.NewRequest("POST", srv.URL, nil))
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 400
+ if resp.StatusCode != http.StatusBadRequest {
+ t.Fatal("unexpected status code")
+ }
+ })
+
+ t.Run("common: when the Content-Type header is missing", func(t *testing.T) {
+ // create and expose the testing collector
+ collector := &OONICollector{}
+ srv := MustNewHTTPServer(collector)
+ defer srv.Close()
+
+ // rewrite the URL to have a path starting with /report
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ URL.Path = "/report"
+
+ // create request
+ //
+ // note: the request has no Content-Type so we should be good here
+ req := runtimex.Try1(http.NewRequest("POST", URL.String(), nil))
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 400
+ if resp.StatusCode != http.StatusBadRequest {
+ t.Fatal("unexpected status code")
+ }
+ })
+
+ t.Run("openReport: when we cannot unmarshal the body", func(t *testing.T) {
+ // create and expose the testing collector
+ collector := &OONICollector{}
+ srv := MustNewHTTPServer(collector)
+ defer srv.Close()
+
+ // rewrite the URL to be exactly /report
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ URL.Path = "/report"
+
+ // create request
+ //
+ // note: the body is empty so parsing should fail
+ req := runtimex.Try1(http.NewRequest("POST", URL.String(), nil))
+
+ // make sure there's content-type
+ req.Header.Set("Content-Type", "application/json")
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 400
+ if resp.StatusCode != http.StatusBadRequest {
+ t.Fatal("unexpected status code")
+ }
+ })
+
+ t.Run("openReport: with invalid data format version", func(t *testing.T) {
+ // create and expose the testing collector
+ collector := &OONICollector{}
+ srv := MustNewHTTPServer(collector)
+ defer srv.Close()
+
+ // rewrite the URL to be exactly /report
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ URL.Path = "/report"
+
+ // create the request body
+ //
+ // note: we're using an invalid data format version to trigger error
+ const invalidDataFormatVersion = "0.3.0"
+ request := &model.OOAPIReportTemplate{
+ DataFormatVersion: invalidDataFormatVersion,
+ }
+ rawreqbody := must.MarshalJSON(request)
+
+ // create request
+ req := runtimex.Try1(http.NewRequest("POST", URL.String(), bytes.NewReader(rawreqbody)))
+
+ // make sure there's content-type
+ req.Header.Set("Content-Type", "application/json")
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 400
+ if resp.StatusCode != http.StatusBadRequest {
+ t.Fatal("unexpected status code")
+ }
+ })
+
+ t.Run("openReport: with invalid format", func(t *testing.T) {
+ // create and expose the testing collector
+ collector := &OONICollector{}
+ srv := MustNewHTTPServer(collector)
+ defer srv.Close()
+
+ // rewrite the URL to be exactly /report
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ URL.Path = "/report"
+
+ // create the request body
+ //
+ // note: we're using an invalid format to trigger error
+ const validDataFormatVersion = "0.2.0"
+ request := &model.OOAPIReportTemplate{
+ DataFormatVersion: validDataFormatVersion,
+ Format: "yaml",
+ }
+ rawreqbody := must.MarshalJSON(request)
+
+ // create request
+ req := runtimex.Try1(http.NewRequest("POST", URL.String(), bytes.NewReader(rawreqbody)))
+
+ // make sure there's content-type
+ req.Header.Set("Content-Type", "application/json")
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 400
+ if resp.StatusCode != http.StatusBadRequest {
+ t.Fatal("unexpected status code")
+ }
+ })
+
+ t.Run("openReport: we can invoke a callback to see the incoming template", func(t *testing.T) {
+ // create and expose the testing collector
+ collector := &OONICollector{}
+ srv := MustNewHTTPServer(collector)
+ defer srv.Close()
+
+ // rewrite the URL to be exactly /report
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ URL.Path = "/report"
+
+ // create the request body
+ //
+ // note: we're using the fake filler to randomly fill and then we're
+ // editing to avoid failures, but this means we have a request that we
+ // can later compare to using google/cmp-go/cmp.Diff.
+ const validDataFormatVersion = "0.2.0"
+ request := &model.OOAPIReportTemplate{}
+ ff := &FakeFiller{}
+ ff.Fill(&request)
+ request.DataFormatVersion = validDataFormatVersion
+ request.Format = "json"
+ rawreqbody := must.MarshalJSON(request)
+
+ // create request
+ req := runtimex.Try1(http.NewRequest("POST", URL.String(), bytes.NewReader(rawreqbody)))
+
+ // make sure there's content-type
+ req.Header.Set("Content-Type", "application/json")
+
+ // prepare to obtain the incoming report template
+ mu := &sync.Mutex{}
+ var incoming *model.OOAPIReportTemplate
+
+ // set the callback to verify the request
+ //
+ // we save the original request and return an error to trigger failure
+ collector.ValidateReportTemplate = func(rt *model.OOAPIReportTemplate) error {
+ mu.Lock()
+ incoming = rt
+ mu.Unlock()
+ return errors.New("mocked error")
+ }
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 400
+ if resp.StatusCode != http.StatusBadRequest {
+ t.Fatal("unexpected status code")
+ }
+
+ // make sure we got what we sent
+ if diff := cmp.Diff(request, incoming); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+
+ t.Run("openReport: we can edit the outgoing response", func(t *testing.T) {
+ // create and expose the testing collector
+ collector := &OONICollector{}
+ srv := MustNewHTTPServer(collector)
+ defer srv.Close()
+
+ // rewrite the URL to be exactly /report
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ URL.Path = "/report"
+
+ // create the request body
+ const validDataFormatVersion = "0.2.0"
+ request := &model.OOAPIReportTemplate{}
+ ff := &FakeFiller{}
+ ff.Fill(&request)
+ request.DataFormatVersion = validDataFormatVersion
+ request.Format = "json"
+ rawreqbody := must.MarshalJSON(request)
+
+ // create request
+ req := runtimex.Try1(http.NewRequest("POST", URL.String(), bytes.NewReader(rawreqbody)))
+
+ // make sure there's content-type
+ req.Header.Set("Content-Type", "application/json")
+
+ // make sure we can edit the response
+ collector.EditOpenReportResponse = func(resp *model.OOAPICollectorOpenResponse) {
+ resp.BackendVersion = "antani-antani-antani"
+ }
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 200
+ if resp.StatusCode != http.StatusOK {
+ t.Fatal("unexpected status code")
+ }
+
+ // read response body
+ rawrespbody := runtimex.Try1(io.ReadAll(resp.Body))
+ var response model.OOAPICollectorOpenResponse
+ must.UnmarshalJSON(rawrespbody, &response)
+
+ // make sure we've got the edited BackendVersion
+ if response.BackendVersion != "antani-antani-antani" {
+ t.Fatal("did not edit the response")
+ }
+ })
+
+ t.Run("openReport: we get a reportID back and format=json", func(t *testing.T) {
+ // create and expose the testing collector
+ collector := &OONICollector{}
+ srv := MustNewHTTPServer(collector)
+ defer srv.Close()
+
+ // rewrite the URL to be exactly /report
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ URL.Path = "/report"
+
+ // create the request body
+ const validDataFormatVersion = "0.2.0"
+ request := &model.OOAPIReportTemplate{}
+ ff := &FakeFiller{}
+ ff.Fill(&request)
+ request.DataFormatVersion = validDataFormatVersion
+ request.Format = "json"
+ rawreqbody := must.MarshalJSON(request)
+
+ // create request
+ req := runtimex.Try1(http.NewRequest("POST", URL.String(), bytes.NewReader(rawreqbody)))
+
+ // make sure there's content-type
+ req.Header.Set("Content-Type", "application/json")
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 200
+ if resp.StatusCode != http.StatusOK {
+ t.Fatal("unexpected status code")
+ }
+
+ // read response body
+ rawrespbody := runtimex.Try1(io.ReadAll(resp.Body))
+ var response model.OOAPICollectorOpenResponse
+ must.UnmarshalJSON(rawrespbody, &response)
+
+ // make sure the fields are okay
+ if response.BackendVersion != "1.3.0" {
+ t.Fatal("unexpected backend version")
+ }
+ if response.ReportID == "" {
+ t.Fatal("empty report ID")
+ }
+ if !slices.Contains(response.SupportedFormats, "json") {
+ t.Fatal("SupportedFormats does not contain the json format")
+ }
+ })
+
+ // This is a convenience function to open a report before submitting
+ openreport := func(t *testing.T, stringURL string) (*model.OOAPIReportTemplate, *model.OOAPICollectorOpenResponse) {
+ // rewrite the URL to be exactly /report
+ URL := runtimex.Try1(url.Parse(stringURL))
+ URL.Path = "/report"
+
+ // create the request body
+ const validDataFormatVersion = "0.2.0"
+ request := &model.OOAPIReportTemplate{}
+ ff := &FakeFiller{}
+ ff.Fill(&request)
+ request.DataFormatVersion = validDataFormatVersion
+ request.Format = "json"
+ rawreqbody := must.MarshalJSON(request)
+
+ // create request
+ req := runtimex.Try1(http.NewRequest("POST", URL.String(), bytes.NewReader(rawreqbody)))
+
+ // make sure there's content-type
+ req.Header.Set("Content-Type", "application/json")
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 200
+ if resp.StatusCode != http.StatusOK {
+ t.Fatal("unexpected status code")
+ }
+
+ // read response body
+ rawrespbody := runtimex.Try1(io.ReadAll(resp.Body))
+ var response model.OOAPICollectorOpenResponse
+ must.UnmarshalJSON(rawrespbody, &response)
+
+ return request, &response
+ }
+
+ t.Run("submit: when the report ID does not exist", func(t *testing.T) {
+ // create and expose the testing collector
+ collector := &OONICollector{}
+ srv := MustNewHTTPServer(collector)
+ defer srv.Close()
+
+ // rewrite the URL to be exactly /report/${reportID}
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ URL.Path = "/report/" + "blah"
+
+ // create request
+ //
+ // note: the body is empty so parsing should fail
+ req := runtimex.Try1(http.NewRequest("POST", URL.String(), nil))
+
+ // make sure there's content-type
+ req.Header.Set("Content-Type", "application/json")
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 400
+ if resp.StatusCode != http.StatusBadRequest {
+ t.Fatal("unexpected status code")
+ }
+ })
+
+ t.Run("submit: when we cannot unmarshal the body", func(t *testing.T) {
+ // create and expose the testing collector
+ collector := &OONICollector{}
+ srv := MustNewHTTPServer(collector)
+ defer srv.Close()
+
+ // first of all let's open a report
+ _, reportInfo := openreport(t, srv.URL)
+
+ // rewrite the URL to be exactly /report/${reportID}
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ URL.Path = "/report/" + reportInfo.ReportID
+
+ // create request
+ //
+ // note: the body is empty so parsing should fail
+ req := runtimex.Try1(http.NewRequest("POST", URL.String(), nil))
+
+ // make sure there's content-type
+ req.Header.Set("Content-Type", "application/json")
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 400
+ if resp.StatusCode != http.StatusBadRequest {
+ t.Fatal("unexpected status code")
+ }
+ })
+
+ t.Run("submit: with invalid format", func(t *testing.T) {
+ // create and expose the testing collector
+ collector := &OONICollector{}
+ srv := MustNewHTTPServer(collector)
+ defer srv.Close()
+
+ // first of all let's open a report
+ _, reportInfo := openreport(t, srv.URL)
+
+ // rewrite the URL to be exactly /report/${reportID}
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ URL.Path = "/report/" + reportInfo.ReportID
+
+ // create the request body
+ //
+ // note: the YAML format here is invalid
+ request := &model.OOAPICollectorUpdateRequest{
+ Format: "yaml",
+ Content: nil,
+ }
+ rawrequestbody := must.MarshalJSON(request)
+
+ // create request
+ req := runtimex.Try1(http.NewRequest("POST", URL.String(), bytes.NewReader(rawrequestbody)))
+
+ // make sure there's content-type
+ req.Header.Set("Content-Type", "application/json")
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 400
+ if resp.StatusCode != http.StatusBadRequest {
+ t.Fatal("unexpected status code")
+ }
+ })
+
+ t.Run("submit: when we cannot unmarshal the nested inside body", func(t *testing.T) {
+ // create and expose the testing collector
+ collector := &OONICollector{}
+ srv := MustNewHTTPServer(collector)
+ defer srv.Close()
+
+ // first of all let's open a report
+ _, reportInfo := openreport(t, srv.URL)
+
+ // rewrite the URL to be exactly /report/${reportID}
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ URL.Path = "/report/" + reportInfo.ReportID
+
+ // create the request body
+ //
+ // note: the content is empty so we cannot parse a JSON
+ request := &model.OOAPICollectorUpdateRequest{
+ Format: "json",
+ Content: "",
+ }
+ rawrequestbody := must.MarshalJSON(request)
+
+ // create request
+ req := runtimex.Try1(http.NewRequest("POST", URL.String(), bytes.NewReader(rawrequestbody)))
+
+ // make sure there's content-type
+ req.Header.Set("Content-Type", "application/json")
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 400
+ if resp.StatusCode != http.StatusBadRequest {
+ t.Fatal("unexpected status code")
+ }
+ })
+
+ t.Run("submit: when the template does not match the expectations", func(t *testing.T) {
+ // create and expose the testing collector
+ collector := &OONICollector{}
+ srv := MustNewHTTPServer(collector)
+ defer srv.Close()
+
+ // first of all let's open a report
+ template, reportInfo := openreport(t, srv.URL)
+
+ // rewrite the URL to be exactly /report/${reportID}
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ URL.Path = "/report/" + reportInfo.ReportID
+
+ // create the measurement
+ //
+ // note: we're changing the test name here
+ measurement := &model.Measurement{
+ DataFormatVersion: template.DataFormatVersion,
+ InputHashes: []string{},
+ MeasurementStartTime: template.TestStartTime,
+ ProbeASN: template.ProbeASN,
+ ProbeCC: template.ProbeCC,
+ ReportID: reportInfo.ReportID,
+ TestKeys: nil,
+ TestName: template.TestName + "blahblah",
+ TestStartTime: template.TestStartTime,
+ TestVersion: template.TestVersion,
+ }
+
+ // create the request body
+ request := &model.OOAPICollectorUpdateRequest{
+ Format: "json",
+ Content: measurement,
+ }
+ rawrequestbody := must.MarshalJSON(request)
+
+ // create request
+ req := runtimex.Try1(http.NewRequest("POST", URL.String(), bytes.NewReader(rawrequestbody)))
+
+ // make sure there's content-type
+ req.Header.Set("Content-Type", "application/json")
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 400
+ if resp.StatusCode != http.StatusBadRequest {
+ t.Fatal("unexpected status code")
+ }
+ })
+
+ t.Run("submit: we can invoke a callback to further validate the measurement", func(t *testing.T) {
+ // create and expose the testing collector
+ collector := &OONICollector{}
+ srv := MustNewHTTPServer(collector)
+ defer srv.Close()
+
+ // first of all let's open a report
+ template, reportInfo := openreport(t, srv.URL)
+
+ // rewrite the URL to be exactly /report/${reportID}
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ URL.Path = "/report/" + reportInfo.ReportID
+
+ // create the measurement
+ measurement := &model.Measurement{
+ DataFormatVersion: template.DataFormatVersion,
+ MeasurementStartTime: template.TestStartTime,
+ ProbeASN: template.ProbeASN,
+ ProbeCC: template.ProbeCC,
+ ReportID: reportInfo.ReportID,
+ SoftwareName: template.SoftwareName,
+ SoftwareVersion: template.SoftwareVersion,
+ TestKeys: nil,
+ TestName: template.TestName,
+ TestStartTime: template.TestStartTime,
+ TestVersion: template.TestVersion,
+ }
+
+ // create the request body
+ request := &model.OOAPICollectorUpdateRequest{
+ Format: "json",
+ Content: measurement,
+ }
+ rawrequestbody := must.MarshalJSON(request)
+
+ // create request
+ req := runtimex.Try1(http.NewRequest("POST", URL.String(), bytes.NewReader(rawrequestbody)))
+
+ // make sure there's content-type
+ req.Header.Set("Content-Type", "application/json")
+
+ // prepare to obtain the incoming measurement
+ mu := &sync.Mutex{}
+ var incoming *model.Measurement
+
+ // setup a callback so we can get the incoming measurement
+ //
+ // note: here we return an error to make the API fail but we save the measurement for later
+ collector.ValidateMeasurement = func(meas *model.Measurement) error {
+ mu.Lock()
+ incoming = meas
+ mu.Unlock()
+ return errors.New("mocked error")
+ }
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 400
+ if resp.StatusCode != http.StatusBadRequest {
+ t.Fatal("unexpected status code")
+ }
+
+ // make sure the measurement received by the API is the expected one
+ if diff := cmp.Diff(measurement, incoming); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+
+ t.Run("submit: we can edit the outgoing response", func(t *testing.T) {
+ // create and expose the testing collector
+ collector := &OONICollector{}
+ srv := MustNewHTTPServer(collector)
+ defer srv.Close()
+
+ // first of all let's open a report
+ template, reportInfo := openreport(t, srv.URL)
+
+ // rewrite the URL to be exactly /report/${reportID}
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ URL.Path = "/report/" + reportInfo.ReportID
+
+ // create the measurement
+ measurement := &model.Measurement{
+ DataFormatVersion: template.DataFormatVersion,
+ MeasurementStartTime: template.TestStartTime,
+ ProbeASN: template.ProbeASN,
+ ProbeCC: template.ProbeCC,
+ ReportID: reportInfo.ReportID,
+ SoftwareName: template.SoftwareName,
+ SoftwareVersion: template.SoftwareVersion,
+ TestKeys: nil,
+ TestName: template.TestName,
+ TestStartTime: template.TestStartTime,
+ TestVersion: template.TestVersion,
+ }
+
+ // create the request body
+ request := &model.OOAPICollectorUpdateRequest{
+ Format: "json",
+ Content: measurement,
+ }
+ rawrequestbody := must.MarshalJSON(request)
+
+ // create request
+ req := runtimex.Try1(http.NewRequest("POST", URL.String(), bytes.NewReader(rawrequestbody)))
+
+ // make sure there's content-type
+ req.Header.Set("Content-Type", "application/json")
+
+ // make sure we can edit the response
+ collector.EditUpdateResponse = func(resp *model.OOAPICollectorUpdateResponse) {
+ resp.MeasurementUID = "blablah"
+ }
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 200
+ if resp.StatusCode != http.StatusOK {
+ t.Fatal("unexpected status code")
+ }
+
+ // read and parse response body
+ rawrespbody := runtimex.Try1(io.ReadAll(resp.Body))
+ var response model.OOAPICollectorUpdateResponse
+ must.UnmarshalJSON(rawrespbody, &response)
+
+ // make sure the measurement UID has been edited
+ if response.MeasurementUID != "blablah" {
+ t.Fatal("did not exit the measurement UID")
+ }
+ })
+
+ t.Run("submit: we get a measurement ID back", func(t *testing.T) {
+ // create and expose the testing collector
+ collector := &OONICollector{}
+ srv := MustNewHTTPServer(collector)
+ defer srv.Close()
+
+ // first of all let's open a report
+ template, reportInfo := openreport(t, srv.URL)
+
+ // rewrite the URL to be exactly /report/${reportID}
+ URL := runtimex.Try1(url.Parse(srv.URL))
+ URL.Path = "/report/" + reportInfo.ReportID
+
+ // create the measurement
+ measurement := &model.Measurement{
+ DataFormatVersion: template.DataFormatVersion,
+ MeasurementStartTime: template.TestStartTime,
+ ProbeASN: template.ProbeASN,
+ ProbeCC: template.ProbeCC,
+ ReportID: reportInfo.ReportID,
+ SoftwareName: template.SoftwareName,
+ SoftwareVersion: template.SoftwareVersion,
+ TestKeys: nil,
+ TestName: template.TestName,
+ TestStartTime: template.TestStartTime,
+ TestVersion: template.TestVersion,
+ }
+
+ // create the request body
+ request := &model.OOAPICollectorUpdateRequest{
+ Format: "json",
+ Content: measurement,
+ }
+ rawrequestbody := must.MarshalJSON(request)
+
+ // create request
+ req := runtimex.Try1(http.NewRequest("POST", URL.String(), bytes.NewReader(rawrequestbody)))
+
+ // make sure there's content-type
+ req.Header.Set("Content-Type", "application/json")
+
+ // issue the request
+ resp, err := http.DefaultClient.Do(req)
+
+ // we don't expect error
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure we close the body
+ defer resp.Body.Close()
+
+ // we expect to see 200
+ if resp.StatusCode != http.StatusOK {
+ t.Fatal("unexpected status code")
+ }
+
+ // read and parse response body
+ rawrespbody := runtimex.Try1(io.ReadAll(resp.Body))
+ var response model.OOAPICollectorUpdateResponse
+ must.UnmarshalJSON(rawrespbody, &response)
+
+ // make sure the measurement UID is not empty
+ if response.MeasurementUID == "" {
+ t.Fatal("the measurement UID is unexpectedly empty")
+ }
+ })
+}
diff --git a/pkg/testingx/tcpx.go b/pkg/testingx/tcpx.go
index ad22d0a39..0e0fcb291 100644
--- a/pkg/testingx/tcpx.go
+++ b/pkg/testingx/tcpx.go
@@ -36,11 +36,11 @@ func tcpMaybeResetNetConn(conn net.Conn) {
SetLinger(sec int) error
}
if setter, good := conn.(connLingerSetter); good {
- setter.SetLinger(0)
+ _ = setter.SetLinger(0)
}
// close the conn to trigger the reset (we MUST call Close here where
// we're using the underlying conn and it doesn't suffice to call it
// inside the http.Handler, where wrapping would not cause a RST)
- conn.Close()
+ _ = conn.Close()
}
diff --git a/pkg/testingx/tlssniproxy.go b/pkg/testingx/tlssniproxy.go
index 10a0f5c91..393cd174c 100644
--- a/pkg/testingx/tlssniproxy.go
+++ b/pkg/testingx/tlssniproxy.go
@@ -126,5 +126,5 @@ func (tp *TLSSNIProxy) handle(clientConn net.Conn) {
func (tp *TLSSNIProxy) forward(wg *sync.WaitGroup, left, right net.Conn) {
defer wg.Done()
- io.Copy(right, left)
+ _, _ = io.Copy(right, left)
}
diff --git a/pkg/testingx/tlsx.go b/pkg/testingx/tlsx.go
index c75d82755..23f6a9edb 100644
--- a/pkg/testingx/tlsx.go
+++ b/pkg/testingx/tlsx.go
@@ -124,7 +124,7 @@ func (p *TLSServer) handle(ctx context.Context, tcpConn net.Conn) {
defer tcpConn.Close()
// create TLS configuration where the handler is responsible for continuing the handshake
- tlsConfig := &tls.Config{
+ tlsConfig := &tls.Config{ // #nosec G402 - code used for testing
GetCertificate: func(chi *tls.ClientHelloInfo) (*tls.Certificate, error) {
return p.handler.GetCertificate(ctx, tcpConn, chi)
},
@@ -214,7 +214,7 @@ type tlsHandlerEOF struct{}
// GetCertificate implements TLSHandler.
func (*tlsHandlerEOF) GetCertificate(ctx context.Context, tcpConn net.Conn, chi *tls.ClientHelloInfo) (*tls.Certificate, error) {
- tcpConn.Close() // close the TCP connection to force EOF during the handshake
+ _ = tcpConn.Close() // close the TCP connection to force EOF during the handshake
return nil, errors.New("internal error")
}
@@ -230,7 +230,7 @@ type tlsHandlerReset struct{}
// GetCertificate implements TLSHandler.
func (*tlsHandlerReset) GetCertificate(ctx context.Context, tcpConn net.Conn, chi *tls.ClientHelloInfo) (*tls.Certificate, error) {
tcpMaybeResetNetConn(tcpConn)
- tcpConn.Close() // just in case to avoid the error returned here to be sent remotely as an alert
+ _ = tcpConn.Close() // just in case to avoid the error returned here to be sent remotely as an alert
return nil, errors.New("internal error")
}
diff --git a/pkg/torlogs/torlogs.go b/pkg/torlogs/torlogs.go
index 01f84dcb1..85c9afd34 100644
--- a/pkg/torlogs/torlogs.go
+++ b/pkg/torlogs/torlogs.go
@@ -49,7 +49,7 @@ func ReadBootstrapLogs(logFilePath string) ([]string, error) {
if logFilePath == "" {
return nil, ErrEmptyLogFilePath
}
- data, err := os.ReadFile(logFilePath)
+ data, err := os.ReadFile(logFilePath) // #nosec G304 - this is working as intended
if err != nil {
return nil, fmt.Errorf("%w: %s", ErrCannotReadLogFile, err.Error())
}
diff --git a/pkg/tunnel/fake.go b/pkg/tunnel/fake.go
index 327a13230..6afa49cd8 100644
--- a/pkg/tunnel/fake.go
+++ b/pkg/tunnel/fake.go
@@ -27,7 +27,7 @@ func (t *fakeTunnel) BootstrapTime() time.Duration {
func (t *fakeTunnel) Stop() {
// Implementation note: closing the listener causes
// the socks5 server.Serve to return an error
- t.once.Do(func() { t.listener.Close() })
+ t.once.Do(func() { _ = t.listener.Close() })
}
// SOCKS5ProxyURL returns the SOCKS5 proxy URL.
diff --git a/pkg/tunnel/tor.go b/pkg/tunnel/tor.go
index 390eedbee..8f88cdd4f 100644
--- a/pkg/tunnel/tor.go
+++ b/pkg/tunnel/tor.go
@@ -46,7 +46,7 @@ func (tt *torTunnel) SOCKS5ProxyURL() *url.URL {
// Stop stops the Tor tunnel
func (tt *torTunnel) Stop() {
- tt.instance.Close()
+ _ = tt.instance.Close()
}
// ErrTorUnableToGetSOCKSProxyAddress indicates that we could not
@@ -99,23 +99,23 @@ func torStart(ctx context.Context, config *Config) (Tunnel, DebugInfo, error) {
instance.StopProcessOnClose = true
start := time.Now()
if err := config.torEnableNetwork(ctx, instance, true); err != nil {
- instance.Close()
+ _ = instance.Close()
return nil, debugInfo, err
}
stop := time.Now()
// Adapted from
info, err := config.torGetInfo(instance.Control, "net/listeners/socks")
if err != nil {
- instance.Close()
+ _ = instance.Close()
return nil, debugInfo, err
}
if len(info) != 1 || info[0].Key != "net/listeners/socks" {
- instance.Close()
+ _ = instance.Close()
return nil, debugInfo, ErrTorUnableToGetSOCKSProxyAddress
}
proxyAddress := info[0].Val
if strings.HasPrefix(proxyAddress, "unix:") {
- instance.Close()
+ _ = instance.Close()
return nil, debugInfo, ErrTorReturnedUnsupportedProxy
}
return &torTunnel{
@@ -128,7 +128,7 @@ func torStart(ctx context.Context, config *Config) (Tunnel, DebugInfo, error) {
// maybeCleanupTunnelDir removes stale files inside
// of the tunnel directory.
func maybeCleanupTunnelDir(dir, logfile string) {
- os.Remove(logfile)
+ _ = os.Remove(logfile)
removeWithGlob(filepath.Join(dir, "torrc-*"))
removeWithGlob(filepath.Join(dir, "control-port-*"))
}
@@ -137,6 +137,6 @@ func maybeCleanupTunnelDir(dir, logfile string) {
func removeWithGlob(pattern string) {
files, _ := filepath.Glob(pattern)
for _, file := range files {
- os.Remove(file)
+ _ = os.Remove(file)
}
}
diff --git a/pkg/tutorial/generator/main.go b/pkg/tutorial/generator/main.go
index 6ae7009e2..08bee77d3 100644
--- a/pkg/tutorial/generator/main.go
+++ b/pkg/tutorial/generator/main.go
@@ -21,7 +21,7 @@ func writeString(w io.Writer, s string) {
// gen1 generates a single file within a chapter.
func gen1(destfile io.Writer, filepath string) {
- srcfile, err := os.Open(filepath)
+ srcfile, err := os.Open(filepath) // #nosec G304 - this is working as intended
if err != nil {
log.Fatal(err)
}
@@ -67,7 +67,7 @@ func gen1(destfile io.Writer, filepath string) {
// gen("./experiment/torsf/chapter01", "main.go")
func gen(dirpath string, files ...string) {
readme := path.Join(dirpath, "README.md")
- destfile, err := os.Create(path.Join(readme))
+ destfile, err := os.Create(path.Join(readme)) // #nosec G304 - this is working as intended
if err != nil {
log.Fatal(err)
}
diff --git a/pkg/tutorial/measurex/chapter04/README.md b/pkg/tutorial/measurex/chapter04/README.md
index c308a126a..8bac14cc1 100644
--- a/pkg/tutorial/measurex/chapter04/README.md
+++ b/pkg/tutorial/measurex/chapter04/README.md
@@ -49,7 +49,7 @@ we have seen in chapter02, using the address argument. Then, if
successful, it will TLS handshake using the given TLS config.
```Go
- m := mx.TLSConnectAndHandshake(ctx, *address, &tls.Config{
+ m := mx.TLSConnectAndHandshake(ctx, *address, &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
ServerName: *sni,
NextProtos: []string{"h2", "http/1.1"},
RootCAs: nil, // use netxlite's default
diff --git a/pkg/tutorial/measurex/chapter04/main.go b/pkg/tutorial/measurex/chapter04/main.go
index d55a42f4d..baadea9da 100644
--- a/pkg/tutorial/measurex/chapter04/main.go
+++ b/pkg/tutorial/measurex/chapter04/main.go
@@ -50,7 +50,7 @@ func main() {
// successful, it will TLS handshake using the given TLS config.
//
// ```Go
- m := mx.TLSConnectAndHandshake(ctx, *address, &tls.Config{
+ m := mx.TLSConnectAndHandshake(ctx, *address, &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
ServerName: *sni,
NextProtos: []string{"h2", "http/1.1"},
RootCAs: nil, // use netxlite's default
diff --git a/pkg/tutorial/measurex/chapter05/README.md b/pkg/tutorial/measurex/chapter05/README.md
index 7c5cb4468..8aabfce2f 100644
--- a/pkg/tutorial/measurex/chapter05/README.md
+++ b/pkg/tutorial/measurex/chapter05/README.md
@@ -51,7 +51,7 @@ The API signature is indeed the same as the previous chapter,
except that here we call the `QUICHandshake` function.
```Go
- m := mx.QUICHandshake(ctx, *address, &tls.Config{
+ m := mx.QUICHandshake(ctx, *address, &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
ServerName: *sni,
NextProtos: []string{"h3"},
RootCAs: nil, // use netxlite's default
diff --git a/pkg/tutorial/measurex/chapter05/main.go b/pkg/tutorial/measurex/chapter05/main.go
index 96a1c7f7d..ce71585e2 100644
--- a/pkg/tutorial/measurex/chapter05/main.go
+++ b/pkg/tutorial/measurex/chapter05/main.go
@@ -52,7 +52,7 @@ func main() {
// except that here we call the `QUICHandshake` function.
//
// ```Go
- m := mx.QUICHandshake(ctx, *address, &tls.Config{
+ m := mx.QUICHandshake(ctx, *address, &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
ServerName: *sni,
NextProtos: []string{"h3"},
RootCAs: nil, // use netxlite's default
diff --git a/pkg/tutorial/measurex/chapter14/README.md b/pkg/tutorial/measurex/chapter14/README.md
index 9c38fbff0..9300d0df6 100644
--- a/pkg/tutorial/measurex/chapter14/README.md
+++ b/pkg/tutorial/measurex/chapter14/README.md
@@ -131,7 +131,7 @@ whether the input URL is HTTP or HTTPS.
m.TCPConnect = append(
m.TCPConnect, measurex.NewArchivalTCPConnectList(tcp.Connect)...)
case "https":
- config := &tls.Config{
+ config := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
ServerName: parsedURL.Hostname(),
NextProtos: []string{"h2", "http/1.1"},
RootCAs: nil, // use netxlite's default
@@ -210,7 +210,7 @@ using an HTTP transport reading a body snapshot.
```Go
if resp != nil {
- resp.Body.Close() // tidy
+ _ = resp.Body.Close() // tidy
}
```
diff --git a/pkg/tutorial/measurex/chapter14/main.go b/pkg/tutorial/measurex/chapter14/main.go
index d85a60870..eb0422d67 100644
--- a/pkg/tutorial/measurex/chapter14/main.go
+++ b/pkg/tutorial/measurex/chapter14/main.go
@@ -132,7 +132,7 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) {
m.TCPConnect = append(
m.TCPConnect, measurex.NewArchivalTCPConnectList(tcp.Connect)...)
case "https":
- config := &tls.Config{
+ config := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
ServerName: parsedURL.Hostname(),
NextProtos: []string{"h2", "http/1.1"},
RootCAs: nil, // use netxlite's default
@@ -211,7 +211,7 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) {
// ```Go
if resp != nil {
- resp.Body.Close() // tidy
+ _ = resp.Body.Close() // tidy
}
// ```
diff --git a/pkg/tutorial/netxlite/chapter01/README.md b/pkg/tutorial/netxlite/chapter01/README.md
index 2b796ff2a..1b83565b6 100644
--- a/pkg/tutorial/netxlite/chapter01/README.md
+++ b/pkg/tutorial/netxlite/chapter01/README.md
@@ -75,7 +75,7 @@ error that occurred and then calls `os.Exit(1)`
Otherwise, we're tidy and close the opened connection.
```Go
- conn.Close()
+ _ = conn.Close()
}
```
diff --git a/pkg/tutorial/netxlite/chapter01/main.go b/pkg/tutorial/netxlite/chapter01/main.go
index dd8e0b97f..740b6469a 100644
--- a/pkg/tutorial/netxlite/chapter01/main.go
+++ b/pkg/tutorial/netxlite/chapter01/main.go
@@ -76,7 +76,7 @@ func main() {
// Otherwise, we're tidy and close the opened connection.
//
// ```Go
- conn.Close()
+ _ = conn.Close()
}
// ```
diff --git a/pkg/tutorial/netxlite/chapter02/README.md b/pkg/tutorial/netxlite/chapter02/README.md
index a0df9b71c..96984c59e 100644
--- a/pkg/tutorial/netxlite/chapter02/README.md
+++ b/pkg/tutorial/netxlite/chapter02/README.md
@@ -62,7 +62,7 @@ CA pool bundled with OONI by passing nil (so we don't
have to trust the system-wide certificate store)
```Go
- tlsConfig := &tls.Config{
+ tlsConfig := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
ServerName: *sni,
NextProtos: []string{"h2", "http/1.1"},
RootCAs: nil,
@@ -90,7 +90,7 @@ like in the previous chapter, we close the connection.
log.Infof("Cipher suite : %s", netxlite.TLSCipherSuiteString(state.CipherSuite))
log.Infof("Negotiated protocol: %s", state.NegotiatedProtocol)
log.Infof("TLS version : %s", netxlite.TLSVersionString(state.Version))
- conn.Close()
+ _ = conn.Close()
}
```
@@ -149,7 +149,7 @@ func dialTLS(ctx context.Context, address string, config *tls.Config) (model.TLS
}
tlsConn, err := handshakeTLS(ctx, tcpConn, config)
if err != nil {
- tcpConn.Close()
+ _ = tcpConn.Close()
return nil, err
}
return tlsConn, nil
diff --git a/pkg/tutorial/netxlite/chapter02/main.go b/pkg/tutorial/netxlite/chapter02/main.go
index 7734a813c..3e433d467 100644
--- a/pkg/tutorial/netxlite/chapter02/main.go
+++ b/pkg/tutorial/netxlite/chapter02/main.go
@@ -63,7 +63,7 @@ func main() {
// have to trust the system-wide certificate store)
//
// ```Go
- tlsConfig := &tls.Config{
+ tlsConfig := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
ServerName: *sni,
NextProtos: []string{"h2", "http/1.1"},
RootCAs: nil,
@@ -91,7 +91,7 @@ func main() {
log.Infof("Cipher suite : %s", netxlite.TLSCipherSuiteString(state.CipherSuite))
log.Infof("Negotiated protocol: %s", state.NegotiatedProtocol)
log.Infof("TLS version : %s", netxlite.TLSVersionString(state.Version))
- conn.Close()
+ _ = conn.Close()
}
// ```
@@ -150,7 +150,7 @@ func dialTLS(ctx context.Context, address string, config *tls.Config) (model.TLS
}
tlsConn, err := handshakeTLS(ctx, tcpConn, config)
if err != nil {
- tcpConn.Close()
+ _ = tcpConn.Close()
return nil, err
}
return tlsConn, nil
diff --git a/pkg/tutorial/netxlite/chapter03/README.md b/pkg/tutorial/netxlite/chapter03/README.md
index 5899eb1c1..2d6f7d702 100644
--- a/pkg/tutorial/netxlite/chapter03/README.md
+++ b/pkg/tutorial/netxlite/chapter03/README.md
@@ -45,7 +45,7 @@ func main() {
flag.Parse()
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
defer cancel()
- tlsConfig := &tls.Config{
+ tlsConfig := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
ServerName: *sni,
NextProtos: []string{"h2", "http/1.1"},
RootCAs: nil,
@@ -59,7 +59,7 @@ func main() {
log.Infof("Cipher suite : %s", netxlite.TLSCipherSuiteString(state.CipherSuite))
log.Infof("Negotiated protocol: %s", state.NegotiatedProtocol)
log.Infof("TLS version : %s", netxlite.TLSVersionString(state.Version))
- conn.Close()
+ _ = conn.Close()
}
func dialTCP(ctx context.Context, address string) (net.Conn, error) {
@@ -101,7 +101,7 @@ func dialTLS(ctx context.Context, address string, config *tls.Config) (model.TLS
}
tlsConn, err := handshakeTLS(ctx, tcpConn, config)
if err != nil {
- tcpConn.Close()
+ _ = tcpConn.Close()
return nil, err
}
return tlsConn, nil
diff --git a/pkg/tutorial/netxlite/chapter03/main.go b/pkg/tutorial/netxlite/chapter03/main.go
index 5041b65b5..577efb05d 100644
--- a/pkg/tutorial/netxlite/chapter03/main.go
+++ b/pkg/tutorial/netxlite/chapter03/main.go
@@ -46,7 +46,7 @@ func main() {
flag.Parse()
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
defer cancel()
- tlsConfig := &tls.Config{
+ tlsConfig := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
ServerName: *sni,
NextProtos: []string{"h2", "http/1.1"},
RootCAs: nil,
@@ -60,7 +60,7 @@ func main() {
log.Infof("Cipher suite : %s", netxlite.TLSCipherSuiteString(state.CipherSuite))
log.Infof("Negotiated protocol: %s", state.NegotiatedProtocol)
log.Infof("TLS version : %s", netxlite.TLSVersionString(state.Version))
- conn.Close()
+ _ = conn.Close()
}
func dialTCP(ctx context.Context, address string) (net.Conn, error) {
@@ -102,7 +102,7 @@ func dialTLS(ctx context.Context, address string, config *tls.Config) (model.TLS
}
tlsConn, err := handshakeTLS(ctx, tcpConn, config)
if err != nil {
- tcpConn.Close()
+ _ = tcpConn.Close()
return nil, err
}
return tlsConn, nil
diff --git a/pkg/tutorial/netxlite/chapter04/README.md b/pkg/tutorial/netxlite/chapter04/README.md
index b7fb03216..db5634c6f 100644
--- a/pkg/tutorial/netxlite/chapter04/README.md
+++ b/pkg/tutorial/netxlite/chapter04/README.md
@@ -47,7 +47,7 @@ The main difference is that we set the ALPN correctly for
QUIC/HTTP3 by using `"h3"` here.
```Go
- config := &tls.Config{
+ config := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
ServerName: *sni,
NextProtos: []string{"h3"},
RootCAs: nil,
@@ -71,7 +71,7 @@ The rest of the main function is pretty much the same.
log.Infof("Cipher suite : %s", netxlite.TLSCipherSuiteString(state.CipherSuite))
log.Infof("Negotiated protocol: %s", state.NegotiatedProtocol)
log.Infof("TLS version : %s", netxlite.TLSVersionString(state.Version))
- qconn.CloseWithError(0, "")
+ _ = qconn.CloseWithError(0, "")
}
```
diff --git a/pkg/tutorial/netxlite/chapter04/main.go b/pkg/tutorial/netxlite/chapter04/main.go
index 550c09671..b03a1c899 100644
--- a/pkg/tutorial/netxlite/chapter04/main.go
+++ b/pkg/tutorial/netxlite/chapter04/main.go
@@ -48,7 +48,7 @@ func main() {
// QUIC/HTTP3 by using `"h3"` here.
//
// ```Go
- config := &tls.Config{
+ config := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
ServerName: *sni,
NextProtos: []string{"h3"},
RootCAs: nil,
@@ -72,7 +72,7 @@ func main() {
log.Infof("Cipher suite : %s", netxlite.TLSCipherSuiteString(state.CipherSuite))
log.Infof("Negotiated protocol: %s", state.NegotiatedProtocol)
log.Infof("TLS version : %s", netxlite.TLSVersionString(state.Version))
- qconn.CloseWithError(0, "")
+ _ = qconn.CloseWithError(0, "")
}
// ```
diff --git a/pkg/tutorial/netxlite/chapter07/README.md b/pkg/tutorial/netxlite/chapter07/README.md
index f9aa91755..6555142ba 100644
--- a/pkg/tutorial/netxlite/chapter07/README.md
+++ b/pkg/tutorial/netxlite/chapter07/README.md
@@ -46,7 +46,7 @@ func main() {
flag.Parse()
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
defer cancel()
- config := &tls.Config{
+ config := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
ServerName: *sni,
NextProtos: []string{"h2", "http/1.1"},
RootCAs: nil,
@@ -105,7 +105,7 @@ using the GET method.
fatal(err)
}
log.Infof("Status code: %d", resp.StatusCode)
- resp.Body.Close()
+ _ = resp.Body.Close()
}
```
@@ -134,7 +134,7 @@ func dialTLS(ctx context.Context, address string, config *tls.Config) (model.TLS
}
tlsConn, err := handshakeTLS(ctx, tcpConn, config)
if err != nil {
- tcpConn.Close()
+ _ = tcpConn.Close()
return nil, err
}
return tlsConn, nil
diff --git a/pkg/tutorial/netxlite/chapter07/main.go b/pkg/tutorial/netxlite/chapter07/main.go
index e514b11ba..dfcad365b 100644
--- a/pkg/tutorial/netxlite/chapter07/main.go
+++ b/pkg/tutorial/netxlite/chapter07/main.go
@@ -47,7 +47,7 @@ func main() {
flag.Parse()
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
defer cancel()
- config := &tls.Config{
+ config := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
ServerName: *sni,
NextProtos: []string{"h2", "http/1.1"},
RootCAs: nil,
@@ -106,7 +106,7 @@ func main() {
fatal(err)
}
log.Infof("Status code: %d", resp.StatusCode)
- resp.Body.Close()
+ _ = resp.Body.Close()
}
// ```
@@ -135,7 +135,7 @@ func dialTLS(ctx context.Context, address string, config *tls.Config) (model.TLS
}
tlsConn, err := handshakeTLS(ctx, tcpConn, config)
if err != nil {
- tcpConn.Close()
+ _ = tcpConn.Close()
return nil, err
}
return tlsConn, nil
diff --git a/pkg/tutorial/netxlite/chapter08/README.md b/pkg/tutorial/netxlite/chapter08/README.md
index 6eee204ab..c0b8d2703 100644
--- a/pkg/tutorial/netxlite/chapter08/README.md
+++ b/pkg/tutorial/netxlite/chapter08/README.md
@@ -44,7 +44,7 @@ func main() {
flag.Parse()
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
defer cancel()
- config := &tls.Config{
+ config := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
ServerName: *sni,
NextProtos: []string{"h3"},
RootCAs: nil,
@@ -75,7 +75,8 @@ seen how to do the same using TLS conns.)
```Go
clnt := &http.Client{Transport: netxlite.NewHTTP3Transport(
- log.Log, netxlite.NewSingleUseQUICDialer(qconn), &tls.Config{},
+ log.Log, netxlite.NewSingleUseQUICDialer(qconn),
+ &tls.Config{}, // #nosec G402 - we need to use a large TLS versions range for measuring
)}
```
@@ -92,7 +93,7 @@ using the GET method.
fatal(err)
}
log.Infof("Status code: %d", resp.StatusCode)
- resp.Body.Close()
+ _ = resp.Body.Close()
}
```
diff --git a/pkg/tutorial/netxlite/chapter08/main.go b/pkg/tutorial/netxlite/chapter08/main.go
index e05ff5798..a9075f66a 100644
--- a/pkg/tutorial/netxlite/chapter08/main.go
+++ b/pkg/tutorial/netxlite/chapter08/main.go
@@ -45,7 +45,7 @@ func main() {
flag.Parse()
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
defer cancel()
- config := &tls.Config{
+ config := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
ServerName: *sni,
NextProtos: []string{"h3"},
RootCAs: nil,
@@ -76,7 +76,8 @@ func main() {
//
// ```Go
clnt := &http.Client{Transport: netxlite.NewHTTP3Transport(
- log.Log, netxlite.NewSingleUseQUICDialer(qconn), &tls.Config{},
+ log.Log, netxlite.NewSingleUseQUICDialer(qconn),
+ &tls.Config{}, // #nosec G402 - we need to use a large TLS versions range for measuring
)}
// ```
//
@@ -93,7 +94,7 @@ func main() {
fatal(err)
}
log.Infof("Status code: %d", resp.StatusCode)
- resp.Body.Close()
+ _ = resp.Body.Close()
}
// ```
diff --git a/pkg/urlx/DESIGN.md b/pkg/urlx/DESIGN.md
new file mode 100644
index 000000000..6823aa962
--- /dev/null
+++ b/pkg/urlx/DESIGN.md
@@ -0,0 +1,32 @@
+# ./internal/urlx
+
+This package contains algorithms to operate on URLs.
+
+## ResolveReference
+
+This function has the following signature:
+
+```Go
+func ResolveReference(baseURL, path, rawQuery string) (string, error)
+```
+
+It solves the problem of computing a composed URL starting from a base URL, an
+extra path and a possibly empty raw query. The algorithm will ignore the path and
+the query of the base URL and only use the scheme and the host.
+
+For example, assuming the following:
+
+```Go
+baseURL := "https://api.ooni.io/antani?foo=bar"
+path := "/api/v1/check-in"
+query := "bar=baz"
+```
+
+This function will return this URL:
+
+```Go
+"https://api.ooni.io/ap1/v1/check-in?bar=baz"
+```
+
+We need this functionality when implementing communication with the probe services,
+where we have a base URL and specific path and optional query for each API.
diff --git a/pkg/urlx/urlx.go b/pkg/urlx/urlx.go
new file mode 100644
index 000000000..d41c061e9
--- /dev/null
+++ b/pkg/urlx/urlx.go
@@ -0,0 +1,32 @@
+// Package urlx contains URL extensions.
+package urlx
+
+import (
+ "net/url"
+)
+
+// ResolveReference constructs a new URL consisting of the given base URL with
+// the path appended to the given path and the optional query.
+//
+// For example, given:
+//
+// URL := "https://api.ooni.io/api/v1"
+// path := "/measurement_meta"
+// rawQuery := "full=true"
+//
+// This function will return:
+//
+// result := "https://api.ooni.io/api/v1/measurement_meta?full=true"
+//
+// This function fails when we cannot parse URL as a [*net.URL].
+func ResolveReference(baseURL, path, rawQuery string) (string, error) {
+ parsedBase, err := url.Parse(baseURL)
+ if err != nil {
+ return "", err
+ }
+ ref := &url.URL{
+ Path: path,
+ RawQuery: rawQuery,
+ }
+ return parsedBase.ResolveReference(ref).String(), nil
+}
diff --git a/pkg/urlx/urlx_test.go b/pkg/urlx/urlx_test.go
new file mode 100644
index 000000000..5e1ba59f4
--- /dev/null
+++ b/pkg/urlx/urlx_test.go
@@ -0,0 +1,127 @@
+package urlx
+
+import (
+ "errors"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+)
+
+func TestResolveReference(t *testing.T) {
+ // testcase is a test case used by this function
+ type testcase struct {
+ // Name is the test case name.
+ Name string
+
+ // BaseURL is the base URL.
+ BaseURL string
+
+ // Path is the extra path to append.
+ Path string
+
+ // RawQuery is the raw query.
+ RawQuery string
+
+ // ExpectErr is the expected err.
+ ExpectErr error
+
+ // ExpectURL is what we expect to see.
+ ExpectURL string
+ }
+
+ cases := []testcase{{
+ Name: "when we cannot parse the base URL",
+ BaseURL: "\t", // invalid
+ Path: "",
+ RawQuery: "",
+ ExpectErr: errors.New(`parse "\t": net/url: invalid control character in URL`),
+ ExpectURL: "",
+ }, {
+ Name: "when there's only the base URL",
+ BaseURL: "https://api.ooni.io/",
+ Path: "",
+ RawQuery: "",
+ ExpectErr: nil,
+ ExpectURL: "https://api.ooni.io/",
+ }, {
+ Name: "when there's only the path",
+ BaseURL: "",
+ Path: "/api/v1/check-in",
+ RawQuery: "",
+ ExpectErr: nil,
+ ExpectURL: "/api/v1/check-in",
+ }, {
+ Name: "when there's only the query",
+ BaseURL: "",
+ Path: "",
+ RawQuery: "key1=value1&key1=value2&key3=value3",
+ ExpectErr: nil,
+ ExpectURL: "?key1=value1&key1=value2&key3=value3",
+ }, {
+ Name: "with base URL and path",
+ BaseURL: "https://api.ooni.io/",
+ Path: "/api/v1/check-in",
+ RawQuery: "",
+ ExpectErr: nil,
+ ExpectURL: "https://api.ooni.io/api/v1/check-in",
+ }, {
+ Name: "with base URL and query",
+ BaseURL: "https://api.ooni.io/",
+ Path: "",
+ RawQuery: "key1=value1&key1=value2&key3=value3",
+ ExpectErr: nil,
+ ExpectURL: "https://api.ooni.io/?key1=value1&key1=value2&key3=value3",
+ }, {
+ Name: "with base URL, path, and query",
+ BaseURL: "https://api.ooni.io/",
+ Path: "/api/v1/check-in",
+ RawQuery: "key1=value1&key1=value2&key3=value3",
+ ExpectErr: nil,
+ ExpectURL: "https://api.ooni.io/api/v1/check-in?key1=value1&key1=value2&key3=value3",
+ }, {
+ Name: "with base URL with path, path, and query",
+ BaseURL: "https://api.ooni.io/api",
+ Path: "/v1/check-in",
+ RawQuery: "key1=value1&key1=value2&key3=value3",
+ ExpectErr: nil,
+ ExpectURL: "https://api.ooni.io/v1/check-in?key1=value1&key1=value2&key3=value3",
+ }, {
+ Name: "with base URL with path and query, path, and query",
+ BaseURL: "https://api.ooni.io/api?foo=bar",
+ Path: "/v1/check-in",
+ RawQuery: "key1=value1&key1=value2&key3=value3",
+ ExpectErr: nil,
+ ExpectURL: "https://api.ooni.io/v1/check-in?key1=value1&key1=value2&key3=value3",
+ }}
+
+ for _, tc := range cases {
+ t.Run(tc.Name, func(t *testing.T) {
+ // invoke the API we're currently testing
+ got, err := ResolveReference(tc.BaseURL, tc.Path, tc.RawQuery)
+
+ // check for errors
+ switch {
+ case err == nil && tc.ExpectErr == nil:
+ if diff := cmp.Diff(tc.ExpectURL, got); diff != "" {
+ t.Fatal(diff)
+ }
+ return
+
+ case err != nil && tc.ExpectErr == nil:
+ t.Fatal("expected err", tc.ExpectErr, "got", err)
+ return
+
+ case err == nil && tc.ExpectErr != nil:
+ t.Fatal("expected err", tc.ExpectErr, "got", err)
+ return
+
+ case err != nil && tc.ExpectErr != nil:
+ if err.Error() != tc.ExpectErr.Error() {
+ t.Fatal("expected err", tc.ExpectErr, "got", err)
+ }
+ return
+ }
+
+ })
+ }
+}
diff --git a/pkg/version/version.go b/pkg/version/version.go
index 27d030c89..a85f02af1 100644
--- a/pkg/version/version.go
+++ b/pkg/version/version.go
@@ -2,4 +2,4 @@
package version
// Version is the ooniprobe version.
-const Version = "3.21.0-alpha"
+const Version = "3.22.0-alpha"
diff --git a/pkg/webconnectivityalgo/calltesthelpers.go b/pkg/webconnectivityalgo/calltesthelpers.go
new file mode 100644
index 000000000..0d0ccc9dd
--- /dev/null
+++ b/pkg/webconnectivityalgo/calltesthelpers.go
@@ -0,0 +1,53 @@
+package webconnectivityalgo
+
+import (
+ "context"
+
+ "github.com/ooni/probe-engine/pkg/httpclientx"
+ "github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+)
+
+// CallWebConnectivityTestHelper invokes the Web Connectivity test helper with the
+// given request object, the given list of available test helpers, and the given session.
+//
+// If the list of test helpers is empty this function immediately returns nil, zero,
+// and the [model.ErrNoAvailableTestHelpers] error to the caller.
+//
+// In case of any other failure, this function returns nil, zero, and an error.
+//
+// On success, it returns the response, the used TH index, and nil.
+//
+// Note that the returned error won't be wrapped, so you need to wrap it yourself.
+func CallWebConnectivityTestHelper(ctx context.Context, creq *model.THRequest,
+ testhelpers []model.OOAPIService, sess model.ExperimentSession) (*model.THResponse, int, error) {
+ // handle the case where there are no available web connectivity test helpers
+ if len(testhelpers) <= 0 {
+ return nil, 0, model.ErrNoAvailableTestHelpers
+ }
+
+ // create overlapped state for performing overlapped HTTP calls
+ overlapped := httpclientx.NewOverlappedPostJSON[*model.THRequest, *model.THResponse](
+ creq, &httpclientx.Config{
+ Authorization: "", // not needed
+ Client: sess.DefaultHTTPClient(),
+ Logger: sess.Logger(),
+ UserAgent: sess.UserAgent(),
+ },
+ )
+
+ // perform the overlapped HTTP API calls
+ cresp, idx, err := overlapped.Run(ctx, httpclientx.NewEndpointFromModelOOAPIServices(testhelpers...)...)
+
+ // handle the case where all test helpers failed
+ if err != nil {
+ return nil, 0, err
+ }
+
+ // apply some sanity checks to the results
+ runtimex.Assert(idx >= 0 && idx < len(testhelpers), "idx out of bounds")
+ runtimex.Assert(cresp != nil, "out is nil")
+
+ // return the results to the web connectivity caller
+ return cresp, idx, nil
+}
diff --git a/pkg/webconnectivityalgo/calltesthelpers_test.go b/pkg/webconnectivityalgo/calltesthelpers_test.go
new file mode 100644
index 000000000..a0d1f2b19
--- /dev/null
+++ b/pkg/webconnectivityalgo/calltesthelpers_test.go
@@ -0,0 +1,306 @@
+package webconnectivityalgo
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "testing"
+ "time"
+
+ "github.com/ooni/probe-engine/pkg/mocks"
+ "github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/netxlite"
+ "github.com/ooni/probe-engine/pkg/testingx"
+)
+
+// This function tests the [CallWebConnectivityTestHelper] function.
+func TestSessionCallWebConnectivityTestHelper(t *testing.T) {
+ // We start with simple tests that exercise the basic functionality of the method
+ // without bothering with having more than one available test helper.
+
+ t.Run("when there are no available test helpers", func(t *testing.T) {
+ // create a new session only initializing the fields that
+ // are going to matter for running this specific test
+ sess := &mocks.Session{
+ MockLogger: func() model.Logger {
+ return model.DiscardLogger
+ },
+ MockDefaultHTTPClient: func() model.HTTPClient {
+ return http.DefaultClient
+ },
+ MockUserAgent: func() string {
+ return model.HTTPHeaderUserAgent
+ },
+ }
+
+ // create a new background context
+ ctx := context.Background()
+
+ // create a fake request for the test helper
+ //
+ // note: no need to fill the request for this test case
+ creq := &model.THRequest{}
+
+ // invoke the API
+ cresp, idx, err := CallWebConnectivityTestHelper(ctx, creq, nil, sess)
+
+ // make sure we get the expected error
+ if !errors.Is(err, model.ErrNoAvailableTestHelpers) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure idx is zero
+ if idx != 0 {
+ t.Fatal("expected zero, got", idx)
+ }
+
+ // make sure cresp is nil
+ if cresp != nil {
+ t.Fatal("expected nil, got", cresp)
+ }
+ })
+
+ t.Run("when the call fails", func(t *testing.T) {
+ // create a local test server that always resets the connection
+ server := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer server.Close()
+
+ // create a new session only initializing the fields that
+ // are going to matter for running this specific test
+ sess := &mocks.Session{
+ MockLogger: func() model.Logger {
+ return model.DiscardLogger
+ },
+ MockDefaultHTTPClient: func() model.HTTPClient {
+ return http.DefaultClient
+ },
+ MockUserAgent: func() string {
+ return model.HTTPHeaderUserAgent
+ },
+ }
+
+ // create a new background context
+ ctx := context.Background()
+
+ // create a fake request for the test helper
+ //
+ // note: no need to fill the request for this test case
+ creq := &model.THRequest{}
+
+ // create the list of test helpers to use
+ testhelpers := []model.OOAPIService{{
+ Address: server.URL,
+ Type: "https",
+ Front: "",
+ }}
+
+ // invoke the API
+ cresp, idx, err := CallWebConnectivityTestHelper(ctx, creq, testhelpers, sess)
+
+ // make sure we get the expected error
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure idx is zero
+ if idx != 0 {
+ t.Fatal("expected zero, got", idx)
+ }
+
+ // make sure cresp is nil
+ if cresp != nil {
+ t.Fatal("expected nil, got", cresp)
+ }
+ })
+
+ t.Run("when the call succeeds", func(t *testing.T) {
+ // create a local test server that always returns an ~empty response
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{}`))
+ }))
+ defer server.Close()
+
+ // create a new session only initializing the fields that
+ // are going to matter for running this specific test
+ sess := &mocks.Session{
+ MockLogger: func() model.Logger {
+ return model.DiscardLogger
+ },
+ MockDefaultHTTPClient: func() model.HTTPClient {
+ return http.DefaultClient
+ },
+ MockUserAgent: func() string {
+ return model.HTTPHeaderUserAgent
+ },
+ }
+
+ // create a new background context
+ ctx := context.Background()
+
+ // create a fake request for the test helper
+ //
+ // note: no need to fill the request for this test case
+ creq := &model.THRequest{}
+
+ // create the list of test helpers to use
+ testhelpers := []model.OOAPIService{{
+ Address: server.URL,
+ Type: "https",
+ Front: "",
+ }}
+
+ // invoke the API
+ cresp, idx, err := CallWebConnectivityTestHelper(ctx, creq, testhelpers, sess)
+
+ // make sure we get the expected error
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure idx is zero
+ if idx != 0 {
+ t.Fatal("expected zero, got", idx)
+ }
+
+ // make sure cresp is not nil
+ if cresp == nil {
+ t.Fatal("expected not nil, got", cresp)
+ }
+ })
+
+ // Now we include some tests that ensure that we can chain (in more or less
+ // smart fashion) API calls using multiple test helper endpoints.
+
+ t.Run("with two test helpers where the first one resets the connection and the second works", func(t *testing.T) {
+ // create a local test server1 that always resets the connection
+ server1 := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer server1.Close()
+
+ // create a local test server2 that always returns an ~empty response
+ server2 := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{}`))
+ }))
+ defer server2.Close()
+
+ // create a new session only initializing the fields that
+ // are going to matter for running this specific test
+ sess := &mocks.Session{
+ MockLogger: func() model.Logger {
+ return model.DiscardLogger
+ },
+ MockDefaultHTTPClient: func() model.HTTPClient {
+ return http.DefaultClient
+ },
+ MockUserAgent: func() string {
+ return model.HTTPHeaderUserAgent
+ },
+ }
+
+ // create a new background context
+ ctx := context.Background()
+
+ // create a fake request for the test helper
+ //
+ // note: no need to fill the request for this test case
+ creq := &model.THRequest{}
+
+ // create the list of test helpers to use
+ testhelpers := []model.OOAPIService{{
+ Address: server1.URL,
+ Type: "https",
+ Front: "",
+ }, {
+ Address: server2.URL,
+ Type: "https",
+ Front: "",
+ }}
+
+ // invoke the API
+ cresp, idx, err := CallWebConnectivityTestHelper(ctx, creq, testhelpers, sess)
+
+ // make sure we get the expected error
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure idx is one
+ if idx != 1 {
+ t.Fatal("expected one, got", idx)
+ }
+
+ // make sure cresp is not nil
+ if cresp == nil {
+ t.Fatal("expected not nil, got", cresp)
+ }
+ })
+
+ t.Run("with two test helpers where the first one times out the connection and the second works", func(t *testing.T) {
+ // create a local test server1 that resets the connection after a ~long delay
+ server1 := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ select {
+ case <-time.After(10 * time.Second):
+ testingx.HTTPHandlerReset().ServeHTTP(w, r)
+ case <-r.Context().Done():
+ return
+ }
+ }))
+ defer server1.Close()
+
+ // create a local test server2 that always returns an ~empty response
+ server2 := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{}`))
+ }))
+ defer server2.Close()
+
+ // create a new session only initializing the fields that
+ // are going to matter for running this specific test
+ sess := &mocks.Session{
+ MockLogger: func() model.Logger {
+ return model.DiscardLogger
+ },
+ MockDefaultHTTPClient: func() model.HTTPClient {
+ return http.DefaultClient
+ },
+ MockUserAgent: func() string {
+ return model.HTTPHeaderUserAgent
+ },
+ }
+
+ // create a new background context
+ ctx := context.Background()
+
+ // create a fake request for the test helper
+ //
+ // note: no need to fill the request for this test case
+ creq := &model.THRequest{}
+
+ // create the list of test helpers to use
+ testhelpers := []model.OOAPIService{{
+ Address: server1.URL,
+ Type: "https",
+ Front: "",
+ }, {
+ Address: server2.URL,
+ Type: "https",
+ Front: "",
+ }}
+
+ // invoke the API
+ cresp, idx, err := CallWebConnectivityTestHelper(ctx, creq, testhelpers, sess)
+
+ // make sure we get the expected error
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure idx is one
+ if idx != 1 {
+ t.Fatal("expected one, got", idx)
+ }
+
+ // make sure cresp is not nil
+ if cresp == nil {
+ t.Fatal("expected not nil, got", cresp)
+ }
+ })
+}
diff --git a/pkg/webconnectivityalgo/dnsoverhttps.go b/pkg/webconnectivityalgo/dnsoverhttps.go
index 6fa392e88..a38b2810d 100644
--- a/pkg/webconnectivityalgo/dnsoverhttps.go
+++ b/pkg/webconnectivityalgo/dnsoverhttps.go
@@ -58,7 +58,7 @@ func NewOpportunisticDNSOverHTTPSURLProvider(urls ...string) *OpportunisticDNSOv
}
func (o *OpportunisticDNSOverHTTPSURLProvider) seed(t time.Time) {
- o.rnd = rand.New(rand.NewSource(t.UnixNano()))
+ o.rnd = rand.New(rand.NewSource(t.UnixNano())) // #nosec G404 -- not really important
}
// MaybeNextURL returns the next URL to measure, if any. Our aim is to perform
diff --git a/pkg/webconnectivityalgo/dnsoverudp.go b/pkg/webconnectivityalgo/dnsoverudp.go
index 06bde13d2..55a683ab2 100644
--- a/pkg/webconnectivityalgo/dnsoverudp.go
+++ b/pkg/webconnectivityalgo/dnsoverudp.go
@@ -26,6 +26,6 @@ var dnsOverUDPResolverAddressIPv4 = []string{
// RandomDNSOverUDPResolverEndpointIPv4 returns a random DNS-over-UDP resolver endpoint using IPv4.
func RandomDNSOverUDPResolverEndpointIPv4() string {
- idx := rand.Intn(len(dnsOverUDPResolverAddressIPv4))
+ idx := rand.Intn(len(dnsOverUDPResolverAddressIPv4)) // #nosec G404 -- not really important
return net.JoinHostPort(dnsOverUDPResolverAddressIPv4[idx], "53")
}
diff --git a/pkg/webconnectivityqa/control.go b/pkg/webconnectivityqa/control.go
index a386dd077..05862997d 100644
--- a/pkg/webconnectivityqa/control.go
+++ b/pkg/webconnectivityqa/control.go
@@ -43,7 +43,7 @@ func controlFailureWithSuccessfulHTTPWebsite() *TestCase {
},
ExpectErr: false,
ExpectTestKeys: &TestKeys{
- ControlFailure: "unknown_failure: httpapi: all endpoints failed: [ connection_reset; connection_reset; connection_reset; connection_reset;]",
+ ControlFailure: "connection_reset",
XStatus: 8, // StatusAnomalyControlUnreachable
Accessible: nil,
Blocking: nil,
@@ -88,7 +88,7 @@ func controlFailureWithSuccessfulHTTPSWebsite() *TestCase {
},
ExpectErr: false,
ExpectTestKeys: &TestKeys{
- ControlFailure: "unknown_failure: httpapi: all endpoints failed: [ connection_reset; connection_reset; connection_reset; connection_reset;]",
+ ControlFailure: "connection_reset",
XStatus: 1, // StatusSuccessSecure
XBlockingFlags: 32, // AnalysisBlockingFlagSuccess
XNullNullFlags: 8, // AnalysisFlagNullNullSuccessfulHTTPS
diff --git a/pkg/webconnectivityqa/dnsblocking.go b/pkg/webconnectivityqa/dnsblocking.go
index cd7aa32b0..a877c94cc 100644
--- a/pkg/webconnectivityqa/dnsblocking.go
+++ b/pkg/webconnectivityqa/dnsblocking.go
@@ -2,6 +2,7 @@ package webconnectivityqa
import (
"github.com/ooni/probe-engine/pkg/netemx"
+ "github.com/ooni/probe-engine/pkg/runtimex"
)
// dnsBlockingAndroidDNSCacheNoData is the case where we're on Android and the getaddrinfo
@@ -73,7 +74,7 @@ func dnsBlockingBOGON() *TestCase {
Input: "https://www.example.com/",
Configure: func(env *netemx.QAEnv) {
env.ISPResolverConfig().RemoveRecord("www.example.com")
- env.ISPResolverConfig().AddRecord("www.example.com", "", "10.10.34.35")
+ runtimex.Try0(env.ISPResolverConfig().AddRecord("www.example.com", "", "10.10.34.35"))
},
ExpectErr: false,
ExpectTestKeys: &TestKeys{
diff --git a/pkg/webconnectivityqa/localhost.go b/pkg/webconnectivityqa/localhost.go
index 8279aaa6c..16766b4e3 100644
--- a/pkg/webconnectivityqa/localhost.go
+++ b/pkg/webconnectivityqa/localhost.go
@@ -1,6 +1,9 @@
package webconnectivityqa
-import "github.com/ooni/probe-engine/pkg/netemx"
+import (
+ "github.com/ooni/probe-engine/pkg/netemx"
+ "github.com/ooni/probe-engine/pkg/runtimex"
+)
// localhostWithHTTP is the case where the website DNS is misconfigured and returns a loopback address.
func localhostWithHTTP() *TestCase {
@@ -11,8 +14,8 @@ func localhostWithHTTP() *TestCase {
Configure: func(env *netemx.QAEnv) {
// make sure all resolvers think the correct answer is localhost
- env.ISPResolverConfig().AddRecord("www.example.com", "", "127.0.0.1")
- env.OtherResolversConfig().AddRecord("www.example.com", "", "127.0.0.1")
+ runtimex.Try0(env.ISPResolverConfig().AddRecord("www.example.com", "", "127.0.0.1"))
+ runtimex.Try0(env.OtherResolversConfig().AddRecord("www.example.com", "", "127.0.0.1"))
},
ExpectErr: false,
@@ -34,8 +37,8 @@ func localhostWithHTTPS() *TestCase {
Configure: func(env *netemx.QAEnv) {
// make sure all resolvers think the correct answer is localhost
- env.ISPResolverConfig().AddRecord("www.example.com", "", "127.0.0.1")
- env.OtherResolversConfig().AddRecord("www.example.com", "", "127.0.0.1")
+ runtimex.Try0(env.ISPResolverConfig().AddRecord("www.example.com", "", "127.0.0.1"))
+ runtimex.Try0(env.OtherResolversConfig().AddRecord("www.example.com", "", "127.0.0.1"))
},
ExpectErr: false,
diff --git a/pkg/webconnectivityqa/redirect.go b/pkg/webconnectivityqa/redirect.go
index 12b4e6ee0..4cae4c25c 100644
--- a/pkg/webconnectivityqa/redirect.go
+++ b/pkg/webconnectivityqa/redirect.go
@@ -391,3 +391,47 @@ func redirectWithBrokenLocationForHTTPS() *TestCase {
},
}
}
+
+// redirectWithMoreThanTenRedirectsAndHTTP is a scenario where the redirect
+// consists of more than ten redirects for http:// URLs.
+func redirectWithMoreThanTenRedirectsAndHTTP() *TestCase {
+ return &TestCase{
+ Name: "redirectWithMoreThanTenRedirectsAndHTTP",
+ Flags: TestCaseFlagNoV04,
+ Input: "http://httpbin.com/redirect/15",
+ LongTest: true,
+ ExpectErr: false,
+ ExpectTestKeys: &TestKeys{
+ DNSExperimentFailure: nil,
+ DNSConsistency: "consistent",
+ HTTPExperimentFailure: `unknown_failure: stopped after too many redirects`,
+ XStatus: 0,
+ XDNSFlags: 0,
+ XBlockingFlags: 0,
+ Accessible: false,
+ Blocking: false,
+ },
+ }
+}
+
+// redirectWithMoreThanTenRedirectsAndHTTPS is a scenario where the redirect
+// consists of more than ten redirects for https:// URLs.
+func redirectWithMoreThanTenRedirectsAndHTTPS() *TestCase {
+ return &TestCase{
+ Name: "redirectWithMoreThanTenRedirectsAndHTTPS",
+ Flags: TestCaseFlagNoV04,
+ Input: "https://httpbin.com/redirect/15",
+ LongTest: true,
+ ExpectErr: false,
+ ExpectTestKeys: &TestKeys{
+ DNSExperimentFailure: nil,
+ DNSConsistency: "consistent",
+ HTTPExperimentFailure: `unknown_failure: stopped after too many redirects`,
+ XStatus: 0,
+ XDNSFlags: 0,
+ XBlockingFlags: 0,
+ Accessible: false,
+ Blocking: false,
+ },
+ }
+}
diff --git a/pkg/webconnectivityqa/testcase.go b/pkg/webconnectivityqa/testcase.go
index 001482cb1..b606ae811 100644
--- a/pkg/webconnectivityqa/testcase.go
+++ b/pkg/webconnectivityqa/testcase.go
@@ -90,6 +90,8 @@ func AllTestCases() []*TestCase {
redirectWithConsistentDNSAndThenEOFForHTTPS(),
redirectWithConsistentDNSAndThenTimeoutForHTTP(),
redirectWithConsistentDNSAndThenTimeoutForHTTPS(),
+ redirectWithMoreThanTenRedirectsAndHTTP(),
+ redirectWithMoreThanTenRedirectsAndHTTPS(),
successWithHTTP(),
successWithHTTPS(),
diff --git a/pkg/x/dsljavascript/consolemodule.go b/pkg/x/dsljavascript/consolemodule.go
index 5e0e9b614..9bd0a5c29 100644
--- a/pkg/x/dsljavascript/consolemodule.go
+++ b/pkg/x/dsljavascript/consolemodule.go
@@ -15,9 +15,9 @@ import (
func (vm *VM) newModuleConsole(gojaVM *goja.Runtime, mod *goja.Object) {
runtimex.Assert(vm.vm == gojaVM, "dsljavascript: unexpected gojaVM pointer value")
exports := mod.Get("exports").(*goja.Object)
- exports.Set("log", vm.consoleLog)
- exports.Set("error", vm.consoleError)
- exports.Set("warn", vm.consoleWarn)
+ runtimex.Try0(exports.Set("log", vm.consoleLog))
+ runtimex.Try0(exports.Set("error", vm.consoleError))
+ runtimex.Try0(exports.Set("warn", vm.consoleWarn))
}
// consoleLog implements console.log
diff --git a/pkg/x/dsljavascript/golangmodule.go b/pkg/x/dsljavascript/golangmodule.go
index cdd1e5cac..e6e7d82f7 100644
--- a/pkg/x/dsljavascript/golangmodule.go
+++ b/pkg/x/dsljavascript/golangmodule.go
@@ -11,7 +11,7 @@ import (
func (vm *VM) newModuleGolang(gojaVM *goja.Runtime, mod *goja.Object) {
runtimex.Assert(vm.vm == gojaVM, "dsljavascript: unexpected gojaVM pointer value")
exports := mod.Get("exports").(*goja.Object)
- exports.Set("timeNow", vm.golangTimeNow)
+ runtimex.Try0(exports.Set("timeNow", vm.golangTimeNow))
}
// golangTimeNow returns the current time using golang [time.Now]
diff --git a/pkg/x/dsljavascript/oonimodule.go b/pkg/x/dsljavascript/oonimodule.go
index defed9de2..0c2a0b251 100644
--- a/pkg/x/dsljavascript/oonimodule.go
+++ b/pkg/x/dsljavascript/oonimodule.go
@@ -15,7 +15,7 @@ import (
func (vm *VM) newModuleOONI(gojaVM *goja.Runtime, mod *goja.Object) {
runtimex.Assert(vm.vm == gojaVM, "dsljavascript: unexpected gojaVM pointer value")
exports := mod.Get("exports").(*goja.Object)
- exports.Set("runDSL", vm.ooniRunDSL)
+ runtimex.Try0(exports.Set("runDSL", vm.ooniRunDSL))
}
func (vm *VM) ooniRunDSL(jsAST *goja.Object, zeroTime time.Time) (string, error) {
diff --git a/pkg/x/dsljavascript/vm.go b/pkg/x/dsljavascript/vm.go
index 72ed35f27..61f5ec952 100644
--- a/pkg/x/dsljavascript/vm.go
+++ b/pkg/x/dsljavascript/vm.go
@@ -10,6 +10,7 @@ import (
"github.com/dop251/goja_nodejs/require"
"github.com/dop251/goja_nodejs/util"
"github.com/ooni/probe-engine/pkg/model"
+ "github.com/ooni/probe-engine/pkg/runtimex"
)
// VMConfig contains configuration for creating a VM.
@@ -103,7 +104,7 @@ func NewVM(config *VMConfig, scriptPath string) (*VM, error) {
registry.RegisterNativeModule("console", vm.newModuleConsole)
// make sure the 'console' object exists in the VM before running scripts
- gojaVM.Set("console", require.Require(gojaVM, "console"))
+ runtimex.Try0(gojaVM.Set("console", require.Require(gojaVM, "console")))
// register the _golang module in JavaScript
registry.RegisterNativeModule("_golang", vm.newModuleGolang)
@@ -124,7 +125,7 @@ func LoadExperiment(config *VMConfig, exPath string) (*VM, error) {
}
// make sure there's an empty dictionary containing exports
- vm.vm.Set("exports", vm.vm.NewObject())
+ runtimex.Try0(vm.vm.Set("exports", vm.vm.NewObject()))
// run the script
if err := vm.RunScript(exPath); err != nil {
@@ -136,7 +137,7 @@ func LoadExperiment(config *VMConfig, exPath string) (*VM, error) {
func (vm *VM) RunScript(exPath string) error {
// read the file content
- content, err := os.ReadFile(exPath)
+ content, err := os.ReadFile(exPath) // #nosec G304 - this is working as intended
if err != nil {
return err
}
diff --git a/pkg/x/dslvm/http.go b/pkg/x/dslvm/http.go
index 071336f91..f9db2c138 100644
--- a/pkg/x/dslvm/http.go
+++ b/pkg/x/dslvm/http.go
@@ -138,7 +138,7 @@ func (sx *HTTPRoundTripStage[T]) roundTrip(ctx context.Context, rtx Runtime, con
}
func (sx *HTTPRoundTripStage[T]) newHTTPRequest(
- ctx context.Context, conn HTTPConnection, logger model.Logger) (*http.Request, error) {
+ ctx context.Context, conn HTTPConnection, _ model.Logger) (*http.Request, error) {
// create the default HTTP request
URL := &url.URL{
Scheme: conn.Scheme(),
diff --git a/pkg/x/dslvm/quic.go b/pkg/x/dslvm/quic.go
index 8fd252d56..5681bf735 100644
--- a/pkg/x/dslvm/quic.go
+++ b/pkg/x/dslvm/quic.go
@@ -167,9 +167,9 @@ func (sx *QUICHandshakeStage) handshake(ctx context.Context, rtx Runtime, endpoi
}
func (sx *QUICHandshakeStage) newTLSConfig() *tls.Config {
- return &tls.Config{
+ return &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
NextProtos: sx.NextProtos,
- InsecureSkipVerify: sx.InsecureSkipVerify,
+ InsecureSkipVerify: sx.InsecureSkipVerify, // #nosec G402 - it's fine to possibly skip verify in a nettest
RootCAs: sx.RootCAs,
ServerName: sx.ServerName,
}
diff --git a/pkg/x/dslvm/tls.go b/pkg/x/dslvm/tls.go
index 26fbb6e37..13e2ab3d5 100644
--- a/pkg/x/dslvm/tls.go
+++ b/pkg/x/dslvm/tls.go
@@ -150,7 +150,7 @@ func (sx *TLSHandshakeStage) handshake(ctx context.Context, rtx Runtime, tcpConn
// handle error case
if err != nil {
rtx.ActiveConnections().Signal() // make sure we release the semaphore
- tcpConn.Conn.Close() // make sure we close the conn
+ _ = tcpConn.Conn.Close() // make sure we close the conn
return
}
@@ -162,9 +162,9 @@ func (sx *TLSHandshakeStage) handshake(ctx context.Context, rtx Runtime, tcpConn
}
func (sx *TLSHandshakeStage) newTLSConfig() *tls.Config {
- return &tls.Config{
+ return &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
NextProtos: sx.NextProtos,
- InsecureSkipVerify: sx.InsecureSkipVerify,
+ InsecureSkipVerify: sx.InsecureSkipVerify, // #nosec G402 - it's fine to possibly skip verify in a nettest
RootCAs: sx.RootCAs,
ServerName: sx.ServerName,
}
diff --git a/pkg/x/dslx/fxasync.go b/pkg/x/dslx/fxasync.go
index c11c553d6..2fea7afa0 100644
--- a/pkg/x/dslx/fxasync.go
+++ b/pkg/x/dslx/fxasync.go
@@ -146,14 +146,6 @@ type matrixPoint[A, B any] struct {
in A
}
-// matrixMin can be replaced with the built-in min when we switch to go1.21.
-func matrixMin(a, b Parallelism) Parallelism {
- if a < b {
- return a
- }
- return b
-}
-
// Matrix invokes each function on each input using N goroutines and streams the results to a channel.
func Matrix[A, B any](ctx context.Context, N Parallelism, inputs []A, functions []Func[A, B]) <-chan *Maybe[B] {
// make output
@@ -172,7 +164,7 @@ func Matrix[A, B any](ctx context.Context, N Parallelism, inputs []A, functions
// spawn goroutines
wg := &sync.WaitGroup{}
- N = matrixMin(1, N)
+ N = min(1, N)
for i := Parallelism(0); i < N; i++ {
wg.Add(1)
go func() {
diff --git a/pkg/x/dslx/fxasync_test.go b/pkg/x/dslx/fxasync_test.go
index 95499d1dd..af93f285f 100644
--- a/pkg/x/dslx/fxasync_test.go
+++ b/pkg/x/dslx/fxasync_test.go
@@ -101,15 +101,3 @@ func TestParallel(t *testing.T) {
}
})
}
-
-func TestMatrixMin(t *testing.T) {
- if v := matrixMin(1, 7); v != 1 {
- t.Fatal("expected to see 1, got", v)
- }
- if v := matrixMin(7, 4); v != 4 {
- t.Fatal("expected to see 4, got", v)
- }
- if v := matrixMin(11, 11); v != 11 {
- t.Fatal("expected to see 11, got", v)
- }
-}
diff --git a/pkg/x/dslx/tls.go b/pkg/x/dslx/tls.go
index 4d3307ec9..b624dd77b 100644
--- a/pkg/x/dslx/tls.go
+++ b/pkg/x/dslx/tls.go
@@ -121,7 +121,7 @@ func tlsNewConfig(address string, defaultALPN []string, domain string, logger mo
// See https://github.com/ooni/probe/issues/2413 to understand
// why we're using nil to force netxlite to use the cached
// default Mozilla cert pool.
- config := &tls.Config{
+ config := &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
NextProtos: append([]string{}, defaultALPN...),
InsecureSkipVerify: false,
RootCAs: nil,