Skip to content

Commit a37f0d2

Browse files
bassosimonebassosimone
authored
fix(qatool): do not save large response bodies on disk (#1512)
It seems unnecessary in light of what we're currently testing. By doing this, we allow for much smaller commits. Tangentially related to ooni/probe#2677.
1 parent 595e167 commit a37f0d2

37 files changed

+935
-905
lines changed

‎internal/minipipeline/normalize.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,5 +50,14 @@ func NormalizeHTTPRequestResults(values []*model.ArchivalHTTPRequestResult) {
5050
for _, entry := range values {
5151
entry.T0 = 0
5252
entry.T = 0
53+
54+
// Avoid storing large bodies because that wastes repository space.
55+
//
56+
// See https://github.com/ooni/probe/issues/2677.
57+
const maxStoreBody = 1 << 14
58+
if len(entry.Response.Body) > maxStoreBody {
59+
entry.Response.Body = model.ArchivalScrubbedMaybeBinaryString("")
60+
entry.Response.BodyIsTruncated = true
61+
}
5362
}
5463
}

‎internal/minipipeline/normalize_test.go

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55

66
"github.com/google/go-cmp/cmp"
77
"github.com/ooni/probe-cli/v3/internal/model"
8+
"github.com/ooni/probe-cli/v3/internal/randx"
89
)
910

1011
func TestNormalizeDNSLookupResults(t *testing.T) {
@@ -229,11 +230,35 @@ func TestNormalizeHTTPRequestResults(t *testing.T) {
229230
T0: 0.11,
230231
T: 0.4,
231232
}, {
233+
Response: model.ArchivalHTTPResponse{
234+
Body: model.ArchivalScrubbedMaybeBinaryString("1234567"),
235+
},
232236
T0: 0.5,
233237
T: 0.66,
238+
}, {
239+
Response: model.ArchivalHTTPResponse{
240+
Body: model.ArchivalScrubbedMaybeBinaryString(randx.Letters(1 << 19)),
241+
},
242+
T0: 0.7,
243+
T: 0.88,
234244
}}
235245
},
236-
expect: []*model.ArchivalHTTPRequestResult{{}, {}},
246+
expect: []*model.ArchivalHTTPRequestResult{
247+
{
248+
// empty
249+
},
250+
{
251+
Response: model.ArchivalHTTPResponse{
252+
Body: model.ArchivalScrubbedMaybeBinaryString("1234567"),
253+
},
254+
},
255+
{
256+
Response: model.ArchivalHTTPResponse{
257+
Body: model.ArchivalScrubbedMaybeBinaryString(""),
258+
BodyIsTruncated: true,
259+
},
260+
},
261+
},
237262
}}
238263

239264
for _, tc := range cases {

‎internal/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/analysis.json

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
"TCPConnectUnexplainedFailureDuringWebFetch": [],
3939
"TCPConnectUnexplainedFailureDuringConnectivityCheck": [],
4040
"TLSHandshakeExpectedFailure": [
41-
40001
41+
40002
4242
],
4343
"TLSHandshakeUnexpectedFailure": [],
4444
"TLSHandshakeUnexpectedFailureDuringWebFetch": [],
@@ -49,7 +49,7 @@
4949
"HTTPRoundTripUnexpectedFailure": [],
5050
"HTTPRoundTripUnexplainedFailure": [],
5151
"HTTPFinalResponseSuccessTLSWithoutControl": null,
52-
"HTTPFinalResponseSuccessTLSWithControl": 40002,
52+
"HTTPFinalResponseSuccessTLSWithControl": 40001,
5353
"HTTPFinalResponseSuccessTCPWithoutControl": null,
5454
"HTTPFinalResponseSuccessTCPWithControl": null,
5555
"HTTPFinalResponseDiffBodyProportionFactor": 1,
@@ -64,7 +64,7 @@
6464
"TagDepth": 0,
6565
"Type": 3,
6666
"Failure": "",
67-
"TransactionID": 40002,
67+
"TransactionID": 40001,
6868
"TagFetchBody": true,
6969
"DNSTransactionID": 20001,
7070
"DNSDomain": "www.example.com",
@@ -78,7 +78,7 @@
7878
"IPAddress": "93.184.216.34",
7979
"IPAddressASN": 15133,
8080
"IPAddressBogon": false,
81-
"EndpointTransactionID": 40002,
81+
"EndpointTransactionID": 40001,
8282
"EndpointProto": "tcp",
8383
"EndpointPort": "443",
8484
"EndpointAddress": "93.184.216.34:443",
@@ -121,7 +121,7 @@
121121
"TagDepth": 0,
122122
"Type": 2,
123123
"Failure": "ssl_unknown_authority",
124-
"TransactionID": 40001,
124+
"TransactionID": 40002,
125125
"TagFetchBody": true,
126126
"DNSTransactionID": 1,
127127
"DNSDomain": "www.example.com",
@@ -135,7 +135,7 @@
135135
"IPAddress": "104.154.89.105",
136136
"IPAddressASN": 396982,
137137
"IPAddressBogon": false,
138-
"EndpointTransactionID": 40001,
138+
"EndpointTransactionID": 40002,
139139
"EndpointProto": "tcp",
140140
"EndpointPort": "443",
141141
"EndpointAddress": "104.154.89.105:443",

‎internal/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/analysis_classic.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
"TCPConnectUnexplainedFailureDuringWebFetch": [],
3131
"TCPConnectUnexplainedFailureDuringConnectivityCheck": [],
3232
"TLSHandshakeExpectedFailure": [
33-
40001
33+
40002
3434
],
3535
"TLSHandshakeUnexpectedFailure": [],
3636
"TLSHandshakeUnexpectedFailureDuringWebFetch": [],
@@ -53,7 +53,7 @@
5353
"TagDepth": 0,
5454
"Type": 2,
5555
"Failure": "ssl_unknown_authority",
56-
"TransactionID": 40001,
56+
"TransactionID": 40002,
5757
"TagFetchBody": true,
5858
"DNSTransactionID": 1,
5959
"DNSDomain": "www.example.com",
@@ -67,7 +67,7 @@
6767
"IPAddress": "104.154.89.105",
6868
"IPAddressASN": 396982,
6969
"IPAddressBogon": false,
70-
"EndpointTransactionID": 40001,
70+
"EndpointTransactionID": 40002,
7171
"EndpointProto": "tcp",
7272
"EndpointPort": "443",
7373
"EndpointAddress": "104.154.89.105:443",

‎internal/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/measurement.json

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,7 @@
218218
"depth=0",
219219
"fetch_body=true"
220220
],
221-
"transaction_id": 40002
221+
"transaction_id": 40001
222222
}
223223
],
224224
"tcp_connect": [
@@ -234,7 +234,7 @@
234234
"depth=0",
235235
"fetch_body=true"
236236
],
237-
"transaction_id": 40001
237+
"transaction_id": 40002
238238
},
239239
{
240240
"ip": "93.184.216.34",
@@ -248,7 +248,7 @@
248248
"depth=0",
249249
"fetch_body=true"
250250
],
251-
"transaction_id": 40002
251+
"transaction_id": 40001
252252
}
253253
],
254254
"tls_handshakes": [
@@ -267,7 +267,7 @@
267267
"fetch_body=true"
268268
],
269269
"tls_version": "",
270-
"transaction_id": 40001
270+
"transaction_id": 40002
271271
},
272272
{
273273
"network": "tcp",
@@ -284,7 +284,7 @@
284284
"fetch_body=true"
285285
],
286286
"tls_version": "TLSv1.3",
287-
"transaction_id": 40002
287+
"transaction_id": 40001
288288
}
289289
],
290290
"x_control_request": {
@@ -301,10 +301,10 @@
301301
]
302302
},
303303
"tcp_connect": [
304-
"104.154.89.105:443",
305-
"104.154.89.105:80",
306304
"93.184.216.34:443",
307-
"93.184.216.34:80"
305+
"93.184.216.34:80",
306+
"104.154.89.105:443",
307+
"104.154.89.105:80"
308308
],
309309
"x_quic_enabled": false
310310
},

‎internal/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations.json

Lines changed: 39 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -262,45 +262,50 @@
262262
"KnownTCPEndpoints": {
263263
"40001": {
264264
"TagDepth": 0,
265-
"Type": 2,
266-
"Failure": "ssl_unknown_authority",
265+
"Type": 3,
266+
"Failure": "",
267267
"TransactionID": 40001,
268268
"TagFetchBody": true,
269-
"DNSTransactionID": 1,
269+
"DNSTransactionID": 20001,
270270
"DNSDomain": "www.example.com",
271271
"DNSLookupFailure": "",
272272
"DNSQueryType": null,
273273
"DNSEngine": null,
274274
"DNSResolvedAddrs": [
275-
"104.154.89.105"
275+
"93.184.216.34"
276276
],
277277
"IPAddressOrigin": "dns",
278-
"IPAddress": "104.154.89.105",
279-
"IPAddressASN": 396982,
278+
"IPAddress": "93.184.216.34",
279+
"IPAddressASN": 15133,
280280
"IPAddressBogon": false,
281281
"EndpointTransactionID": 40001,
282282
"EndpointProto": "tcp",
283283
"EndpointPort": "443",
284-
"EndpointAddress": "104.154.89.105:443",
284+
"EndpointAddress": "93.184.216.34:443",
285285
"TCPConnectFailure": "",
286-
"TLSHandshakeFailure": "ssl_unknown_authority",
286+
"TLSHandshakeFailure": "",
287287
"TLSServerName": "www.example.com",
288-
"HTTPRequestURL": null,
289-
"HTTPFailure": null,
290-
"HTTPResponseStatusCode": null,
291-
"HTTPResponseBodyLength": null,
292-
"HTTPResponseBodyIsTruncated": null,
293-
"HTTPResponseHeadersKeys": null,
288+
"HTTPRequestURL": "https://www.example.com/",
289+
"HTTPFailure": "",
290+
"HTTPResponseStatusCode": 200,
291+
"HTTPResponseBodyLength": 1533,
292+
"HTTPResponseBodyIsTruncated": false,
293+
"HTTPResponseHeadersKeys": {
294+
"Alt-Svc": true,
295+
"Content-Length": true,
296+
"Content-Type": true,
297+
"Date": true
298+
},
294299
"HTTPResponseLocation": null,
295-
"HTTPResponseTitle": null,
296-
"HTTPResponseIsFinal": null,
300+
"HTTPResponseTitle": "Default Web Page",
301+
"HTTPResponseIsFinal": true,
297302
"ControlDNSDomain": "www.example.com",
298303
"ControlDNSLookupFailure": "",
299304
"ControlDNSResolvedAddrs": [
300305
"93.184.216.34"
301306
],
302307
"ControlTCPConnectFailure": "",
303-
"ControlTLSHandshakeFailure": "ssl_unknown_authority",
308+
"ControlTLSHandshakeFailure": "",
304309
"ControlHTTPFailure": "",
305310
"ControlHTTPResponseStatusCode": 200,
306311
"ControlHTTPResponseBodyLength": 1533,
@@ -314,50 +319,45 @@
314319
},
315320
"40002": {
316321
"TagDepth": 0,
317-
"Type": 3,
318-
"Failure": "",
322+
"Type": 2,
323+
"Failure": "ssl_unknown_authority",
319324
"TransactionID": 40002,
320325
"TagFetchBody": true,
321-
"DNSTransactionID": 20001,
326+
"DNSTransactionID": 1,
322327
"DNSDomain": "www.example.com",
323328
"DNSLookupFailure": "",
324329
"DNSQueryType": null,
325330
"DNSEngine": null,
326331
"DNSResolvedAddrs": [
327-
"93.184.216.34"
332+
"104.154.89.105"
328333
],
329334
"IPAddressOrigin": "dns",
330-
"IPAddress": "93.184.216.34",
331-
"IPAddressASN": 15133,
335+
"IPAddress": "104.154.89.105",
336+
"IPAddressASN": 396982,
332337
"IPAddressBogon": false,
333338
"EndpointTransactionID": 40002,
334339
"EndpointProto": "tcp",
335340
"EndpointPort": "443",
336-
"EndpointAddress": "93.184.216.34:443",
341+
"EndpointAddress": "104.154.89.105:443",
337342
"TCPConnectFailure": "",
338-
"TLSHandshakeFailure": "",
343+
"TLSHandshakeFailure": "ssl_unknown_authority",
339344
"TLSServerName": "www.example.com",
340-
"HTTPRequestURL": "https://www.example.com/",
341-
"HTTPFailure": "",
342-
"HTTPResponseStatusCode": 200,
343-
"HTTPResponseBodyLength": 1533,
344-
"HTTPResponseBodyIsTruncated": false,
345-
"HTTPResponseHeadersKeys": {
346-
"Alt-Svc": true,
347-
"Content-Length": true,
348-
"Content-Type": true,
349-
"Date": true
350-
},
345+
"HTTPRequestURL": null,
346+
"HTTPFailure": null,
347+
"HTTPResponseStatusCode": null,
348+
"HTTPResponseBodyLength": null,
349+
"HTTPResponseBodyIsTruncated": null,
350+
"HTTPResponseHeadersKeys": null,
351351
"HTTPResponseLocation": null,
352-
"HTTPResponseTitle": "Default Web Page",
353-
"HTTPResponseIsFinal": true,
352+
"HTTPResponseTitle": null,
353+
"HTTPResponseIsFinal": null,
354354
"ControlDNSDomain": "www.example.com",
355355
"ControlDNSLookupFailure": "",
356356
"ControlDNSResolvedAddrs": [
357357
"93.184.216.34"
358358
],
359359
"ControlTCPConnectFailure": "",
360-
"ControlTLSHandshakeFailure": "",
360+
"ControlTLSHandshakeFailure": "ssl_unknown_authority",
361361
"ControlHTTPFailure": "",
362362
"ControlHTTPResponseStatusCode": 200,
363363
"ControlHTTPResponseBodyLength": 1533,

‎internal/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations_classic.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,11 +55,11 @@
5555
}
5656
],
5757
"KnownTCPEndpoints": {
58-
"40001": {
58+
"40002": {
5959
"TagDepth": 0,
6060
"Type": 2,
6161
"Failure": "ssl_unknown_authority",
62-
"TransactionID": 40001,
62+
"TransactionID": 40002,
6363
"TagFetchBody": true,
6464
"DNSTransactionID": 1,
6565
"DNSDomain": "www.example.com",
@@ -73,7 +73,7 @@
7373
"IPAddress": "104.154.89.105",
7474
"IPAddressASN": 396982,
7575
"IPAddressBogon": false,
76-
"EndpointTransactionID": 40001,
76+
"EndpointTransactionID": 40002,
7777
"EndpointProto": "tcp",
7878
"EndpointPort": "443",
7979
"EndpointAddress": "104.154.89.105:443",

0 commit comments

Comments
 (0)