Skip to content

Commit

Permalink
Merge pull request tlsfuzzer#891 from tlsfuzzer/no-pairwise-print
Browse files Browse the repository at this point in the history 
Don't print pairwise test results
  • Loading branch information
@tomato42
tomato42 authored Jan 24, 2025
2 parents 7fd19a6 + 6634b6a commit 9cb621d
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 44 deletions.
8 changes: 7 additions & 1 deletion scripts/test-bleichenbacher-timing-pregenerate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ def help_msg():
print(" --verbose-analysis Enable verbose progress of analysis.")
print(" --status-delay num How often to print the status line. Default: 2.0 seconds")
print(" --status-newline Use a newline for line end instead of carriage return.")
print(" --summary-only Print summary result only, don't print pairwise results")
print(" --no-alert Don't expect the server to send an alert before closing connection.")
print(" --help this message")

Expand Down Expand Up @@ -156,6 +157,7 @@ def main():
delay = None
carriage_return = None
no_alert = False
summary_only = False

argv = sys.argv[1:]
opts, args = getopt.getopt(argv,
Expand All @@ -174,6 +176,7 @@ def main():
"verbose-analysis",
"status-delay=",
"status-newline",
"summary-only",
"no-alert"])
for opt, arg in opts:
if opt == '-h':
Expand Down Expand Up @@ -240,6 +243,8 @@ def main():
verbose_analysis = True
elif opt == "--status-delay":
delay = float(arg)
elif opt == "--summary-only":
summary_only = True
elif opt == "--status-newline":
carriage_return = '\n'
elif opt == "--no-alert":
Expand Down Expand Up @@ -1049,7 +1054,8 @@ def main():
no_quickack=no_quickack,
verbose_analysis=verbose_analysis,
delay=delay,
carriage_return=carriage_return)
carriage_return=carriage_return,
summary_only=summary_only)
print("Pre-generating pre-master secret values...")

with open(
Expand Down
54 changes: 35 additions & 19 deletions tests/test_tlsfuzzer_analysis.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -926,7 +926,7 @@ def test_command_line(self):
output, True, True, True, False, False, None, None,
None, None, None, False, True, 1e-09,
'measurements.csv', False, True, True, True, True,
True, True, True)
True, True, True, False)

def test_call_with_no_sample_stats(self):
output = "/tmp"
Expand All @@ -942,7 +942,7 @@ def test_call_with_no_sample_stats(self):
output, True, True, True, False, False, None, None,
None, None, None, False, True, 1e-09,
'measurements.csv', False, True, True, True, True,
True, True, False)
True, True, False, False)

def test_minimal_analysis(self):
output = "/tmp"
Expand All @@ -958,7 +958,7 @@ def test_minimal_analysis(self):
output, False, False, False, False, False, None, None,
None, None, None, False, True, 1e-09,
'measurements.csv', False, False, False, True, False,
False, False, False)
False, False, False, False)

def test_call_with_no_box_test(self):
output = "/tmp"
Expand All @@ -974,7 +974,7 @@ def test_call_with_no_box_test(self):
output, True, True, True, False, False, None, None,
None, None, None, False, True, 1e-09,
'measurements.csv', False, True, True, True, True,
False, True, True)
False, True, True, False)

def test_call_with_no_le_sign_test(self):
output = "/tmp"
Expand All @@ -990,7 +990,7 @@ def test_call_with_no_le_sign_test(self):
output, True, True, True, False, False, None, None,
None, None, None, False, True, 1e-09,
'measurements.csv', False, True, True, True, True,
True, False, True)
True, False, True, False)

def test_call_with_delay_and_CR(self):
output = "/tmp"
Expand All @@ -1007,7 +1007,7 @@ def test_call_with_delay_and_CR(self):
output, True, True, True, False, False, None, None,
None, 3.5, '\n', False, True, 1e-09,
'measurements.csv', False, True, True, True,
True, True, True, True)
True, True, True, True, False)

def test_call_with_workers(self):
output = "/tmp"
Expand All @@ -1023,7 +1023,7 @@ def test_call_with_workers(self):
output, True, True, True, False, False, None, None,
200, None, None, False, True, 1e-09,
'measurements.csv', False, True, True, True,
True, True, True, True)
True, True, True, True, False)

def test_call_with_verbose(self):
output = "/tmp"
Expand All @@ -1039,7 +1039,23 @@ def test_call_with_verbose(self):
output, True, True, True, False, True, None, None,
None, None, None, False, True, 1e-09,
'measurements.csv', False, True, True, True,
True, True, True, True)
True, True, True, True, False)

def test_call_with_summary_only(self):
output = "/tmp"
args = ["analysis.py", "-o", output, "--summary-only"]
mock_init = mock.Mock()
mock_init.return_value = None
with mock.patch('tlsfuzzer.analysis.Analysis.generate_report') as mock_report:
with mock.patch('tlsfuzzer.analysis.Analysis.__init__', mock_init):
with mock.patch("sys.argv", args):
main()
mock_report.assert_called_once()
mock_init.assert_called_once_with(
output, True, True, True, False, False, None, None,
None, None, None, False, True, 1e-09,
'measurements.csv', False, True, True, True,
True, True, True, True, True)

def test_call_with_multithreaded_plots(self):
output = "/tmp"
Expand All @@ -1055,7 +1071,7 @@ def test_call_with_multithreaded_plots(self):
output, True, True, True, True, False, None, None,
None, None, None, False, True, 1e-09,
'measurements.csv', False, True, True, True,
True, True, True, True)
True, True, True, True, False)

def test_call_with_no_plots(self):
output = "/tmp"
Expand All @@ -1073,7 +1089,7 @@ def test_call_with_no_plots(self):
output, False, False, False, False, False, None, None,
None, None, None, False, True, 1e-09,
'measurements.csv', False, True, True, True,
False, True, True, True)
False, True, True, True, False)

def test_call_with_frequency(self):
output = "/tmp"
Expand All @@ -1089,7 +1105,7 @@ def test_call_with_frequency(self):
output, True, True, True, False, False, 10*1e6, None,
None, None, None, False, True, 1e-09,
'measurements.csv', False, True, True, True,
True, True, True, True)
True, True, True, True, False)

def test_call_with_alpha(self):
output = "/tmp"
Expand All @@ -1105,7 +1121,7 @@ def test_call_with_alpha(self):
output, True, True, True, False, False, None, 1e-3,
None, None, None, False, True, 1e-09,
'measurements.csv', False, True, True, True,
True, True, True, True)
True, True, True, True, False)

def test_call_with_bit_size_measurements(self):
output = "/tmp"
Expand All @@ -1123,7 +1139,7 @@ def test_call_with_bit_size_measurements(self):
output, True, True, True, False, False, None, None,
None, None, None, True, True, 1e-09,
'measurements.csv', False, True, True, True,
True, True, True, True)
True, True, True, True, False)

def test_call_with_skip_sanity(self):
output = "/tmp"
Expand All @@ -1141,7 +1157,7 @@ def test_call_with_skip_sanity(self):
output, True, True, True, False, False, None, None,
None, None, None, True, True, 1e-09,
'measurements.csv', True, True, True, True,
True, True, True, True)
True, True, True, True, False)

def test_call_with_custom_measurements_filename(self):
output = "/tmp"
Expand All @@ -1161,7 +1177,7 @@ def test_call_with_custom_measurements_filename(self):
output, True, True, True, False, False, None, None,
None, None, None, True, True, 1e-09,
measurements_filename, False, True, True, True,
True, True, True, True)
True, True, True, True, False)

def test_call_with_no_smart_analysis(self):
output = "/tmp"
Expand All @@ -1180,7 +1196,7 @@ def test_call_with_no_smart_analysis(self):
output, True, True, True, False, False, None, None,
None, None, None, True, False, 1e-09,
'measurements.csv', False, True, True, True,
True, True, True, True)
True, True, True, True, False)

def test_call_with_parametrized_smart_analysis(self):
output = "/tmp"
Expand All @@ -1200,7 +1216,7 @@ def test_call_with_parametrized_smart_analysis(self):
output, True, True, True, False, False, None, None,
None, None, None, True, True,
bit_size_desire_ci * 1e-9, 'measurements.csv', False,
True, True, True, True, True, True, True)
True, True, True, True, True, True, True, False)

def test_call_with_Hamming_weight(self):
output = "/tmp"
Expand All @@ -1217,7 +1233,7 @@ def test_call_with_Hamming_weight(self):
output, True, True, True, False, False, None, None,
None, None, None, True, True, 1e-9,
'measurements.csv', False, True, True, True,
True, True, True, True)
True, True, True, True, False)
mock_report.assert_called_once_with(
bit_size=False, hamming_weight=True)

Expand All @@ -1237,7 +1253,7 @@ def test_call_Hamming_weight_with_minimal_analysis(self):
output, True, True, True, False, False, None, None,
None, None, None, True, True, 1e-9,
'measurements.csv', False, False, False, False,
True, True, True, True)
True, True, True, True, False)
mock_report.assert_called_once_with(
bit_size=False, hamming_weight=True)

Expand Down
59 changes: 37 additions & 22 deletions tlsfuzzer/analysis.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ def help_msg():
similar.
--alpha num Acceptable probability of a false positive. Default: 1e-5.
--verbose Print the current task
--summary-only Print only summary of the test, skip pairwise results
--workers num Number of worker processes to use for paralelizable
computation. More workers will finish analysis faster, but
will require more memory to do so. By default: number of
Expand Down Expand Up @@ -132,6 +133,7 @@ def main():
sign_test = True
bit_size_analysis = False
smart_analysis = True
summary_only = False
bit_size_desired_ci = 1e-9
measurements_filename = "measurements.csv"
skip_sanity = False
Expand Down Expand Up @@ -160,6 +162,7 @@ def main():
"measurements=",
"skip-sanity",
"Hamming-weight",
"summary-only",
"verbose"])

for opt, arg in opts:
Expand Down Expand Up @@ -208,6 +211,8 @@ def main():
workers = int(arg)
elif opt == "--verbose":
verbose = True
elif opt == "--summary-only":
summary_only = True
elif opt == "--status-delay":
delay = float(arg)
elif opt == "--status-newline":
Expand All @@ -233,7 +238,7 @@ def main():
smart_analysis, bit_size_desired_ci,
measurements_filename, skip_sanity, wilcoxon_test,
t_test, sign_test, box_plot, box_test,
le_sign_test, sample_stats)
le_sign_test, sample_stats, summary_only)

ret = analysis.generate_report(
bit_size=bit_size_analysis,
Expand All @@ -257,8 +262,9 @@ def __init__(self, output, draw_ecdf_plot=True, draw_scatter_plot=True,
measurements_filename="measurements.csv", skip_sanity=False,
run_wilcoxon_test=True, run_t_test=True, run_sign_test=True,
draw_box_plot=True, run_box_test=True, run_le_sign_test=True,
gen_sample_stats=True):
gen_sample_stats=True, summary_only=False):
self.verbose = verbose
self.summary_only = summary_only
self.output = output
self.clock_frequency = clock_frequency
self.class_names = []
Expand Down Expand Up @@ -1205,41 +1211,50 @@ def _write_individual_results(self):
if self.run_box_test:
result = box_results[pair]
if result:
print("Box test {0} vs {1}: {0} {2} {1}".format(
index1,
index2,
result))
if not self.summary_only:
print("Box test {0} vs {1}: {0} {2} {1}".format(
index1,
index2,
result))
box_write = result
else:
print("Box test {} vs {}: No difference".format(
index1,
index2))
if self.run_wilcoxon_test:
if not self.summary_only:
print("Box test {} vs {}: No difference".format(
index1,
index2))
if self.run_wilcoxon_test and not self.summary_only:
print("Wilcoxon signed-rank test {} vs {}: {:.3}"
.format(index1, index2, wilcox_results[pair]))
print("Sign test {} vs {}: {:.3}"
.format(index1, index2, sign_results[pair]))
if not self.summary_only:
print("Sign test {} vs {}: {:.3}"
.format(index1, index2, sign_results[pair]))
if self.run_le_sign_test:
print("Sign test, probability that {1} < {0}: {2:.3}"
.format(index1, index2, sign_less_results[pair]))
print("Sign test, probability that {1} > {0}: {2:.3}"
.format(index1, index2, sign_greater_results[pair]))
if not self.summary_only:
print("Sign test, probability that {1} < {0}: {2:.3}"
.format(index1, index2, sign_less_results[pair]))
print("Sign test, probability that {1} > {0}: {2:.3}"
.format(index1, index2,
sign_greater_results[pair]))
if sign_results[pair] > 0.05:
sign_test_relation = "="
elif sign_less_results[pair] > sign_greater_results[pair]:
sign_test_relation = "<"
else:
sign_test_relation = ">"
if not self.summary_only:
print("Sign test interpretation: {} {} {}"
.format(index2, sign_test_relation, index1))
if self.run_t_test:
if self.run_t_test and not self.summary_only:
print("Dependent t-test for paired samples {} vs {}: {:.3}"
.format(index1, index2, ttest_results[pair]))
print("{} vs {} stats: mean: {:.3}, SD: {:.3}, median: {:.3}, "
"IQR: {:.3}, MAD: {:.3}".format(
index1, index2, diff_stats["mean"], diff_stats["SD"],
diff_stats["median"], diff_stats["IQR"],
diff_stats["MAD"]))
if not self.summary_only:
print("{} vs {} stats: mean: {:.3}, SD: {:.3}, "
"median: {:.3}, "
"IQR: {:.3}, MAD: {:.3}".format(
index1, index2, diff_stats["mean"],
diff_stats["SD"],
diff_stats["median"], diff_stats["IQR"],
diff_stats["MAD"]))

# If either of the pairwise tests shows a small p-value with
# Bonferroni correction consider it a possible side-channel
Expand Down
6 changes: 4 additions & 2 deletions tlsfuzzer/timing_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ class TimingRunner:
def __init__(self, name, tests, out_dir, ip_address, port, interface,
affinity=None, skip_extract=False, skip_analysis=False,
alpha=None, no_quickack=False, verbose_analysis=False,
delay=None, carriage_return=None):
delay=None, carriage_return=None, summary_only=False):
"""
Check if tcpdump is present and setup instance parameters.
Expand Down Expand Up @@ -65,6 +65,7 @@ def __init__(self, name, tests, out_dir, ip_address, port, interface,
self.verbose_analysis = verbose_analysis
self.delay = delay
self.carriage_return = carriage_return
self.summary_only = summary_only

self.tcpdump_running = True
self.tcpdump_output = None
Expand Down Expand Up @@ -203,7 +204,8 @@ def analyse(self):
analysis = Analysis(self.out_dir, alpha=self.alpha,
verbose=self.verbose_analysis,
delay=self.delay,
carriage_return=self.carriage_return)
carriage_return=self.carriage_return,
summary_only=self.summary_only)
return analysis.generate_report()

print("Analysis is not available. "
Expand Down

0 comments on commit 9cb621d

Please sign in to comment.