Skip to content

Commit

Permalink
analysis: disabling calculation of sample statistics
Browse files Browse the repository at this point in the history
  • Loading branch information
@tomato42
tomato42 committed Sep 11, 2024
1 parent 555ab27 commit b16c6d1
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 20 deletions.
52 changes: 34 additions & 18 deletions tests/test_tlsfuzzer_analysis.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -926,7 +926,23 @@ def test_command_line(self):
output, True, True, True, False, False, None, None,
None, None, None, False, True, 1e-09, 4,
'measurements.csv', False, True, True, True, True,
True, True)
True, True, True)

def test_call_with_no_sample_stats(self):
output = "/tmp"
args = ["analysis.py", "-o", output, "--no-sample-stats"]
mock_init = mock.Mock()
mock_init.return_value = None
with mock.patch('tlsfuzzer.analysis.Analysis.generate_report') as mock_report:
with mock.patch('tlsfuzzer.analysis.Analysis.__init__', mock_init):
with mock.patch("sys.argv", args):
main()
mock_report.assert_called_once()
mock_init.assert_called_once_with(
output, True, True, True, False, False, None, None,
None, None, None, False, True, 1e-09, 4,
'measurements.csv', False, True, True, True, True,
True, True, False)

def test_minimal_analysis(self):
output = "/tmp"
Expand All @@ -942,7 +958,7 @@ def test_minimal_analysis(self):
output, False, False, False, False, False, None, None,
None, None, None, False, True, 1e-09, 4,
'measurements.csv', False, False, False, True, False,
False, False)
False, False, False)

def test_call_with_no_box_test(self):
output = "/tmp"
Expand All @@ -958,7 +974,7 @@ def test_call_with_no_box_test(self):
output, True, True, True, False, False, None, None,
None, None, None, False, True, 1e-09, 4,
'measurements.csv', False, True, True, True, True,
False, True)
False, True, True)

def test_call_with_no_le_sign_test(self):
output = "/tmp"
Expand All @@ -974,7 +990,7 @@ def test_call_with_no_le_sign_test(self):
output, True, True, True, False, False, None, None,
None, None, None, False, True, 1e-09, 4,
'measurements.csv', False, True, True, True, True,
True, False)
True, False, True)

def test_call_with_delay_and_CR(self):
output = "/tmp"
Expand All @@ -991,7 +1007,7 @@ def test_call_with_delay_and_CR(self):
output, True, True, True, False, False, None, None,
None, 3.5, '\n', False, True, 1e-09, 4,
'measurements.csv', False, True, True, True,
True, True, True)
True, True, True, True)

def test_call_with_workers(self):
output = "/tmp"
Expand All @@ -1007,7 +1023,7 @@ def test_call_with_workers(self):
output, True, True, True, False, False, None, None,
200, None, None, False, True, 1e-09, 4,
'measurements.csv', False, True, True, True,
True, True, True)
True, True, True, True)

def test_call_with_verbose(self):
output = "/tmp"
Expand All @@ -1023,7 +1039,7 @@ def test_call_with_verbose(self):
output, True, True, True, False, True, None, None,
None, None, None, False, True, 1e-09, 4,
'measurements.csv', False, True, True, True,
True, True, True)
True, True, True, True)

def test_call_with_multithreaded_plots(self):
output = "/tmp"
Expand All @@ -1039,7 +1055,7 @@ def test_call_with_multithreaded_plots(self):
output, True, True, True, True, False, None, None,
None, None, None, False, True, 1e-09, 4,
'measurements.csv', False, True, True, True,
True, True, True)
True, True, True, True)

def test_call_with_no_plots(self):
output = "/tmp"
Expand All @@ -1057,7 +1073,7 @@ def test_call_with_no_plots(self):
output, False, False, False, False, False, None, None,
None, None, None, False, True, 1e-09, 4,
'measurements.csv', False, True, True, True,
False, True, True)
False, True, True, True)

def test_call_with_frequency(self):
output = "/tmp"
Expand All @@ -1073,7 +1089,7 @@ def test_call_with_frequency(self):
output, True, True, True, False, False, 10*1e6, None,
None, None, None, False, True, 1e-09, 4,
'measurements.csv', False, True, True, True,
True, True, True)
True, True, True, True)

def test_call_with_alpha(self):
output = "/tmp"
Expand All @@ -1089,7 +1105,7 @@ def test_call_with_alpha(self):
output, True, True, True, False, False, None, 1e-3,
None, None, None, False, True, 1e-09, 4,
'measurements.csv', False, True, True, True,
True, True, True)
True, True, True, True)

def test_call_with_bit_size_measurements(self):
output = "/tmp"
Expand All @@ -1107,7 +1123,7 @@ def test_call_with_bit_size_measurements(self):
output, True, True, True, False, False, None, None,
None, None, None, True, True, 1e-09, 4,
'measurements.csv', False, True, True, True,
True, True, True)
True, True, True, True)

def test_call_with_skip_sanity(self):
output = "/tmp"
Expand All @@ -1125,7 +1141,7 @@ def test_call_with_skip_sanity(self):
output, True, True, True, False, False, None, None,
None, None, None, True, True, 1e-09, 4,
'measurements.csv', True, True, True, True,
True, True, True)
True, True, True, True)

def test_call_with_custom_measurements_filename(self):
output = "/tmp"
Expand All @@ -1145,7 +1161,7 @@ def test_call_with_custom_measurements_filename(self):
output, True, True, True, False, False, None, None,
None, None, None, True, True, 1e-09, 4,
measurements_filename, False, True, True, True,
True, True, True)
True, True, True, True)

def test_call_with_no_smart_analysis(self):
output = "/tmp"
Expand All @@ -1164,7 +1180,7 @@ def test_call_with_no_smart_analysis(self):
output, True, True, True, False, False, None, None,
None, None, None, True, False, 1e-09, 4,
'measurements.csv', False, True, True, True,
True, True, True)
True, True, True, True)

def test_call_with_parametrized_smart_analysis(self):
output = "/tmp"
Expand All @@ -1187,7 +1203,7 @@ def test_call_with_parametrized_smart_analysis(self):
None, None, None, True, True,
bit_size_desire_ci * 1e-9, bit_recognition_size,
'measurements.csv', False, True, True, True,
True, True, True)
True, True, True, True)

def test_call_with_Hamming_weight(self):
output = "/tmp"
Expand All @@ -1204,7 +1220,7 @@ def test_call_with_Hamming_weight(self):
output, True, True, True, False, False, None, None,
None, None, None, True, True, 1e-9, 4,
'measurements.csv', False, True, True, True,
True, True, True)
True, True, True, True)
mock_report.assert_called_once_with(
bit_size=False, hamming_weight=True)

Expand All @@ -1224,7 +1240,7 @@ def test_call_Hamming_weight_with_minimal_analysis(self):
output, True, True, True, False, False, None, None,
None, None, None, True, True, 1e-9, 4,
'measurements.csv', False, False, False, False,
True, True, True)
True, True, True, True)
mock_report.assert_called_once_with(
bit_size=False, hamming_weight=True)

Expand Down
14 changes: 12 additions & 2 deletions tlsfuzzer/analysis.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ def help_msg():
--no-t-test Don't run the paired sample t-test
--no-sign-test [Hamming weight only] Don't run the sign test
--no-le-sign-test Don't run the less-equal, greater-equal sign tests
--no-sample-stats Don't calculate sample statistics (sample_stats.csv)
--minimal-analysis Run just the pairwise sign tests, Friedman test, and
bootstrapping of confidence intervals (i.e. minimal amount
of calculation necessary to generate report.txt)
Expand Down Expand Up @@ -123,6 +124,7 @@ def main():
box_plot = True
box_test = True
le_sign_test = True
sample_stats = True
verbose = False
clock_freq = None
alpha = None
Expand All @@ -149,6 +151,7 @@ def main():
"no-box-test",
"no-wilcoxon-test",
"no-le-sign-test",
"no-sample-stats",
"minimal-analysis",
"multithreaded-graph",
"clock-frequency=",
Expand Down Expand Up @@ -189,6 +192,8 @@ def main():
wilcoxon_test = False
elif opt == "--no-le-sign-test":
le_sign_test = False
elif opt == "--no-sample-stats":
sample_stats = False
elif opt == "--minimal-analysis":
ecdf_plot = False
scatter_plot = False
Expand All @@ -198,6 +203,7 @@ def main():
wilcoxon_test = False
t_test = False
le_sign_test = False
sample_stats = False
elif opt == "--multithreaded-graph":
multithreaded_graph = True
elif opt == "--clock-frequency":
Expand Down Expand Up @@ -235,7 +241,7 @@ def main():
smart_analysis, bit_size_desired_ci,
bit_recognition_size, measurements_filename,
skip_sanity, wilcoxon_test, t_test, sign_test,
box_plot, box_test, le_sign_test)
box_plot, box_test, le_sign_test, sample_stats)

ret = analysis.generate_report(
bit_size=bit_size_analysis,
Expand All @@ -258,7 +264,8 @@ def __init__(self, output, draw_ecdf_plot=True, draw_scatter_plot=True,
bit_size_desired_ci=1e-9, bit_recognition_size=4,
measurements_filename="measurements.csv", skip_sanity=False,
run_wilcoxon_test=True, run_t_test=True, run_sign_test=True,
draw_box_plot=True, run_box_test=True, run_le_sign_test=True):
draw_box_plot=True, run_box_test=True, run_le_sign_test=True,
gen_sample_stats=True):
self.verbose = verbose
self.output = output
self.clock_frequency = clock_frequency
Expand All @@ -272,6 +279,7 @@ def __init__(self, output, draw_ecdf_plot=True, draw_scatter_plot=True,
self.run_t_test = run_t_test
self.run_sign_test = run_sign_test
self.run_le_sign_test = run_le_sign_test
self.gen_sample_stats = gen_sample_stats
self.multithreaded_graph = multithreaded_graph
self.workers = workers
if alpha is None:
Expand Down Expand Up @@ -1276,6 +1284,8 @@ def _write_legend(self):

def _write_sample_stats(self):
"""Write summary statistics of samples to sample_stats.csv file."""
if not self.gen_sample_stats:
return None
if self.verbose:
start_time = time.time()
print("[i] Writing summary statistics of samples to file")
Expand Down

0 comments on commit b16c6d1

Please sign in to comment.