From fd2c4b3034b59e6d9747214f8e66ddee693d6a90 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 11 Jan 2024 17:43:33 +0100 Subject: [PATCH 1/3] analysis: ability to run minimal analysis for actionable results --- tests/test_tlsfuzzer_analysis.py | 98 +++++++++++++--- tlsfuzzer/analysis.py | 192 +++++++++++++++++++++---------- 2 files changed, 212 insertions(+), 78 deletions(-) diff --git a/tests/test_tlsfuzzer_analysis.py b/tests/test_tlsfuzzer_analysis.py index efceacd49..db8c68bbc 100644 --- a/tests/test_tlsfuzzer_analysis.py +++ b/tests/test_tlsfuzzer_analysis.py @@ -896,7 +896,56 @@ def test_command_line(self): mock_init.assert_called_once_with( output, True, True, True, False, False, None, None, None, None, None, False, True, 1e-09, 4, - 'measurements.csv', False, True, True, True) + 'measurements.csv', False, True, True, True, True, + True, True) + + def test_minimal_analysis(self): + output = "/tmp" + args = ["analysis.py", "-o", output, "--minimal-analysis"] + mock_init = mock.Mock() + mock_init.return_value = None + with mock.patch('tlsfuzzer.analysis.Analysis.generate_report') as mock_report: + with mock.patch('tlsfuzzer.analysis.Analysis.__init__', mock_init): + with mock.patch("sys.argv", args): + main() + mock_report.assert_called_once() + mock_init.assert_called_once_with( + output, False, False, False, False, False, None, None, + None, None, None, False, True, 1e-09, 4, + 'measurements.csv', False, False, False, True, False, + False, False) + + def test_call_with_no_box_test(self): + output = "/tmp" + args = ["analysis.py", "-o", output, "--no-box-test"] + mock_init = mock.Mock() + mock_init.return_value = None + with mock.patch('tlsfuzzer.analysis.Analysis.generate_report') as mock_report: + with mock.patch('tlsfuzzer.analysis.Analysis.__init__', mock_init): + with mock.patch("sys.argv", args): + main() + mock_report.assert_called_once() + mock_init.assert_called_once_with( + output, True, True, True, False, False, None, None, + None, None, None, False, True, 1e-09, 4, + 'measurements.csv', False, True, True, True, True, + False, True) + + def test_call_with_no_le_sign_test(self): + output = "/tmp" + args = ["analysis.py", "-o", output, "--no-le-sign-test"] + mock_init = mock.Mock() + mock_init.return_value = None + with mock.patch('tlsfuzzer.analysis.Analysis.generate_report') as mock_report: + with mock.patch('tlsfuzzer.analysis.Analysis.__init__', mock_init): + with mock.patch("sys.argv", args): + main() + mock_report.assert_called_once() + mock_init.assert_called_once_with( + output, True, True, True, False, False, None, None, + None, None, None, False, True, 1e-09, 4, + 'measurements.csv', False, True, True, True, True, + True, False) def test_call_with_delay_and_CR(self): output = "/tmp" @@ -912,7 +961,8 @@ def test_call_with_delay_and_CR(self): mock_init.assert_called_once_with( output, True, True, True, False, False, None, None, None, 3.5, '\n', False, True, 1e-09, 4, - 'measurements.csv', False, True, True, True) + 'measurements.csv', False, True, True, True, + True, True, True) def test_call_with_workers(self): output = "/tmp" @@ -927,7 +977,8 @@ def test_call_with_workers(self): mock_init.assert_called_once_with( output, True, True, True, False, False, None, None, 200, None, None, False, True, 1e-09, 4, - 'measurements.csv', False, True, True, True) + 'measurements.csv', False, True, True, True, + True, True, True) def test_call_with_verbose(self): output = "/tmp" @@ -942,7 +993,8 @@ def test_call_with_verbose(self): mock_init.assert_called_once_with( output, True, True, True, False, True, None, None, None, None, None, False, True, 1e-09, 4, - 'measurements.csv', False, True, True, True) + 'measurements.csv', False, True, True, True, + True, True, True) def test_call_with_multithreaded_plots(self): output = "/tmp" @@ -957,12 +1009,14 @@ def test_call_with_multithreaded_plots(self): mock_init.assert_called_once_with( output, True, True, True, True, False, None, None, None, None, None, False, True, 1e-09, 4, - 'measurements.csv', False, True, True, True) + 'measurements.csv', False, True, True, True, + True, True, True) def test_call_with_no_plots(self): output = "/tmp" args = ["analysis.py", "-o", output, "--no-ecdf-plot", - "--no-scatter-plot", "--no-conf-interval-plot"] + "--no-scatter-plot", "--no-conf-interval-plot", + "--no-box-plot"] mock_init = mock.Mock() mock_init.return_value = None with mock.patch('tlsfuzzer.analysis.Analysis.generate_report') as mock_report: @@ -973,7 +1027,8 @@ def test_call_with_no_plots(self): mock_init.assert_called_once_with( output, False, False, False, False, False, None, None, None, None, None, False, True, 1e-09, 4, - 'measurements.csv', False, True, True, True) + 'measurements.csv', False, True, True, True, + False, True, True) def test_call_with_frequency(self): output = "/tmp" @@ -988,7 +1043,8 @@ def test_call_with_frequency(self): mock_init.assert_called_once_with( output, True, True, True, False, False, 10*1e6, None, None, None, None, False, True, 1e-09, 4, - 'measurements.csv', False, True, True, True) + 'measurements.csv', False, True, True, True, + True, True, True) def test_call_with_alpha(self): output = "/tmp" @@ -1003,7 +1059,8 @@ def test_call_with_alpha(self): mock_init.assert_called_once_with( output, True, True, True, False, False, None, 1e-3, None, None, None, False, True, 1e-09, 4, - 'measurements.csv', False, True, True, True) + 'measurements.csv', False, True, True, True, + True, True, True) def test_call_with_bit_size_measurements(self): output = "/tmp" @@ -1020,7 +1077,8 @@ def test_call_with_bit_size_measurements(self): mock_init.assert_called_once_with( output, True, True, True, False, False, None, None, None, None, None, True, True, 1e-09, 4, - 'measurements.csv', False, True, True, True) + 'measurements.csv', False, True, True, True, + True, True, True) def test_call_with_skip_sanity(self): output = "/tmp" @@ -1037,7 +1095,8 @@ def test_call_with_skip_sanity(self): mock_init.assert_called_once_with( output, True, True, True, False, False, None, None, None, None, None, True, True, 1e-09, 4, - 'measurements.csv', True, True, True, True) + 'measurements.csv', True, True, True, True, + True, True, True) def test_call_with_custom_measurements_filename(self): output = "/tmp" @@ -1056,7 +1115,8 @@ def test_call_with_custom_measurements_filename(self): mock_init.assert_called_once_with( output, True, True, True, False, False, None, None, None, None, None, True, True, 1e-09, 4, - measurements_filename, False, True, True, True) + measurements_filename, False, True, True, True, + True, True, True) def test_call_with_no_smart_analysis(self): output = "/tmp" @@ -1074,7 +1134,8 @@ def test_call_with_no_smart_analysis(self): mock_init.assert_called_once_with( output, True, True, True, False, False, None, None, None, None, None, True, False, 1e-09, 4, - 'measurements.csv', False, True, True, True) + 'measurements.csv', False, True, True, True, + True, True, True) def test_call_with_parametrized_smart_analysis(self): output = "/tmp" @@ -1096,7 +1157,8 @@ def test_call_with_parametrized_smart_analysis(self): output, True, True, True, False, False, None, None, None, None, None, True, True, bit_size_desire_ci * 1e-9, bit_recognition_size, - 'measurements.csv', False, True, True, True) + 'measurements.csv', False, True, True, True, + True, True, True) def test_call_with_Hamming_weight(self): output = "/tmp" @@ -1112,7 +1174,8 @@ def test_call_with_Hamming_weight(self): mock_init.assert_called_once_with( output, True, True, True, False, False, None, None, None, None, None, True, True, 1e-9, 4, - 'measurements.csv', False, True, True, True) + 'measurements.csv', False, True, True, True, + True, True, True) mock_report.assert_called_once_with( bit_size=False, hamming_weight=True) @@ -1131,7 +1194,8 @@ def test_call_Hamming_weight_with_minimal_analysis(self): mock_init.assert_called_once_with( output, True, True, True, False, False, None, None, None, None, None, True, True, 1e-9, 4, - 'measurements.csv', False, False, False, False) + 'measurements.csv', False, False, False, False, + True, True, True) mock_report.assert_called_once_with( bit_size=False, hamming_weight=True) @@ -2380,4 +2444,4 @@ def test_hamming_analysis_not_verbose(self): self.assertLess(1e-6, analysis.skillings_mack_test( os.path.join(tmpdirname, "measurements.bin"))) - self.assertEqual(ret, 0) \ No newline at end of file + self.assertEqual(ret, 0) diff --git a/tlsfuzzer/analysis.py b/tlsfuzzer/analysis.py index 12979970f..f889f590a 100644 --- a/tlsfuzzer/analysis.py +++ b/tlsfuzzer/analysis.py @@ -55,13 +55,18 @@ def help_msg(): print("""Usage: analysis [-o output] -o output Directory where to place results (required) and where timing.csv or measurements.csv is located - --no-ecdf-plot Don't create the ecdf_plot.png file - --no-scatter-plot Don't create the scatter_plot.png file - --no-conf-interval-plot Don't create the conf_interval_plot*.png files + --no-box-plot Don't create the box_plot.png file + --no-ecdf-plot Don't create the ECDF graphs + --no-scatter-plot Don't create the scatter plot graphs + --no-conf-interval-plot Don't create the confidence interval graphs + --no-box-test Don't run the box test --no-wilcoxon-test Don't run the Wilcoxon signed rank test - for pairwise measurements - --no-t-test Don't run the paired sample t-test for pairwise measurements - --no-sign-test [Hamming weight only] Don't run the sign test + --no-t-test Don't run the paired sample t-test + --no-sign-test [Hamming weight only] Don't run the sign test + --no-le-sign-test Don't run the less-equal, greater-equal sign tests + --minimal-analysis Run just the pairwise sign tests, Friedman test, and + bootstrapping of confidence intervals (i.e. minimal amount + of calculation necessary to generate report.txt) --multithreaded-graph Create graph and calculate statistical tests at the same time. Note: this increases memory usage of analysis by a factor of 8. @@ -115,6 +120,9 @@ def main(): scatter_plot = True conf_int_plot = True multithreaded_graph = False + box_plot = True + box_test = True + le_sign_test = True verbose = False clock_freq = None alpha = None @@ -137,7 +145,11 @@ def main(): "no-conf-interval-plot", "no-t-test", "no-sign-test", + "no-box-plot", + "no-box-test", "no-wilcoxon-test", + "no-le-sign-test", + "minimal-analysis", "multithreaded-graph", "clock-frequency=", "alpha=", @@ -169,8 +181,23 @@ def main(): sign_test = False elif opt == "--no-t-test": t_test = False + elif opt == "--no-box-plot": + box_plot = False + elif opt == "--no-box-test": + box_test = False elif opt == "--no-wilcoxon-test": wilcoxon_test = False + elif opt == "--no-le-sign-test": + le_sign_test = False + elif opt == "--minimal-analysis": + ecdf_plot = False + scatter_plot = False + conf_int_plot = False + box_plot = False + box_test = False + wilcoxon_test = False + t_test = False + le_sign_test = False elif opt == "--multithreaded-graph": multithreaded_graph = True elif opt == "--clock-frequency": @@ -207,7 +234,8 @@ def main(): bit_size_analysis or hamming_weight_analysis, smart_analysis, bit_size_desired_ci, bit_recognition_size, measurements_filename, - skip_sanity, wilcoxon_test, t_test, sign_test) + skip_sanity, wilcoxon_test, t_test, sign_test, + box_plot, box_test, le_sign_test) ret = analysis.generate_report( bit_size=bit_size_analysis, @@ -229,7 +257,8 @@ def __init__(self, output, draw_ecdf_plot=True, draw_scatter_plot=True, bit_size_analysis=False, smart_bit_size_analysis=True, bit_size_desired_ci=1e-9, bit_recognition_size=4, measurements_filename="measurements.csv", skip_sanity=False, - run_wilcoxon_test=True, run_t_test=True, run_sign_test=True): + run_wilcoxon_test=True, run_t_test=True, run_sign_test=True, + draw_box_plot=True, run_box_test=True, run_le_sign_test=True): self.verbose = verbose self.output = output self.clock_frequency = clock_frequency @@ -237,9 +266,12 @@ def __init__(self, output, draw_ecdf_plot=True, draw_scatter_plot=True, self.draw_ecdf_plot = draw_ecdf_plot self.draw_scatter_plot = draw_scatter_plot self.draw_conf_interval_plot = draw_conf_interval_plot + self.draw_box_plot = draw_box_plot + self.run_box_test = run_box_test self.run_wilcoxon_test = run_wilcoxon_test self.run_t_test = run_t_test self.run_sign_test = run_sign_test + self.run_le_sign_test = run_le_sign_test self.multithreaded_graph = multithreaded_graph self.workers = workers if alpha is None: @@ -421,6 +453,8 @@ def _box_test(data1, data2, quantile_start, quantile_end): def box_test(self): """Cross-test all classes with the box test""" + if not self.run_box_test: + return None if self.verbose: start_time = time.time() print("[i] Starting the box_test") @@ -438,6 +472,8 @@ def _wilcox_test(data1, data2): def wilcoxon_test(self): """Cross-test all classes with the Wilcoxon signed-rank test""" + if not self.run_wilcoxon_test: + return None if self.verbose: start_time = time.time() print("[i] Starting Wilcoxon signed-rank test") @@ -455,6 +491,8 @@ def _rel_t_test(data1, data2): def rel_t_test(self): """Cross-test all classes using the t-test for dependent, paired samples.""" + if not self.run_t_test: + return None if self.verbose: start_time = time.time() print("[i] Starting t-test for dependent, paired samples") @@ -573,6 +611,8 @@ def _calc_percentiles(self): def box_plot(self): """Generate box plot for the test classes.""" + if not self.draw_box_plot: + return None if self.verbose: start_time = time.time() print("[i] Generating the box plot") @@ -1097,8 +1137,12 @@ def _write_individual_results(self): box_results = self.box_test() wilcox_results = self.wilcoxon_test() sign_results = self.sign_test() - sign_less_results = self.sign_test(alternative="less") - sign_greater_results = self.sign_test(alternative="greater") + if self.run_le_sign_test: + sign_less_results = self.sign_test(alternative="less") + sign_greater_results = self.sign_test(alternative="greater") + else: + sign_less_results = None + sign_greater_results = None ttest_results = self.rel_t_test() desc_stats = self.desc_stats() @@ -1107,45 +1151,58 @@ def _write_individual_results(self): sign_p_vals = [] with open(report_filename, 'w') as file: writer = csv.writer(file) - writer.writerow(["Class 1", "Class 2", "Box test", - "Wilcoxon signed-rank test", - "Sign test", "Sign test less", - "Sign test greater", - "paired t-test", "mean", "SD", - "median", "IQR", "MAD"]) + columns = ["Class 1", "Class 2"] + if self.run_box_test: + columns += ["Box test"] + if self.run_wilcoxon_test: + columns += ["Wilcoxon signed-rank test"] + columns += ["Sign test"] + if self.run_le_sign_test: + columns += ["Sign test less", "Sign test greater"] + if self.run_t_test: + columns += ["paired t-test"] + columns += ["mean", "SD", "median", "IQR", "MAD"] + writer.writerow(columns) worst_pair = None worst_p = None worst_median_difference = None - for pair, result in sorted(box_results.items()): + for pair, result in sorted(sign_results.items()): index1, index2 = pair diff_stats = desc_stats[pair] box_write = "=" - if result: - print("Box test {0} vs {1}: {0} {2} {1}".format(index1, - index2, - result)) - box_write = result - else: - print("Box test {} vs {}: No difference".format(index1, - index2)) - print("Wilcoxon signed-rank test {} vs {}: {:.3}" - .format(index1, index2, wilcox_results[pair])) + if self.run_box_test: + result = box_results[pair] + if result: + print("Box test {0} vs {1}: {0} {2} {1}".format( + index1, + index2, + result)) + box_write = result + else: + print("Box test {} vs {}: No difference".format( + index1, + index2)) + if self.run_wilcoxon_test: + print("Wilcoxon signed-rank test {} vs {}: {:.3}" + .format(index1, index2, wilcox_results[pair])) print("Sign test {} vs {}: {:.3}" .format(index1, index2, sign_results[pair])) - print("Sign test, probability that {1} < {0}: {2:.3}" - .format(index1, index2, sign_less_results[pair])) - print("Sign test, probability that {1} > {0}: {2:.3}" - .format(index1, index2, sign_greater_results[pair])) - if sign_results[pair] > 0.05: - sign_test_relation = "=" - elif sign_less_results[pair] > sign_greater_results[pair]: - sign_test_relation = "<" - else: - sign_test_relation = ">" - print("Sign test interpretation: {} {} {}" - .format(index2, sign_test_relation, index1)) - print("Dependent t-test for paired samples {} vs {}: {:.3}" - .format(index1, index2, ttest_results[pair])) + if self.run_le_sign_test: + print("Sign test, probability that {1} < {0}: {2:.3}" + .format(index1, index2, sign_less_results[pair])) + print("Sign test, probability that {1} > {0}: {2:.3}" + .format(index1, index2, sign_greater_results[pair])) + if sign_results[pair] > 0.05: + sign_test_relation = "=" + elif sign_less_results[pair] > sign_greater_results[pair]: + sign_test_relation = "<" + else: + sign_test_relation = ">" + print("Sign test interpretation: {} {} {}" + .format(index2, sign_test_relation, index1)) + if self.run_t_test: + print("Dependent t-test for paired samples {} vs {}: {:.3}" + .format(index1, index2, ttest_results[pair])) print("{} vs {} stats: mean: {:.3}, SD: {:.3}, median: {:.3}, " "IQR: {:.3}, MAD: {:.3}".format( index1, index2, diff_stats["mean"], diff_stats["SD"], @@ -1154,39 +1211,52 @@ def _write_individual_results(self): # If either of the pairwise tests shows a small p-value with # Bonferroni correction consider it a possible side-channel - if wilcox_results[pair] < self.alpha / len(sign_results) or \ - sign_results[pair] < self.alpha / len(sign_results) or\ - ttest_results[pair] < self.alpha / len(sign_results): + if (self.run_wilcoxon_test and + wilcox_results[pair] < self.alpha / len(sign_results)) or \ + sign_results[pair] < self.alpha / len(sign_results) or\ + (self.run_t_test and + ttest_results[pair] < self.alpha / len(sign_results)): difference = 1 - wilcox_p = wilcox_results[pair] + if self.run_wilcoxon_test: + wilcox_p = wilcox_results[pair] + else: + wilcox_p = 1 sign_p = sign_results[pair] - ttest_p = ttest_results[pair] + if self.run_t_test: + ttest_p = ttest_results[pair] + else: + ttest_p = 1 row = [self.class_names[index1], - self.class_names[index2], - box_write, - wilcox_p, - sign_p, - sign_less_results[pair], - sign_greater_results[pair], - ttest_p, - diff_stats["mean"], - diff_stats["SD"], - diff_stats["median"], - diff_stats["IQR"], - diff_stats["MAD"] - ] + self.class_names[index2]] + if self.run_box_test: + row.append(box_write) + if self.run_wilcoxon_test: + row.append(wilcox_p) + row.append(sign_p) + if self.run_le_sign_test: + row.extend([sign_less_results[pair], + sign_greater_results[pair]]) + if self.run_t_test: + row.append(ttest_p) + row.extend([ + diff_stats["mean"], + diff_stats["SD"], + diff_stats["median"], + diff_stats["IQR"], + diff_stats["MAD"] + ]) writer.writerow(row) p_vals.append(wilcox_p) sign_p_vals.append(sign_p) median_difference = abs(diff_stats["median"]) - if worst_pair is None or wilcox_p < worst_p or \ + if worst_pair is None or sign_p < worst_p or \ worst_median_difference is None or \ worst_median_difference < median_difference: worst_pair = pair - worst_p = wilcox_p + worst_p = sign_p worst_median_difference = median_difference if self.verbose: From 555ab2710d636e414ec5b0adad4b97984e6d365b Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 11 Jan 2024 17:53:44 +0100 Subject: [PATCH 2/3] analysis: run sample stats analysis in separate process --- tests/test_tlsfuzzer_analysis.py | 41 +++++++++++++++++++++++++++----- tlsfuzzer/analysis.py | 4 +++- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/tests/test_tlsfuzzer_analysis.py b/tests/test_tlsfuzzer_analysis.py index db8c68bbc..56ad8f6b7 100644 --- a/tests/test_tlsfuzzer_analysis.py +++ b/tests/test_tlsfuzzer_analysis.py @@ -80,8 +80,8 @@ def test_report( #mock_box.assert_called_once() #mock_scatter.assert_called_once() # we're writing to report.csv, legend.csv, - # sample_stats.csv, and report.txt - self.assertEqual(mock_open.call_count, 4) + # and report.txt + self.assertEqual(mock_open.call_count, 3) self.assertEqual(ret, 0) @mock.patch("tlsfuzzer.analysis.Analysis._convert_to_binary") @@ -111,10 +111,39 @@ def test_report_multithreaded( #mock_box.assert_called_once() #mock_scatter.assert_called_once() # we're writing to report.csv, legend.csv, - # sample_stats.csv, and report.txt - self.assertEqual(mock_open.call_count, 4) + # report.txt + self.assertEqual(mock_open.call_count, 3) self.assertEqual(ret, 0) + @mock.patch("tlsfuzzer.analysis.Analysis._convert_to_binary") + @mock.patch("__main__.__builtins__.open", new_callable=mock.mock_open) + @mock.patch("builtins.print") + @mock.patch("tlsfuzzer.analysis.Analysis.load_data") + @mock.patch("tlsfuzzer.analysis.Analysis.graph_worst_pair") + @mock.patch("tlsfuzzer.analysis.Analysis.conf_interval_plot") + @mock.patch("tlsfuzzer.analysis.Analysis.diff_scatter_plot") + @mock.patch("tlsfuzzer.analysis.Analysis.scatter_plot") + @mock.patch("tlsfuzzer.analysis.Analysis.box_plot") + @mock.patch("tlsfuzzer.analysis.Analysis.diff_ecdf_plot") + @mock.patch("tlsfuzzer.analysis.Analysis.ecdf_plot") + def test_write_sample_stats( + self, mock_ecdf, mock_diff_ecdf, mock_box, mock_scatter, + mock_diff_scatter, mock_conf_int, mock_graph_worst_pair, + mock_load_data, mock_print, mock_open, mock_convert_to_binary, + ): + mock_load_data.return_value = self.timings + + analysis = Analysis("/tmp", verbose=True) + ret = analysis._write_sample_stats() + + mock_load_data.assert_called() + #mock_ecdf.assert_called_once() + #mock_box.assert_called_once() + #mock_scatter.assert_called_once() + # we're writing to sample_stats.csv + self.assertEqual(mock_open.call_count, 1) + self.assertIsNone(ret) + @mock.patch("builtins.print") @mock.patch("__main__.__builtins__.open", new_callable=mock.mock_open) @mock.patch("scipy.stats.friedmanchisquare") @@ -143,8 +172,8 @@ def test_report_neq( #mock_box.assert_called_once() #mock_scatter.assert_called_once() # we're writing to report.csv, legend.csv, - # sample_stats.csv, and report.txt - self.assertEqual(mock_open.call_count, 4) + # and report.txt + self.assertEqual(mock_open.call_count, 3) self.assertEqual(ret, 1) @mock.patch("tlsfuzzer.analysis.Analysis._convert_to_binary") diff --git a/tlsfuzzer/analysis.py b/tlsfuzzer/analysis.py index f889f590a..f8802ce48 100644 --- a/tlsfuzzer/analysis.py +++ b/tlsfuzzer/analysis.py @@ -1961,7 +1961,9 @@ def generate_report(self, bit_size=False, hamming_weight=False): self._write_legend() - self._write_sample_stats() + processes.append( + self._start_thread(self._write_sample_stats, + "Generation of sample statistics failed")) difference, p_vals, sign_p_vals, worst_pair = \ self._write_individual_results() From b16c6d1c38fcc84e49c2a0f689a0543fe0a1b81e Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 13 Jan 2024 01:21:48 +0100 Subject: [PATCH 3/3] analysis: disabling calculation of sample statistics --- tests/test_tlsfuzzer_analysis.py | 52 +++++++++++++++++++++----------- tlsfuzzer/analysis.py | 14 +++++++-- 2 files changed, 46 insertions(+), 20 deletions(-) diff --git a/tests/test_tlsfuzzer_analysis.py b/tests/test_tlsfuzzer_analysis.py index 56ad8f6b7..ad931c0f2 100644 --- a/tests/test_tlsfuzzer_analysis.py +++ b/tests/test_tlsfuzzer_analysis.py @@ -926,7 +926,23 @@ def test_command_line(self): output, True, True, True, False, False, None, None, None, None, None, False, True, 1e-09, 4, 'measurements.csv', False, True, True, True, True, - True, True) + True, True, True) + + def test_call_with_no_sample_stats(self): + output = "/tmp" + args = ["analysis.py", "-o", output, "--no-sample-stats"] + mock_init = mock.Mock() + mock_init.return_value = None + with mock.patch('tlsfuzzer.analysis.Analysis.generate_report') as mock_report: + with mock.patch('tlsfuzzer.analysis.Analysis.__init__', mock_init): + with mock.patch("sys.argv", args): + main() + mock_report.assert_called_once() + mock_init.assert_called_once_with( + output, True, True, True, False, False, None, None, + None, None, None, False, True, 1e-09, 4, + 'measurements.csv', False, True, True, True, True, + True, True, False) def test_minimal_analysis(self): output = "/tmp" @@ -942,7 +958,7 @@ def test_minimal_analysis(self): output, False, False, False, False, False, None, None, None, None, None, False, True, 1e-09, 4, 'measurements.csv', False, False, False, True, False, - False, False) + False, False, False) def test_call_with_no_box_test(self): output = "/tmp" @@ -958,7 +974,7 @@ def test_call_with_no_box_test(self): output, True, True, True, False, False, None, None, None, None, None, False, True, 1e-09, 4, 'measurements.csv', False, True, True, True, True, - False, True) + False, True, True) def test_call_with_no_le_sign_test(self): output = "/tmp" @@ -974,7 +990,7 @@ def test_call_with_no_le_sign_test(self): output, True, True, True, False, False, None, None, None, None, None, False, True, 1e-09, 4, 'measurements.csv', False, True, True, True, True, - True, False) + True, False, True) def test_call_with_delay_and_CR(self): output = "/tmp" @@ -991,7 +1007,7 @@ def test_call_with_delay_and_CR(self): output, True, True, True, False, False, None, None, None, 3.5, '\n', False, True, 1e-09, 4, 'measurements.csv', False, True, True, True, - True, True, True) + True, True, True, True) def test_call_with_workers(self): output = "/tmp" @@ -1007,7 +1023,7 @@ def test_call_with_workers(self): output, True, True, True, False, False, None, None, 200, None, None, False, True, 1e-09, 4, 'measurements.csv', False, True, True, True, - True, True, True) + True, True, True, True) def test_call_with_verbose(self): output = "/tmp" @@ -1023,7 +1039,7 @@ def test_call_with_verbose(self): output, True, True, True, False, True, None, None, None, None, None, False, True, 1e-09, 4, 'measurements.csv', False, True, True, True, - True, True, True) + True, True, True, True) def test_call_with_multithreaded_plots(self): output = "/tmp" @@ -1039,7 +1055,7 @@ def test_call_with_multithreaded_plots(self): output, True, True, True, True, False, None, None, None, None, None, False, True, 1e-09, 4, 'measurements.csv', False, True, True, True, - True, True, True) + True, True, True, True) def test_call_with_no_plots(self): output = "/tmp" @@ -1057,7 +1073,7 @@ def test_call_with_no_plots(self): output, False, False, False, False, False, None, None, None, None, None, False, True, 1e-09, 4, 'measurements.csv', False, True, True, True, - False, True, True) + False, True, True, True) def test_call_with_frequency(self): output = "/tmp" @@ -1073,7 +1089,7 @@ def test_call_with_frequency(self): output, True, True, True, False, False, 10*1e6, None, None, None, None, False, True, 1e-09, 4, 'measurements.csv', False, True, True, True, - True, True, True) + True, True, True, True) def test_call_with_alpha(self): output = "/tmp" @@ -1089,7 +1105,7 @@ def test_call_with_alpha(self): output, True, True, True, False, False, None, 1e-3, None, None, None, False, True, 1e-09, 4, 'measurements.csv', False, True, True, True, - True, True, True) + True, True, True, True) def test_call_with_bit_size_measurements(self): output = "/tmp" @@ -1107,7 +1123,7 @@ def test_call_with_bit_size_measurements(self): output, True, True, True, False, False, None, None, None, None, None, True, True, 1e-09, 4, 'measurements.csv', False, True, True, True, - True, True, True) + True, True, True, True) def test_call_with_skip_sanity(self): output = "/tmp" @@ -1125,7 +1141,7 @@ def test_call_with_skip_sanity(self): output, True, True, True, False, False, None, None, None, None, None, True, True, 1e-09, 4, 'measurements.csv', True, True, True, True, - True, True, True) + True, True, True, True) def test_call_with_custom_measurements_filename(self): output = "/tmp" @@ -1145,7 +1161,7 @@ def test_call_with_custom_measurements_filename(self): output, True, True, True, False, False, None, None, None, None, None, True, True, 1e-09, 4, measurements_filename, False, True, True, True, - True, True, True) + True, True, True, True) def test_call_with_no_smart_analysis(self): output = "/tmp" @@ -1164,7 +1180,7 @@ def test_call_with_no_smart_analysis(self): output, True, True, True, False, False, None, None, None, None, None, True, False, 1e-09, 4, 'measurements.csv', False, True, True, True, - True, True, True) + True, True, True, True) def test_call_with_parametrized_smart_analysis(self): output = "/tmp" @@ -1187,7 +1203,7 @@ def test_call_with_parametrized_smart_analysis(self): None, None, None, True, True, bit_size_desire_ci * 1e-9, bit_recognition_size, 'measurements.csv', False, True, True, True, - True, True, True) + True, True, True, True) def test_call_with_Hamming_weight(self): output = "/tmp" @@ -1204,7 +1220,7 @@ def test_call_with_Hamming_weight(self): output, True, True, True, False, False, None, None, None, None, None, True, True, 1e-9, 4, 'measurements.csv', False, True, True, True, - True, True, True) + True, True, True, True) mock_report.assert_called_once_with( bit_size=False, hamming_weight=True) @@ -1224,7 +1240,7 @@ def test_call_Hamming_weight_with_minimal_analysis(self): output, True, True, True, False, False, None, None, None, None, None, True, True, 1e-9, 4, 'measurements.csv', False, False, False, False, - True, True, True) + True, True, True, True) mock_report.assert_called_once_with( bit_size=False, hamming_weight=True) diff --git a/tlsfuzzer/analysis.py b/tlsfuzzer/analysis.py index f8802ce48..7ccf40b5a 100644 --- a/tlsfuzzer/analysis.py +++ b/tlsfuzzer/analysis.py @@ -64,6 +64,7 @@ def help_msg(): --no-t-test Don't run the paired sample t-test --no-sign-test [Hamming weight only] Don't run the sign test --no-le-sign-test Don't run the less-equal, greater-equal sign tests + --no-sample-stats Don't calculate sample statistics (sample_stats.csv) --minimal-analysis Run just the pairwise sign tests, Friedman test, and bootstrapping of confidence intervals (i.e. minimal amount of calculation necessary to generate report.txt) @@ -123,6 +124,7 @@ def main(): box_plot = True box_test = True le_sign_test = True + sample_stats = True verbose = False clock_freq = None alpha = None @@ -149,6 +151,7 @@ def main(): "no-box-test", "no-wilcoxon-test", "no-le-sign-test", + "no-sample-stats", "minimal-analysis", "multithreaded-graph", "clock-frequency=", @@ -189,6 +192,8 @@ def main(): wilcoxon_test = False elif opt == "--no-le-sign-test": le_sign_test = False + elif opt == "--no-sample-stats": + sample_stats = False elif opt == "--minimal-analysis": ecdf_plot = False scatter_plot = False @@ -198,6 +203,7 @@ def main(): wilcoxon_test = False t_test = False le_sign_test = False + sample_stats = False elif opt == "--multithreaded-graph": multithreaded_graph = True elif opt == "--clock-frequency": @@ -235,7 +241,7 @@ def main(): smart_analysis, bit_size_desired_ci, bit_recognition_size, measurements_filename, skip_sanity, wilcoxon_test, t_test, sign_test, - box_plot, box_test, le_sign_test) + box_plot, box_test, le_sign_test, sample_stats) ret = analysis.generate_report( bit_size=bit_size_analysis, @@ -258,7 +264,8 @@ def __init__(self, output, draw_ecdf_plot=True, draw_scatter_plot=True, bit_size_desired_ci=1e-9, bit_recognition_size=4, measurements_filename="measurements.csv", skip_sanity=False, run_wilcoxon_test=True, run_t_test=True, run_sign_test=True, - draw_box_plot=True, run_box_test=True, run_le_sign_test=True): + draw_box_plot=True, run_box_test=True, run_le_sign_test=True, + gen_sample_stats=True): self.verbose = verbose self.output = output self.clock_frequency = clock_frequency @@ -272,6 +279,7 @@ def __init__(self, output, draw_ecdf_plot=True, draw_scatter_plot=True, self.run_t_test = run_t_test self.run_sign_test = run_sign_test self.run_le_sign_test = run_le_sign_test + self.gen_sample_stats = gen_sample_stats self.multithreaded_graph = multithreaded_graph self.workers = workers if alpha is None: @@ -1276,6 +1284,8 @@ def _write_legend(self): def _write_sample_stats(self): """Write summary statistics of samples to sample_stats.csv file.""" + if not self.gen_sample_stats: + return None if self.verbose: start_time = time.time() print("[i] Writing summary statistics of samples to file")