From 6834c9883efde28f227bc28eca854959322cda7e Mon Sep 17 00:00:00 2001 From: "Allen D. Householder" Date: Mon, 25 Sep 2023 16:17:25 -0400 Subject: [PATCH] roll in unit tests --- test/cvd_states/test_cvss_patterns.py | 64 ++++ test/cvd_states/test_embargo.py | 48 +++ test/cvd_states/test_hypercube.py | 91 ++++- test/cvd_states/test_info.py | 30 ++ test/cvd_states/test_pattern_utils.py | 57 +++ test/cvd_states/test_potential_actions.py | 34 ++ test/cvd_states/test_ssvc_patterns.py | 80 +++++ test/cvd_states/test_stategraph.py | 402 ++++++++++++++++++++++ test/cvd_states/test_validations.py | 217 ++++++++++++ test/cvd_states/test_vep_patterns.py | 30 ++ test/cvd_states/test_zeroday_patterns.py | 78 +++++ 11 files changed, 1130 insertions(+), 1 deletion(-) create mode 100644 test/cvd_states/test_cvss_patterns.py create mode 100644 test/cvd_states/test_embargo.py create mode 100644 test/cvd_states/test_info.py create mode 100644 test/cvd_states/test_pattern_utils.py create mode 100644 test/cvd_states/test_potential_actions.py create mode 100644 test/cvd_states/test_ssvc_patterns.py create mode 100644 test/cvd_states/test_stategraph.py create mode 100644 test/cvd_states/test_validations.py create mode 100644 test/cvd_states/test_vep_patterns.py create mode 100644 test/cvd_states/test_zeroday_patterns.py diff --git a/test/cvd_states/test_cvss_patterns.py b/test/cvd_states/test_cvss_patterns.py new file mode 100644 index 00000000..97e5447d --- /dev/null +++ b/test/cvd_states/test_cvss_patterns.py @@ -0,0 +1,64 @@ +# Copyright (c) 2023. Carnegie Mellon University +# +# See LICENSE for details + +import unittest +from enum import Enum + +import vultron.cvd_states.patterns.cvss31 as cvss +from vultron.cvd_states.hypercube import CVDmodel + + +class MyTestCase(unittest.TestCase): + def setUp(self): + self.model = CVDmodel() + + def tearDown(self): + pass + + def test_cvss_31_e(self): + for state in self.model.states: + result = cvss.cvss_31_e(state) + # result should always be a list of non-zero length of strings of non-zero length + self.assertIsInstance(result, list) + for item in result: + self.assertIsInstance(item, Enum) + + def test_cvss_31_rl(self): + for state in self.model.states: + result = cvss.cvss_31_rl(state) + # result should always be a list of non-zero length of strings of non-zero length + self.assertIsInstance(result, list) + for item in result: + self.assertIsInstance(item, Enum) + + def test_cvss_31(self): + for state in self.model.states: + result = cvss.cvss_31(state) + # result should always be a list of non-zero length of strings of non-zero length + self.assertIsInstance(result, list) + for item in result: + self.assertIsInstance(item, Enum) + + def test_cvss_exploitation_state(self): + for state in self.model.states: + result = cvss.cvss_31_e(state) + # if A in state, then Exploitation: Active should be in result + if "A" in state: + self.assertIn(cvss.CVSS_31_E.HIGH, result) + self.assertIn(cvss.CVSS_31_E.FUNCTIONAL, result) + elif "X" in state: + self.assertIn(cvss.CVSS_31_E.HIGH, result) + self.assertIn(cvss.CVSS_31_E.FUNCTIONAL, result) + self.assertIn(cvss.CVSS_31_E.PROOF_OF_CONCEPT, result) + else: + self.assertIn(cvss.CVSS_31_E.UNPROVEN, result) + self.assertIn(cvss.CVSS_31_E.NOT_DEFINED, result) + + self.assertNotIn(cvss.CVSS_31_E.PROOF_OF_CONCEPT, result) + self.assertNotIn(cvss.CVSS_31_E.FUNCTIONAL, result) + self.assertNotIn(cvss.CVSS_31_E.HIGH, result) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/cvd_states/test_embargo.py b/test/cvd_states/test_embargo.py new file mode 100644 index 00000000..7214c9e8 --- /dev/null +++ b/test/cvd_states/test_embargo.py @@ -0,0 +1,48 @@ +# Copyright (c) 2023. Carnegie Mellon University +# +# See LICENSE for details + +import unittest + +import vultron.cvd_states.hypercube as sg +import vultron.cvd_states.patterns.embargo as mb +from vultron.cvd_states.errors import StateValidationError + + +class MyTestCase(unittest.TestCase): + def setUp(self): + self.model = sg.CVDmodel() + + def tearDown(self): + pass + + def test_reject_bogus_states(self): + funcs = [mb.can_start_embargo, mb.embargo_viable] + bogus_states = ["abbso", "abs", "absd", "absdx", "absdxa", "absdpxa"] + for func in funcs: + for state in bogus_states: + self.assertRaises(StateValidationError, func, state) + + def test_embargo_viable(self): + m = self.model + + # walk all the states and check the embargo_viable function + for state in m.states: + if "pxa" in state: + self.assertTrue(mb.embargo_viable(state)) + else: + self.assertFalse(mb.embargo_viable(state)) + + def test_can_start_embargo(self): + m = self.model + + # walk all the states and check the can_start_embargo function + for state in m.states: + if "dpxa" in state: + self.assertTrue(mb.can_start_embargo(state)) + else: + self.assertFalse(mb.can_start_embargo(state)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/cvd_states/test_hypercube.py b/test/cvd_states/test_hypercube.py index f1afd71a..24966661 100644 --- a/test/cvd_states/test_hypercube.py +++ b/test/cvd_states/test_hypercube.py @@ -3,11 +3,100 @@ import vultron.cvd_states.hypercube as hc import vultron.cvd_states.states +from vultron.cvd_states.errors import TransitionValidationError from vultron.cvd_states.states import CS_pxa +from vultron.cvd_states.validations import is_valid_transition class MyTestCase(unittest.TestCase): - pass + def test_proto_states(self): + ps = hc._proto_states() + self.assertEqual(len(ps), 6) + for x in "vfdpxa": + s = f"{x.lower()}{x.upper()}" + self.assertIn(s, ps) + + def test_create_states(self): + states = hc._create_states() + self.assertEqual(len(states), 32) + count = 0 + for p, x, a in product("pP", "xX", "aA"): + s = "".join([p, x, a]) + for allowed in ["vfd", "Vfd", "VFd", "VFD"]: + a = f"{allowed}{s}" + self.assertIn(a, states) + count += 1 + for disallowed in ["vFd", "vfD", "vFD", "VfD"]: + d = f"{disallowed}{s}" + self.assertNotIn(d, states) + self.assertEqual(count, len(states)) + + def test_create_graph(self): + G = hc._create_graph() + states = hc._create_states() + self.assertEqual(len(G.nodes), len(states)) + for state in hc._create_states(): + self.assertIn(state, G.nodes) + + def test_diffstate(self): + states = hc._create_states() + for s1, s2 in product(states, states): + try: + is_valid_transition(s1, s2) + except TransitionValidationError: + continue + + # only one character should be different + self.assertNotEqual(s1, s2) + diff = [x for x in zip(s1, s2) if x[0] != x[1]] + self.assertEqual(len(diff), 1) + + # and it should be a lower to upper case transition + self.assertTrue(diff[0][0].islower()) + self.assertTrue(diff[0][1].isupper()) + self.assertEqual(diff[0][0].upper(), diff[0][1]) + + def test_model_states(self): + states = hc._create_states() + m = hc.CVDmodel() + self.assertEqual(len(m.states), len(states)) + for state in states: + self.assertIn(state, m.states) + + def test_model_graph(self): + m = hc.CVDmodel() + self.assertEqual(len(m.G.nodes), len(m.states)) + for state in m.states: + self.assertIn(state, m.G.nodes) + + def test_model_previous_state(self): + m = hc.CVDmodel() + for state in m.states: + previous = m.previous_state(state) + for pred in m.G.predecessors(state): + self.assertIn(pred, previous) + + def test_model_next_state(self): + m = hc.CVDmodel() + for state in m.states: + next = m.next_state(state) + for succ in m.G.successors(state): + self.assertIn(succ, next) + + def test_paths_between(self): + m = hc.CVDmodel() + paths = list(m.paths_between("vfdpxa", "VFDPXA")) + self.assertEqual(len(paths), 70) + + def test_paths_from(self): + m = hc.CVDmodel() + paths = list(m.paths_from("vfdpxa")) + self.assertEqual(len(paths), 70) + + def test_paths_to(self): + m = hc.CVDmodel() + paths = list(m.paths_to("VFDPXA")) + self.assertEqual(len(paths), 70) if __name__ == "__main__": diff --git a/test/cvd_states/test_info.py b/test/cvd_states/test_info.py new file mode 100644 index 00000000..d2e4a04b --- /dev/null +++ b/test/cvd_states/test_info.py @@ -0,0 +1,30 @@ +# Copyright (c) 2023. Carnegie Mellon University +# +# See LICENSE for details + +import unittest +from enum import Enum + +import vultron.cvd_states.patterns.info as info +from vultron.cvd_states.hypercube import CVDmodel + + +class MyTestCase(unittest.TestCase): + def setUp(self): + self.model = CVDmodel() + + def tearDown(self): + pass + + def test_info(self): + for state in self.model.states: + result = info.info(state) + # result should always be a list of non-zero length of strings of non-zero length + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + for item in result: + self.assertIsInstance(item, Enum) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/cvd_states/test_pattern_utils.py b/test/cvd_states/test_pattern_utils.py new file mode 100644 index 00000000..26b09716 --- /dev/null +++ b/test/cvd_states/test_pattern_utils.py @@ -0,0 +1,57 @@ +# Copyright (c) 2023. Carnegie Mellon University +# +# See LICENSE for details + +import unittest +from enum import IntEnum + +from vultron.cvd_states.enums.utils import ( + enum_item_in_list, + enum_list_to_string_list, + uniq_enum_iter, + unique_enum_list, +) + + +class Foo(IntEnum): + No = 0 + Yes = 1 + + +class Bar(IntEnum): + No = 0 + Yes = 1 + + +class MyTestCase(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + pass + + def test_enum_list_to_string_list(self): + list_of_items = [Foo.No, Bar.Yes] + results = enum_list_to_string_list(list_of_items) + self.assertEqual(results, ["Foo.No", "Bar.Yes"]) + + def test_enum_item_in_list(self): + list_of_items = [Foo.No, Bar.Yes] + for test_item in [Foo.No, Bar.Yes]: + self.assertTrue(enum_item_in_list(test_item, list_of_items)) + for test_item in [Foo.Yes, Bar.No]: + self.assertFalse(enum_item_in_list(test_item, list_of_items)) + + def test_uniq_enum_iter(self): + list_of_items = [Foo.No, Bar.Yes, Foo.No, Bar.Yes] + results = list(uniq_enum_iter(list_of_items)) + self.assertEqual(results, [Foo.No, Bar.Yes]) + + def test_unique_enum_list(self): + list_of_items = [Foo.No, Bar.Yes, Foo.No, Bar.Yes] + results = unique_enum_list(list_of_items) + self.assertEqual(results, [Foo.No, Bar.Yes]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/cvd_states/test_potential_actions.py b/test/cvd_states/test_potential_actions.py new file mode 100644 index 00000000..05b69424 --- /dev/null +++ b/test/cvd_states/test_potential_actions.py @@ -0,0 +1,34 @@ +# Copyright (c) 2023. Carnegie Mellon University +# +# See LICENSE for details + +import unittest + +from vultron.cvd_states.hypercube import CVDmodel +from vultron.cvd_states.patterns import potential_actions as pa + + +class MyTestCase(unittest.TestCase): + def setUp(self): + self.model = CVDmodel() + pass + + def tearDown(self): + pass + + def test_pa_action(self): + for state in self.model.states: + result = pa.action(state) + # actions should be a list + self.assertIsInstance(result, list) + # actions should be non-empty + self.assertGreater(len(result), 0) + # actions should be a list of enums + for a in result: + self.assertIsInstance(a, pa.Actions) + # actions should be unique + self.assertEqual(len(result), len(set(result))) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/cvd_states/test_ssvc_patterns.py b/test/cvd_states/test_ssvc_patterns.py new file mode 100644 index 00000000..d3be0e22 --- /dev/null +++ b/test/cvd_states/test_ssvc_patterns.py @@ -0,0 +1,80 @@ +# Copyright (c) 2023. Carnegie Mellon University +# +# See LICENSE for details + +import unittest + +from vultron.cvd_states.enums.ssvc_2 import ( + SSVC_2_Enum, + SSVC_2_Exploitation, + SSVC_2_Report_Public, + SSVC_2_Supplier_Contacted, +) +from vultron.cvd_states.enums.utils import enum_item_in_list +from vultron.cvd_states.hypercube import CVDmodel +from vultron.cvd_states.patterns.ssvc import ( + ssvc, +) + + +class MyTestCase(unittest.TestCase): + def setUp(self): + self.model = CVDmodel() + + def tearDown(self): + pass + + def test_ssvc(self): + for state in self.model.states: + result = ssvc(state) + # result should always be a list of non-zero length of ssvc enums + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + for item in result: + self.assertIsInstance(item, SSVC_2_Enum) + + def test_ssvc_exploitation_state(self): + for state in self.model.states: + result = ssvc(state) + # if A in state, then Exploitation: Active should be in result + if "A" in state: + self.assertTrue(enum_item_in_list(SSVC_2_Exploitation.ACTIVE, result)) + elif "X" in state: + self.assertTrue(enum_item_in_list(SSVC_2_Exploitation.POC, result)) + else: + self.assertTrue(enum_item_in_list(SSVC_2_Exploitation.NONE, result)) + self.assertFalse(enum_item_in_list(SSVC_2_Exploitation.POC, result)) + self.assertFalse(enum_item_in_list(SSVC_2_Exploitation.ACTIVE, result)) + + def test_ssvc_report_public_state(self): + for state in self.model.states: + # if P in state, then Report: Public should be in result + result = ssvc(state) + + if "P" in state: + self.assertTrue(enum_item_in_list(SSVC_2_Report_Public.YES, result)) + self.assertFalse(enum_item_in_list(SSVC_2_Report_Public.NO, result)) + else: + self.assertTrue(enum_item_in_list(SSVC_2_Report_Public.NO, result)) + self.assertFalse(enum_item_in_list(SSVC_2_Report_Public.YES, result)) + + def test_ssvc_vendor_aware_state(self): + for state in self.model.states: + # if V in state, then Supplier Contacted: Yes should be in result + result = ssvc(state) + if "V" in state: + self.assertTrue( + enum_item_in_list(SSVC_2_Supplier_Contacted.YES, result) + ) + self.assertFalse( + enum_item_in_list(SSVC_2_Supplier_Contacted.NO, result) + ) + else: + self.assertTrue(enum_item_in_list(SSVC_2_Supplier_Contacted.NO, result)) + self.assertFalse( + enum_item_in_list(SSVC_2_Supplier_Contacted.YES, result) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/cvd_states/test_stategraph.py b/test/cvd_states/test_stategraph.py new file mode 100644 index 00000000..28fc37f8 --- /dev/null +++ b/test/cvd_states/test_stategraph.py @@ -0,0 +1,402 @@ +# Copyright (c) 2023. Carnegie Mellon University +# +# See LICENSE for details + +import unittest + +import networkx as nx + +import vultron.cvd_states.errors as err +import vultron.cvd_states.hypercube as sg +import vultron.cvd_states.validations as val + + +class TestStategraph(unittest.TestCase): + def setUp(self): + self.model = sg.CVDmodel() + + def tearDown(self): + pass + + def test__create_graph(self): + G = sg._create_graph() + + # G should have all the states as nodes + for n in sg._create_states(): + self.assertTrue(n in G) + + # the valid graph should have 58 edges + self.assertEqual(58, len(G.edges)) + + # G should have valid edges with labels + for a, b, t in G.edges(data="label"): + self.assertIsNone(val.is_valid_transition(a, b)) + self.assertEqual(sg._diffstate(a, b), t) + self.assertTrue(t in "VFDPXA") + + self.assertTrue(nx.is_directed_acyclic_graph(G)) + + n_paths = len(list(nx.all_simple_paths(G, "vfdpxa", "VFDPXA"))) + self.assertEqual(70, n_paths) + + def test__create_states(self): + states = sg._create_states() + # enforce causality rules + for x in states: + self.assertNotIn("vF", x, x) + self.assertNotIn("fD", x, x) + + # should be 32 states + self.assertEqual(32, len(states)) + + def test_H(self): + m = self.model + + H = m.H + self.assertEqual(70, len(H)) + for h in H: + self.assertIsNone(val.is_valid_history(h), h) + + def test_paths_from(self): + m = self.model + + H = m.sequences_from("VFDpxa") + self.assertEqual(5, len(H), H) + + H = m.sequences_from("Vfdpxa") + self.assertEqual(42, len(H), H) + + H = m.sequences_from("vfdPxa") + self.assertEqual(12, len(H)) + + H = m.sequences_from("vfdpXa") + self.assertEqual(3, len(H), H) + + H = m.sequences_from("vfdpxA") + self.assertEqual(13, len(H), H) + + def test_paths_to(self): + m = self.model + + H = m.sequences_to("VFDPXa") + self.assertEqual(13, len(H), H) + + H = m.sequences_to("VFDPxA") + self.assertEqual(19, len(H), H) + + H = m.sequences_to("VFDpXA") + self.assertEqual(4, len(H)) + + H = m.sequences_to("VFdPXA") + self.assertEqual(34, len(H), H) + + H = m.sequences_to("VfdPXA") + self.assertEqual(12, len(H), H) + + H = m.sequences_to("vfdPXA") + self.assertEqual(1, len(H), H) + + def test_walk_from(self): + m = self.model + start = "vfdpxa" + + path, probabilities = m.walk_from(start) + self.assertGreaterEqual(len(path), 0) + self.assertLessEqual(len(path), 6) + self.assertEqual(len(path), len(probabilities)) + + # check each hop probability is 0 < p <= 1 + for _p in probabilities: + self.assertGreater(_p, 0) + self.assertLessEqual(_p, 1) + + import math + + # check combined path probability is 0 < p <= 1 + p = math.prod(probabilities) + self.assertGreater(p, 0) + self.assertLessEqual(p, 1) + + def test_compute_h_probabilities(self): + m = self.model + H_prob = m.H_prob + + # make sure the keys match up + self.assertEqual(len(m.H), len(H_prob)) + self.assertEqual(set(m.H), set(H_prob.keys())) + + # make sure the probabilities are sane + for v in H_prob.values(): + self.assertGreater(v, 0) + self.assertLessEqual(v, 1) + + def test_compute_tfidf(self): + m = self.model + df = m._compute_tfidf() + + self.assertIn("tfidf", df.columns) + self.assertIn("rank", df.columns) + tfidf = df["tfidf"] + self.assertGreaterEqual(tfidf.min(), 0, tfidf.min()) + self.assertLessEqual(tfidf.max(), 20, tfidf.max()) + + def test_compute_s_scores(self): + m = self.model + + score = m._compute_s_scores() + for k, v in score.items(): + self.assertIn(k, m.states) + self.assertGreaterEqual(v, -10) + self.assertLessEqual(v, 10) + + def test_init_df(self): + m = self.model + df = m._init_H_df() + + for c in ["h", "p", "D0 except first + total = df.sum() + for row in total[1:]: + self.assertGreater(row, 0) + self.assertEqual(total.iloc[0], 0) + + # every node has outdegree >0 except last + total = df.sum(axis=1) + for row in total[:-1]: + self.assertGreater(row, 0) + self.assertEqual(total.iloc[-1], 0) + + def test_state_transition(self): + m = self.model + df = m.state_transition_matrix() + + total = df.sum(axis=1) + # all rows add to 1 except the last + for row in total[:-1]: + self.assertEqual(row, 1) + self.assertEqual(total.iloc[-1], 0) + + def test_init_sdf(self): + m = self.model + df = m._init_sdf() + + self.assertEqual(len(df), len(m.states)) + for s in m.states: + self.assertIn(s, df.index) + + self.assertIn("rank", df.columns) + self.assertIn("pagerank", df.columns) + self.assertFalse(any(df["rank"].isnull())) + self.assertFalse(any(df["pagerank"].isnull())) + + def test_find_states(self): + m = self.model + pat = ".f.PxA" + matches = m.find_states(pat) + self.assertEqual(len(matches), 2) + self.assertIn("vfdPxA", matches) + self.assertIn("VfdPxA", matches) + + self.assertRaises(err.CVDmodelError, m.find_states, "abcdefg") + + def test_next_state(self): + m = self.model + + from itertools import product + + for s1, s2 in product(m.states, m.states): + t = sg._diffstate(s1, s2) + # diffstate returns none on invalid transitions + if t is not None: + self.assertEqual(m.next_state(s1, t), s2, (s1, s2, t)) + + for state in m.states: + if state == "VFDPXA": + continue + self.assertGreater(len(m.next_state(state)), 0) + + def test_compute_pagerank(self): + m = self.model + + pr = m.compute_pagerank() + + # expect VFDPXA to be max element of pr + import operator + + max_key = max(pr.items(), key=operator.itemgetter(1))[0] + self.assertEqual("VFDPXA", max_key) + + # self.assertFalse(True,m.compute_pagerank()) + + def test_proto_states(self): + ps = sg._proto_states() + + for e in sg.EVENTS: + x = f"{e.lower()}{e.upper()}" + self.assertIn(x, ps) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/cvd_states/test_validations.py b/test/cvd_states/test_validations.py new file mode 100644 index 00000000..2fc7735b --- /dev/null +++ b/test/cvd_states/test_validations.py @@ -0,0 +1,217 @@ +# Copyright (c) 2023. Carnegie Mellon University +# +# See LICENSE for details + +import random +import string +import unittest +from itertools import permutations, product + +import vultron.cvd_states.errors as err +import vultron.cvd_states.validations as v +from vultron.cvd_states.hypercube import CVDmodel + +alpha = string.ascii_lowercase + +ok_pfx = [ + "vfd", + "Vfd", + "VFd", + "VFD", +] + +bogus_pfx = [ + "vFd", + "vfD", + "vFD", + "VfD", +] + +ok_states = ["".join(parts) for parts in product(ok_pfx, "pP", "xX", "aA")] +# note these aren't all possible bad states, just the ones closest to the ok states +bad_states = ["".join(parts) for parts in product(bogus_pfx, "pP", "xX", "aA")] + + +class MyTestCase(unittest.TestCase): + def setUp(self): + self.sg = CVDmodel() + + def tearDown(self): + pass + + def test_ok_states(self): + self.assertEqual(32, len(ok_states)) + for state in ok_states: + self.assertIn(state, self.sg.states) + + def test_bad_states(self): + for state in bad_states: + self.assertNotIn(state, self.sg.states) + + self.assertEqual(32, len(bad_states)) + + def test_is_valid_pattern_bad_length(self): + """ + Fuzz the is_valid_pattern function with strings of various lengths + :return: + """ + # wrong length + for length in range(1, 10): + if length == 6: + continue + for i in range(100): + test_str = "".join((random.choice(alpha) for _ in range(length))) + self.assertEqual(length, len(test_str)) + with self.assertRaises(err.PatternValidationError): + v.is_valid_pattern(test_str) + + def test_is_valid_pattern_bad_chars(self): + """ + Fuzz the is_valid_pattern function with strings of invalid chars + :return: + """ + # wrong chars + for i in range(1000): + ch = [a for a in alpha if a not in "vfdpxa"] + test_str = "".join((random.choice(ch) for _ in range(6))) + self.assertEqual(6, len(test_str)) + with self.assertRaises(err.PatternValidationError): + v.is_valid_pattern(test_str) + + def test_is_valid_pattern_bad_order(self): + # exhaustive test + # chars out of place + for letters in permutations("vfdpxa"): + test_str = "".join(letters) + if test_str == "vfdpxa": + continue + with self.assertRaises(err.PatternValidationError): + v.is_valid_pattern(test_str) + + def test_is_valid_pattern_ok(self): + # exhaustive test + # ok patterns + # dots or random case are ok + for parts in product("vV.", "fF.", "dD.", "pP.", "xX.", "aA."): + test_str = "".join(parts) + self.assertIsNone(v.is_valid_pattern(test_str)) + + def test_is_valid_state_bad_length(self): + """ + Fuzz the is_valid_state function with strings of various lengths + :return: + """ + # wrong length + for length in range(1, 10): + if length == 6: + continue + for i in range(100): + test_str = "".join((random.choice(alpha) for _ in range(length))) + self.assertEqual(length, len(test_str)) + with self.assertRaises(err.StateValidationError): + v.is_valid_state(test_str) + + def test_is_valid_state_bad_chars(self): + """ + Fuzz the is_valid_state function with strings of invalid chars + :return: + """ + # wrong chars + for i in range(1000): + ch = [a for a in alpha if a not in "vfdpxa"] + test_str = "".join((random.choice(ch) for _ in range(6))) + self.assertEqual(6, len(test_str)) + with self.assertRaises(err.StateValidationError): + v.is_valid_state(test_str) + + def test_is_valid_state_bad_pattern(self): + for parts in product(bogus_pfx, "pP", "xX", "aA"): + state = "".join(parts) + with self.assertRaises(err.StateValidationError): + v.is_valid_state(state) + + def test_is_valid_state_ok(self): + for parts in product(ok_pfx, "pP", "xX", "aA"): + state = "".join(parts) + self.assertIsNone(v.is_valid_state(state)) + + def test_is_valid_transition_PV_causality(self): + # validation semantics + # none = ok + # raises error = not ok + + # v -> V + self.assertIsNone(v.is_valid_transition("vfdpxa", "Vfdpxa")) + + # vp -> vP + self.assertIsNone(v.is_valid_transition("vfdpxa", "vfdPxa")) + + # vP -> VP + self.assertIsNone(v.is_valid_transition("vfdPxa", "VfdPxa")) + + # vPx -> vPX + with self.assertRaises(err.TransitionValidationError): + v.is_valid_transition("vfdPxa", "vfdPXa") + + # vPa -> vPA + with self.assertRaises(err.TransitionValidationError): + v.is_valid_transition("vfdPxa", "vfdPxA") + + def test_is_valid_transition_XP_causality(self): + # pX -> PX + for pfx in ok_pfx: + self.assertIsNone(v.is_valid_transition(f"{pfx}pXa", f"{pfx}PXa")) + # pXa -> pXA + with self.assertRaises(err.TransitionValidationError): + v.is_valid_transition(f"{pfx}pXa", f"{pfx}pXA") + + def test_is_valid_transition_bad_states(self): + for a, b in product(ok_states, bad_states): + with self.assertRaises(err.TransitionValidationError): + v.is_valid_transition(a, b) + + with self.assertRaises(err.TransitionValidationError): + v.is_valid_transition(b, a) + + def test_is_valid_transition_ok_states(self): + # all single char changes from valid state to valid state are ok + for a, b in product(ok_states, ok_states): + diff = [] + for c1, c2 in zip(a, b): + if c1 != c2: + diff.append((c1, c2)) + if len(diff) == 1: + x, y = diff[0] + if not x.islower(): + with self.assertRaises(err.TransitionValidationError): + v.is_valid_transition(a, b) + + if not y.isupper(): + with self.assertRaises(err.TransitionValidationError): + v.is_valid_transition(a, b) + + elif len(diff) == 0: + self.assertIsNone(v.is_valid_transition(a, b, allow_null=True)) + + with self.assertRaises(err.TransitionValidationError): + v.is_valid_transition(a, b, allow_null=False) + else: + with self.assertRaises(err.TransitionValidationError): + v.is_valid_transition(a, b) + + def test_is_valid_history_bad(self): + invalid = ["vfdpxa", "FVDPXA", "AXVDFP", "aaa", "aaaaaaa", "VFFDPX", "XAPVFD"] + + for h in invalid: + with self.assertRaises(err.HistoryValidationError): + v.is_valid_history(h) + + def test_is_valid_history_ok(self): + valid = self.sg.histories + self.assertEqual(70, len(valid)) + for h in valid: + self.assertIsNone(v.is_valid_history(h)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/cvd_states/test_vep_patterns.py b/test/cvd_states/test_vep_patterns.py new file mode 100644 index 00000000..4c589d29 --- /dev/null +++ b/test/cvd_states/test_vep_patterns.py @@ -0,0 +1,30 @@ +# Copyright (c) 2023. Carnegie Mellon University +# +# See LICENSE for details + +import unittest + +import vultron.cvd_states.enums.vep +from vultron.cvd_states.hypercube import CVDmodel +from vultron.cvd_states.patterns import vep + + +class MyTestCase(unittest.TestCase): + def setUp(self): + self.model = CVDmodel() + + def tearDown(self): + pass + + def test_vep(self): + for state in self.model.states: + result = vep.vep(state) + # result should always be a list of non-zero length of strings of non-zero length + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + for item in result: + self.assertIsInstance(item, vultron.cvd_states.enums.vep.VEP) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/cvd_states/test_zeroday_patterns.py b/test/cvd_states/test_zeroday_patterns.py new file mode 100644 index 00000000..3dc16217 --- /dev/null +++ b/test/cvd_states/test_zeroday_patterns.py @@ -0,0 +1,78 @@ +# Copyright (c) 2023. Carnegie Mellon University +# +# See LICENSE for details + +import unittest + +import vultron.cvd_states.patterns.zerodays as zd +from vultron.cvd_states.hypercube import CVDmodel + +not_zero_day_states = [ + "Vfdpxa", + "VFdpxa", + "VFdPxa", + "VFdPxA", + "VFdPXa", + "VFdPXA", + "VFDpxa", + "VFDPxa", + "VFDPxA", + "VFDPXa", + "VFDPXA", +] + +not_zero_day_histories = [ + "VFPAXD", + "VFPADX", + "VFPXAD", + "VFPXDA", + "VFPDAX", + "VFPDXA", + "VFDPAX", + "VFDPXA", +] + + +class MyTestCase(unittest.TestCase): + def setUp(self): + self.sg = CVDmodel() + + def tearDown(self): + pass + + def test_zeroday_type(self): + for state in self.sg.states: + result = zd.zeroday_type(state) + # should be a list + self.assertIsInstance(result, list) + if state in not_zero_day_states: + # should be n + self.assertEqual(len(result), 1) + self.assertIn(zd.ZeroDayType.NOT_APPLICABLE, result) + else: + # should be non-empty + self.assertGreater(len(result), 0) + # should be a list of enums + for r in result: + self.assertIsInstance(r, zd.ZeroDayType) + # should be unique + self.assertEqual(len(result), len(set(result))) + + def test_type_from_history(self): + for history in self.sg.histories: + result = zd.type_from_history(history) + if history in not_zero_day_histories: + # should be empty + self.assertEqual(len(result), 0) + else: + # should be non-empty + self.assertGreaterEqual(len(result), 1) + # should be a list of enums + for r in result: + self.assertIsInstance(r, zd.ZeroDayType) + # should be unique + self.assertEqual(len(result), len(set(result))) + + +if __name__ == "__main__": + unittest.main()