Feature/unit tests/infra 30094/standard lib (#296)

* test_test_generator

* fix for ut internal error

* test_file_monitor_ingestor

* too long line splitted

* test_savedsearches_parser

* test optimization

* savedsearches tests in addon_parser.__init__

* mock_object optimization

* test_app_test_generator.py

* os.path.join mock fix

* removing unreachable code

* exclude addon_basic.py from coverage calculation

.coveragerc

@@ -1,11 +1,12 @@
 # .coveragerc to control coverage.py
 [run]
 omit =
-    */test_templates.py
+    pytest_splunk_addon/standard_lib/*/test_templates.py
     pytest_splunk_addon/standard_lib/cim_compliance/base_report.py
     pytest_splunk_addon/standard_lib/cim_compliance/base_table.py
     pytest_splunk_addon/standard_lib/cim_tests/base_schema.py
     pytest_splunk_addon/standard_lib/event_ingestors/base_event_ingestor.py
+    pytest_splunk_addon/standard_lib/addon_basic.py
 
 [report]
 exclude_lines =

pytest_splunk_addon/standard_lib/app_test_generator.py

@@ -120,11 +120,8 @@ def generate_tests(self, fixture):
                         test_type="line_breaker"
                         )
                 )
-            if isinstance(pytest_params, str):
-                LOGGER.warning(pytest_params)
 
-            elif pytest_params:
-                yield from sorted(pytest_params, key=lambda param: param.id)
+            yield from sorted(pytest_params, key=lambda param: param.id)
 
     def dedup_tests(self, test_list, fixture):
         """

tests/unit/tests_standard_lib/conftest.py

@@ -5,8 +5,8 @@
 
 @pytest.fixture()
 def mock_object(monkeypatch):
-    def create_mock_object(object_path):
-        mo = MagicMock()
+    def create_mock_object(object_path, **kwargs):
+        mo = MagicMock(**kwargs)
         monkeypatch.setattr(object_path, mo)
         return mo
 
@@ -20,6 +20,13 @@ def open_mock(monkeypatch):
     return open_mock
 
 
+@pytest.fixture()
+def os_path_join_file_mock(mock_object):
+    os = mock_object("os.path.join")
+    os.side_effect = lambda *x: "/".join(x)
+    return os
+
+
 @pytest.fixture()
 def json_load_mock(mock_object):
     return mock_object("json.load")

tests/unit/tests_standard_lib/test_addon_parser/test_pytest_addon_init.py

@@ -7,6 +7,7 @@
 PROPS_RETURN_VALUE = "Props_return_value"
 TAGS_RETURN_VALUE = "Tags_return_value"
 EVENTTYPE_RETURN_VALUE = "Eventtype_return_value"
+SAVEDSEARCH_RETURN_VALUE = "Savedsearch_return_value"
 TEST_VALUE = "Test_value"
 ADDON_PARSER_PATH = "pytest_splunk_addon.standard_lib.addon_parser"
 
@@ -19,11 +20,14 @@ def addonparser():
         f"{ADDON_PARSER_PATH}.tags_parser.TagsParser"
     ) as tags_mock, patch(
         f"{ADDON_PARSER_PATH}.eventtype_parser.EventTypeParser"
-    ) as eventtype_mock:
+    ) as eventtype_mock, patch(
+        f"{ADDON_PARSER_PATH}.savedsearches_parser.SavedSearchParser"
+    ) as savedsearch_mock:
         app_mock.return_value = APP_RETURN_VALUE
         props_mock.return_value = PROPS_RETURN_VALUE
         tags_mock.return_value = TAGS_RETURN_VALUE
         eventtype_mock.return_value = EVENTTYPE_RETURN_VALUE
+        savedsearch_mock.return_value = SAVEDSEARCH_RETURN_VALUE
         import pytest_splunk_addon.standard_lib.addon_parser
 
         importlib.reload(pytest_splunk_addon.standard_lib.addon_parser)
@@ -37,6 +41,7 @@ def test_addonparser_init(addonparser):
     assert ap.props_parser == PROPS_RETURN_VALUE
     assert ap.tags_parser == TAGS_RETURN_VALUE
     assert ap.eventtype_parser == EVENTTYPE_RETURN_VALUE
+    assert ap.savedsearch_parser == SAVEDSEARCH_RETURN_VALUE
 
 
 @pytest.mark.parametrize(
@@ -45,6 +50,7 @@ def test_addonparser_init(addonparser):
         ("get_tags", "tags_parser"),
         ("get_props_fields", "props_parser"),
         ("get_eventtypes", "eventtype_parser"),
+        ("get_savedsearches", "savedsearch_parser"),
     ],
 )
 def test_get_methods(addonparser, monkeypatch, function, obj_to_mock):

tests/unit/tests_standard_lib/test_addon_parser/test_savedsearches_parser.py

@@ -0,0 +1,77 @@
+import pytest
+from unittest.mock import patch, PropertyMock
+from pytest_splunk_addon.standard_lib.addon_parser.savedsearches_parser import (
+    SavedSearchParser,
+)
+
+output_to_build = {
+    "basic_search": {
+        "search": "_internal | stats count by sourcetype",
+    },
+    "search_earliest_time": {
+        "search": "index = _internal | stats count by sourcetype | outputlookup saved_search_data.csv",
+        "dispatch.earliest_time": "-4d",
+    },
+    "empty_search_latest_time": {
+        "search": "",
+        "dispatch.latest_time": "-1s",
+    },
+}
+
+
+@pytest.fixture(scope="module")
+def parsed_output(build_parsed_output):
+    return build_parsed_output(output_to_build)
+
+
+@pytest.fixture()
+def parser_instance(parsed_output, parser):
+    return parser(SavedSearchParser, "get_config", parsed_output)
+
+
+def test_savedsearches(parser_instance):
+    assert list(parser_instance.savedsearches.sects.keys()) == [
+        "basic_search",
+        "search_earliest_time",
+        "empty_search_latest_time",
+    ]
+    parser_instance.app.get_config.assert_called_once_with("savedsearches.conf")
+
+
+def test_no_savedsearches_config_file(parser_instance):
+    parser_instance.app.get_config.side_effect = OSError
+    assert parser_instance.savedsearches is None
+
+
+def test_get_savedsearches(parser_instance):
+    out = list(parser_instance.get_savedsearches())
+    assert out == [
+        {
+            "stanza": "basic_search",
+            "search": "_internal | stats count by sourcetype",
+            "dispatch.earliest_time": "0",
+            "dispatch.latest_time": "now",
+        },
+        {
+            "stanza": "search_earliest_time",
+            "search": "index = _internal | stats count by sourcetype | outputlookup saved_search_data.csv",
+            "dispatch.earliest_time": "-4d",
+            "dispatch.latest_time": "now",
+        },
+        {
+            "stanza": "empty_search_latest_time",
+            "search": 'index = "main"',
+            "dispatch.earliest_time": "0",
+            "dispatch.latest_time": "-1s",
+        },
+    ]
+
+
+def test_get_savedsearches_without_config_file(parser):
+    with patch.object(
+        SavedSearchParser, "savedsearches", new_callable=PropertyMock
+    ) as savedsearches_mock:
+        savedsearches_mock.return_value = None
+        parser_instance = parser(SavedSearchParser, "get_config", {})
+        output = [search for search in parser_instance.get_savedsearches() if search]
+        assert output == [], "savedsearches returned when no config file exists"

tests/unit/tests_standard_lib/test_addon_parser/test_tags_parser.py

@@ -56,8 +56,7 @@ def test_tags_can_be_parsed_and_returned(parser_instance):
 def test_get_tags_calls_app_get_config(parser_instance):
     for _ in parser_instance.get_tags():
         pass
-    parser_instance.app.get_config.assert_called_once()
-    parser_instance.app.get_config.assert_called_with("tags.conf")
+    parser_instance.app.get_config.assert_called_once_with("tags.conf")
 
 
 def test_no_tags_config_file(parser_instance):

tests/unit/tests_standard_lib/test_app_test_generator.py

@@ -0,0 +1,188 @@
+import pytest
+from unittest.mock import patch
+from collections import namedtuple
+from pytest_splunk_addon.standard_lib.app_test_generator import AppTestGenerator
+
+module = "pytest_splunk_addon.standard_lib.app_test_generator"
+config = {
+    "splunk_app": "fake_app",
+    "field_bank": "fake_field_bank",
+    "splunk_dm_path": "fake_path",
+    "store_events": True,
+    "splunk_data_generator": "psa.conf",
+}
+pytest_config = namedtuple("Config", ["getoption"])
+test_config = pytest_config(getoption=lambda x, *y: config[x])
+test_config_without_dm_path = pytest_config(
+    getoption=lambda x, *y: config[x] if x != "splunk_dm_path" else None
+)
+params = namedtuple("ParameterSet", ["values", "id"])
+
+
+@pytest.fixture()
+def app_test_generator(mock_object):
+    fieldtest_generator = mock_object(f"{module}.FieldTestGenerator")
+    cim_test_generator = mock_object(f"{module}.CIMTestGenerator")
+    indextime_test_generator = mock_object(f"{module}.IndexTimeTestGenerator")
+    requirement_test_generator = mock_object(f"{module}.ReqsTestGenerator")
+    for mock_element in [
+        fieldtest_generator,
+        cim_test_generator,
+        indextime_test_generator,
+        requirement_test_generator,
+    ]:
+        setattr(mock_element, "return_value", mock_element)
+
+
+@pytest.mark.parametrize(
+    "simple_config, path",
+    [
+        (test_config, "fake_path"),
+        (test_config_without_dm_path, "/fake_dir/data_models"),
+    ],
+)
+def test_app_test_generator_instantiation(
+    mock_object, os_path_join_file_mock, app_test_generator, simple_config, path
+):
+    os_path_dirname_mock = mock_object("os.path.dirname")
+    os_path_dirname_mock.return_value = "/fake_dir"
+    atg = AppTestGenerator(simple_config)
+    atg.fieldtest_generator.assert_called_once_with(
+        config["splunk_app"], field_bank=config["field_bank"]
+    )
+    atg.cim_test_generator.assert_called_once_with(config["splunk_app"], path)
+    atg.requirement_test_generator.assert_called_once_with(config["splunk_app"])
+    atg.indextime_test_generator.assert_called_once_with()
+
+
+@pytest.mark.parametrize(
+    "fixture, called_function, test_generator, generator_args, generator_kwargs, expected_tests, dedup_call_count",
+    [
+        (
+            "splunk_searchtime_fields",
+            "fieldtest_generator",
+            lambda fixture: (f"{fixture}_test_{i + 1}" for i in range(3)),
+            ["splunk_searchtime_fields"],
+            {},
+            [
+                "splunk_searchtime_fields_test_1",
+                "splunk_searchtime_fields_test_2",
+                "splunk_searchtime_fields_test_3",
+            ],
+            1,
+        ),
+        (
+            "splunk_searchtime_cim",
+            "cim_test_generator",
+            lambda fixture: (f"{fixture}_test_{i + 1}" for i in range(3)),
+            ["splunk_searchtime_cim"],
+            {},
+            [
+                "splunk_searchtime_cim_test_1",
+                "splunk_searchtime_cim_test_2",
+                "splunk_searchtime_cim_test_3",
+            ],
+            1,
+        ),
+        (
+            "splunk_searchtime_requirement",
+            "requirement_test_generator",
+            lambda fixture: (f"{fixture}_test_{i + 1}" for i in range(3)),
+            ["splunk_searchtime_requirement"],
+            {},
+            [
+                "splunk_searchtime_requirement_test_1",
+                "splunk_searchtime_requirement_test_2",
+                "splunk_searchtime_requirement_test_3",
+            ],
+            1,
+        ),
+        (
+            "splunk_indextime_key_fields",
+            "indextime_test_generator",
+            lambda x, app_path, config_path, test_type: (
+                params(values=f"splunk_indextime_{test_type}_test_{3 - i}", id=3 - i)
+                for i in range(3)
+            ),
+            [True],
+            {
+                "app_path": "fake_app",
+                "config_path": "psa.conf",
+                "test_type": "key_fields",
+            },
+            [
+                params(values=f"splunk_indextime_key_fields_test_1", id=1),
+                params(values=f"splunk_indextime_key_fields_test_2", id=2),
+                params(values=f"splunk_indextime_key_fields_test_3", id=3),
+            ],
+            0,
+        ),
+        (
+            "splunk_indextime_time",
+            "indextime_test_generator",
+            lambda x, app_path, config_path, test_type: (
+                params(values=f"splunk_indextime_{test_type}_test_{3 - i}", id=3 - i)
+                for i in range(3)
+            ),
+            [True],
+            {"app_path": "fake_app", "config_path": "psa.conf", "test_type": "_time"},
+            [
+                params(values=f"splunk_indextime__time_test_1", id=1),
+                params(values=f"splunk_indextime__time_test_2", id=2),
+                params(values=f"splunk_indextime__time_test_3", id=3),
+            ],
+            0,
+        ),
+        (
+            "splunk_indextime_line_breaker",
+            "indextime_test_generator",
+            lambda x, app_path, config_path, test_type: (
+                params(values=f"splunk_indextime_{test_type}_test_{3 - i}", id=3 - i)
+                for i in range(3)
+            ),
+            [True],
+            {
+                "app_path": "fake_app",
+                "config_path": "psa.conf",
+                "test_type": "line_breaker",
+            },
+            [
+                params(values=f"splunk_indextime_line_breaker_test_1", id=1),
+                params(values=f"splunk_indextime_line_breaker_test_2", id=2),
+                params(values=f"splunk_indextime_line_breaker_test_3", id=3),
+            ],
+            0,
+        ),
+    ],
+)
+def test_generate_tests(
+    app_test_generator,
+    fixture,
+    called_function,
+    test_generator,
+    generator_args,
+    generator_kwargs,
+    expected_tests,
+    dedup_call_count,
+):
+    atg = AppTestGenerator(test_config)
+    setattr(getattr(atg, called_function).generate_tests, "side_effect", test_generator)
+    with patch.object(
+        AppTestGenerator, "dedup_tests", side_effect=lambda x, y: x
+    ) as dedup_mock:
+        out = list(atg.generate_tests(fixture))
+        assert out == expected_tests
+        getattr(atg, called_function).generate_tests.assert_called_once_with(
+            *generator_args, **generator_kwargs
+        )
+        assert dedup_mock.call_count == dedup_call_count
+
+
+def test_dedup_tests(app_test_generator):
+    parameter_list = [params(values=f"val{x}", id=x) for x in range(7)]
+    atg = AppTestGenerator(test_config)
+    out = []
+    for parameters in [parameter_list[:3], parameter_list[2:5]]:
+        out.extend(atg.dedup_tests(parameters, "splunk_searchtime"))
+    assert out == parameter_list[:5]
+    assert atg.seen_tests == {("splunk_searchtime", x) for x in range(5)}