From 7899f1789bfd129666336ad7bd6ee43fecf43596 Mon Sep 17 00:00:00 2001 From: Augustine Lee Date: Thu, 26 Sep 2024 05:05:57 +0000 Subject: [PATCH] Add custom message for pre/post sanity check results Expand custom message feature to find potentially risky tests and modify existing custom message to be expandable for future use. --- tests/common/plugins/sanity_check/__init__.py | 63 ++++++++++++++++++- tests/conftest.py | 9 +-- 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/tests/common/plugins/sanity_check/__init__.py b/tests/common/plugins/sanity_check/__init__.py index 24c6634c41..2d77106769 100644 --- a/tests/common/plugins/sanity_check/__init__.py +++ b/tests/common/plugins/sanity_check/__init__.py @@ -17,6 +17,15 @@ logger = logging.getLogger(__name__) SUPPORTED_CHECKS = checks.CHECK_ITEMS +DUT_CHEK_LIST = ['core_dump_check_pass', 'config_db_check_pass'] +CACHE_LIST = ['core_dump_check_pass', 'config_db_check_pass', + 'pre_sanity_check_failed', 'post_sanity_check_failed', + 'pre_sanity_recovered', 'post_sanity_recovered'] + + +def reset_cache_list(request): + for item in CACHE_LIST: + request.config.cache.set(item, None) def pytest_sessionfinish(session, exitstatus): @@ -28,6 +37,8 @@ def pytest_sessionfinish(session, exitstatus): session.config.cache.set("pre_sanity_check_failed", None) if post_sanity_failed: session.config.cache.set("post_sanity_check_failed", None) + for key in CACHE_LIST: + session.config.cache.set(key, None) if pre_sanity_failed and not post_sanity_failed: session.exitstatus = constants.PRE_SANITY_CHECK_FAILED_RC @@ -119,6 +130,49 @@ def do_checks(request, check_items, *args, **kwargs): return check_results +@pytest.fixture(scope="module", autouse=True) +def log_custom_msg(request): + yield + module_name = request.node.name + items = request.session.items + for item in items: + if item.module.__name__ + ".py" == module_name.split("/")[-1]: + customMsgDict = {} + dutChekResults = {} + for key in DUT_CHEK_LIST: + if request.config.cache.get(key, None) is False: + dutChekResults[key] = False + if dutChekResults: + customMsgDict['DutChekResult'] = dutChekResults + + # Check pre_sanity_checks results + preSanityCheckResults = {} + if request.config.cache.get("pre_sanity_check_failed", None): + preSanityCheckResults['pre_sanity_check_failed'] = True + # pre_sanity_recovered should be None in healthy case, record either True/False + if request.config.cache.get("pre_sanity_recovered", None) is not None: + preSanityCheckResults['pre_sanity_recovered'] = request.config.cache.get("pre_sanity_recovered", None) + if preSanityCheckResults: + customMsgDict['PreSanityCheckResults'] = preSanityCheckResults + + # Check post_sanity_checks results + postSanityCheckResults = {} + if request.config.cache.get("post_sanity_check_failed", None): + postSanityCheckResults['post_sanity_check_failed'] = True + # post_sanity_recovered should be None in healthy case, record either True/False + if request.config.cache.get("post_sanity_recovered", None) is not None: + preSanityCheckResults['post_sanity_recovered'] = request.config.cache.get("post_sanity_recovered", None) + if postSanityCheckResults: + customMsgDict['PostSanityCheckResults'] = postSanityCheckResults + + # if we have any custom message to log, append it to user_properties + if customMsgDict: + logger.debug("customMsgDict: {}".format(customMsgDict)) + item.user_properties.append(('CustomMsg', json.dumps(customMsgDict))) + + reset_cache_list(request) + + @pytest.fixture(scope="module") def sanity_check_full(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo): logger.info("Prepare sanity check") @@ -269,8 +323,10 @@ def sanity_check_full(localhost, duthosts, request, fanouthosts, nbrhosts, tbinf def recover_on_sanity_check_failure(duthosts, failed_results, fanouthosts, localhost, nbrhosts, check_items, recover_method, request, tbinfo, sanity_check_stage: str): cache_key = "pre_sanity_check_failed" + recovery_cache_key = "pre_sanity_recovered" if sanity_check_stage == STAGE_POST_TEST: cache_key = "post_sanity_check_failed" + recovery_cache_key = "post_sanity_recovered" try: dut_failed_results = defaultdict(list) @@ -299,6 +355,7 @@ def recover_on_sanity_check_failure(duthosts, failed_results, fanouthosts, local except BaseException as e: request.config.cache.set(cache_key, True) + request.config.cache.set(recovery_cache_key, False) logger.error(f"Recovery of sanity check failed with exception: {repr(e)}") pt_assert( @@ -313,13 +370,17 @@ def recover_on_sanity_check_failure(duthosts, failed_results, fanouthosts, local new_failed_results = [result for result in new_check_results if result['failed']] if new_failed_results: request.config.cache.set(cache_key, True) + request.config.cache.set(recovery_cache_key, False) pt_assert(False, f"!!!!!!!!!!!!!!!! {sanity_check_stage} sanity check after recovery failed: !!!!!!!!!!!!!!!!\n" f"{json.dumps(new_failed_results, indent=4, default=fallback_serializer)}") + # Record recovery success + request.config.cache.set(recovery_cache_key, True) +# make sure teardown of log_custom_msg happens after sanity_check @pytest.fixture(scope="module", autouse=True) -def sanity_check(request): +def sanity_check(request, log_custom_msg): if request.config.option.skip_sanity: logger.info("Skip sanity check according to command line argument") yield diff --git a/tests/conftest.py b/tests/conftest.py index 43115f37ec..b1362c402c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2364,13 +2364,8 @@ def _remove_entry(table_name, key_name, config): logger.info("Core dump and config check passed for {}".format(module_name)) if check_result: - items = request.session.items - for item in items: - if item.module.__name__ + ".py" == module_name.split("/")[-1]: - item.user_properties.append(('CustomMsg', json.dumps({'DutChekResult': { - 'core_dump_check_pass': core_dump_check_pass, - 'config_db_check_pass': config_db_check_pass - }}))) + request.config.cache.set("core_dump_check_pass", core_dump_check_pass) + request.config.cache.set("config_db_check_pass", config_db_check_pass) @pytest.fixture(scope="function")