From ab078380fd460b760ad40569ff6861356f8fccf6 Mon Sep 17 00:00:00 2001 From: augusdn Date: Fri, 27 Sep 2024 13:57:50 +1000 Subject: [PATCH] Add custom message for pre/post sanity check results (#14755) Expand custom message feature to find potentially risky tests and modify existing custom message to be expandable for future use. Description of PR Expand custom message feature to find potentially risky tests and modify existing custom message to be expandable for future use. Approach What is the motivation for this PR? To enhance debugging by adding extra information into customMsg we already have, and also make it expandable so others can add debugging information to be utlised in ADE co-authorized by: jianquanye@microsoft.com --- tests/common/plugins/sanity_check/__init__.py | 63 ++++++++++++++++++- tests/conftest.py | 9 +-- 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/tests/common/plugins/sanity_check/__init__.py b/tests/common/plugins/sanity_check/__init__.py index 24c6634c41..2d77106769 100644 --- a/tests/common/plugins/sanity_check/__init__.py +++ b/tests/common/plugins/sanity_check/__init__.py @@ -17,6 +17,15 @@ logger = logging.getLogger(__name__) SUPPORTED_CHECKS = checks.CHECK_ITEMS +DUT_CHEK_LIST = ['core_dump_check_pass', 'config_db_check_pass'] +CACHE_LIST = ['core_dump_check_pass', 'config_db_check_pass', + 'pre_sanity_check_failed', 'post_sanity_check_failed', + 'pre_sanity_recovered', 'post_sanity_recovered'] + + +def reset_cache_list(request): + for item in CACHE_LIST: + request.config.cache.set(item, None) def pytest_sessionfinish(session, exitstatus): @@ -28,6 +37,8 @@ def pytest_sessionfinish(session, exitstatus): session.config.cache.set("pre_sanity_check_failed", None) if post_sanity_failed: session.config.cache.set("post_sanity_check_failed", None) + for key in CACHE_LIST: + session.config.cache.set(key, None) if pre_sanity_failed and not post_sanity_failed: session.exitstatus = constants.PRE_SANITY_CHECK_FAILED_RC @@ -119,6 +130,49 @@ def do_checks(request, check_items, *args, **kwargs): return check_results +@pytest.fixture(scope="module", autouse=True) +def log_custom_msg(request): + yield + module_name = request.node.name + items = request.session.items + for item in items: + if item.module.__name__ + ".py" == module_name.split("/")[-1]: + customMsgDict = {} + dutChekResults = {} + for key in DUT_CHEK_LIST: + if request.config.cache.get(key, None) is False: + dutChekResults[key] = False + if dutChekResults: + customMsgDict['DutChekResult'] = dutChekResults + + # Check pre_sanity_checks results + preSanityCheckResults = {} + if request.config.cache.get("pre_sanity_check_failed", None): + preSanityCheckResults['pre_sanity_check_failed'] = True + # pre_sanity_recovered should be None in healthy case, record either True/False + if request.config.cache.get("pre_sanity_recovered", None) is not None: + preSanityCheckResults['pre_sanity_recovered'] = request.config.cache.get("pre_sanity_recovered", None) + if preSanityCheckResults: + customMsgDict['PreSanityCheckResults'] = preSanityCheckResults + + # Check post_sanity_checks results + postSanityCheckResults = {} + if request.config.cache.get("post_sanity_check_failed", None): + postSanityCheckResults['post_sanity_check_failed'] = True + # post_sanity_recovered should be None in healthy case, record either True/False + if request.config.cache.get("post_sanity_recovered", None) is not None: + preSanityCheckResults['post_sanity_recovered'] = request.config.cache.get("post_sanity_recovered", None) + if postSanityCheckResults: + customMsgDict['PostSanityCheckResults'] = postSanityCheckResults + + # if we have any custom message to log, append it to user_properties + if customMsgDict: + logger.debug("customMsgDict: {}".format(customMsgDict)) + item.user_properties.append(('CustomMsg', json.dumps(customMsgDict))) + + reset_cache_list(request) + + @pytest.fixture(scope="module") def sanity_check_full(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo): logger.info("Prepare sanity check") @@ -269,8 +323,10 @@ def sanity_check_full(localhost, duthosts, request, fanouthosts, nbrhosts, tbinf def recover_on_sanity_check_failure(duthosts, failed_results, fanouthosts, localhost, nbrhosts, check_items, recover_method, request, tbinfo, sanity_check_stage: str): cache_key = "pre_sanity_check_failed" + recovery_cache_key = "pre_sanity_recovered" if sanity_check_stage == STAGE_POST_TEST: cache_key = "post_sanity_check_failed" + recovery_cache_key = "post_sanity_recovered" try: dut_failed_results = defaultdict(list) @@ -299,6 +355,7 @@ def recover_on_sanity_check_failure(duthosts, failed_results, fanouthosts, local except BaseException as e: request.config.cache.set(cache_key, True) + request.config.cache.set(recovery_cache_key, False) logger.error(f"Recovery of sanity check failed with exception: {repr(e)}") pt_assert( @@ -313,13 +370,17 @@ def recover_on_sanity_check_failure(duthosts, failed_results, fanouthosts, local new_failed_results = [result for result in new_check_results if result['failed']] if new_failed_results: request.config.cache.set(cache_key, True) + request.config.cache.set(recovery_cache_key, False) pt_assert(False, f"!!!!!!!!!!!!!!!! {sanity_check_stage} sanity check after recovery failed: !!!!!!!!!!!!!!!!\n" f"{json.dumps(new_failed_results, indent=4, default=fallback_serializer)}") + # Record recovery success + request.config.cache.set(recovery_cache_key, True) +# make sure teardown of log_custom_msg happens after sanity_check @pytest.fixture(scope="module", autouse=True) -def sanity_check(request): +def sanity_check(request, log_custom_msg): if request.config.option.skip_sanity: logger.info("Skip sanity check according to command line argument") yield diff --git a/tests/conftest.py b/tests/conftest.py index e983a49589..815e0b5878 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2343,13 +2343,8 @@ def _remove_entry(table_name, key_name, config): logger.info("Core dump and config check passed for {}".format(module_name)) if check_result: - items = request.session.items - for item in items: - if item.module.__name__ + ".py" == module_name.split("/")[-1]: - item.user_properties.append(('CustomMsg', json.dumps({'DutChekResult': { - 'core_dump_check_pass': core_dump_check_pass, - 'config_db_check_pass': config_db_check_pass - }}))) + request.config.cache.set("core_dump_check_pass", core_dump_check_pass) + request.config.cache.set("config_db_check_pass", config_db_check_pass) @pytest.fixture(scope="function")