diff --git a/automation/script/module.py b/automation/script/module.py index 94de87aa9..46181a087 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -1375,6 +1375,7 @@ def _run(self, i): r = update_env_with_values(env) if r['return']>0: return r + # Clean some output files clean_tmp_files(clean_files, recursion_spaces) @@ -1529,6 +1530,7 @@ def _run(self, i): import json logging.debug(json.dumps(env, indent=2, sort_keys=True)) + # Check chain of pre hook dependencies on other CM scripts if len(prehook_deps)>0: logging.debug(recursion_spaces + ' - Checking prehook dependencies on other CM scripts:') @@ -3009,17 +3011,13 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a if not run_state['fake_deps']: import copy - tmp_run_state_deps = copy.deepcopy(run_state['deps']) - run_state['deps'] = [] - tmp_parent = run_state['parent'] + run_state_copy = copy.deepcopy(run_state) + run_state_copy['deps'] = [] - run_state['parent'] = run_state['script_id'] + run_state_copy['parent'] = run_state['script_id'] if len(run_state['script_variation_tags']) > 0: - run_state['parent'] += " ( " + ',_'.join(run_state['script_variation_tags']) + " )" - - tmp_script_id = run_state['script_id'] - tmp_script_variation_tags = run_state['script_variation_tags'] + run_state_copy['parent'] += " ( " + ',_'.join(run_state['script_variation_tags']) + " )" # Run collective script via CM API: # Not very efficient but allows logging - can be optimized later @@ -3039,7 +3037,7 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a 'verbose':verbose, 'silent':run_state.get('tmp_silent', False), 'time':show_time, - 'run_state':run_state + 'run_state':run_state_copy } @@ -3053,15 +3051,12 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a r = self.cmind.access(ii) if r['return']>0: return r - run_state['deps'] = tmp_run_state_deps - run_state['parent'] = tmp_parent - run_state['script_id'] = tmp_script_id - run_state['script_variation_tags'] = tmp_script_variation_tags + run_state['version_info'] = run_state_copy.get('version_info') # Restore local env env.update(tmp_env) r = update_env_with_values(env) - if r['return']>0: return r + if r['return']>0: return r return {'return': 0} diff --git a/script/app-mlperf-inference-nvidia/customize.py b/script/app-mlperf-inference-nvidia/customize.py index 97f3db0f2..4304d6525 100644 --- a/script/app-mlperf-inference-nvidia/customize.py +++ b/script/app-mlperf-inference-nvidia/customize.py @@ -91,7 +91,10 @@ def preprocess(i): if not os.path.exists(target_data_path_base_dir): cmds.append(f"mkdir -p {target_data_path_base_dir}") - if not os.path.exists(target_data_path): + inference_cases_json_path = os.path.join(env['MLPERF_SCRATCH_PATH'], 'data', 'KiTS19', 'inference_cases.json') + calibration_cases_json_path = os.path.join(env['MLPERF_SCRATCH_PATH'], 'data', 'KiTS19', 'calibration_cases.json') + + if not os.path.exists(target_data_path) or not os.path.exists(inference_cases_json_path) or not os.path.exists(calibration_cases_json_path): #cmds.append(f"ln -sf {env['CM_DATASET_PATH']} {target_data_path}") cmds.append("make download_data BENCHMARKS='3d-unet'") diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index ff7378b93..66808dc10 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -16,6 +16,10 @@ "new_env_keys": [ "+PATH" ], + "env": { + "CM_GENERIC_SYS_UTIL_INSTALL_NEEDED": "no", + "CM_SYS_UTIL_VERSION_CMD": "" + }, "tags": [ "get", "sys-util", @@ -24,6 +28,40 @@ ], "uid": "bb0393afa8404a11", "variations": { + "detect": { + "group": "mode", + "default": true, + "names": [ + "detect-sys-util" + ], + "env": { + "CM_GENERIC_SYS_UTIL_RUN_MODE": "detect" + }, + "prehook_deps": [ + { + "tags": "get,generic-sys-util,_install", + "inherit_variation_tags": true, + "names": [ + "install-sys-util" + ], + "skip_inherit_variation_groups": [ + "mode" + ], + "enable_if_env": { + "CM_GENERIC_SYS_UTIL_INSTALL_NEEDED": + [ + "yes" + ] + } + } + ] + }, + "install": { + "group": "mode", + "env": { + "CM_GENERIC_SYS_UTIL_RUN_MODE": "install" + } + }, "cmake": { "env": { "CM_SYS_UTIL_NAME": "cmake" @@ -602,13 +640,19 @@ }, "wget": { "env": { - "CM_SYS_UTIL_NAME": "wget" + "CM_SYS_UTIL_NAME": "wget", + "CM_SYS_UTIL_VERSION_CMD": "wget --version", + "CM_SYS_UTIL_VERSION_RE": "Wget\\s*([\\d.]+)" }, "state": { "wget": { - "apt": "wget" + "apt": "wget", + "brew": "wget" } - } + }, + "new_env_keys": [ + "CM_WGET_VERSION" + ] }, "wkhtmltopdf": { "env": { diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 0c065909b..648f13915 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -8,6 +8,26 @@ def preprocess(i): env = i['env'] state = i['state'] + automation = i['automation'] + + #Use VERSION_CMD and CHECK_CMD if no CHECK_CMD is set + if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and env.get('CM_SYS_UTIL_CHECK_CMD', '') == '': + env['CM_SYS_UTIL_CHECK_CMD'] = env['CM_SYS_UTIL_VERSION_CMD'] + + if env.get('CM_GENERIC_SYS_UTIL_RUN_MODE', '') == "detect": + if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '': + r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) + if r['return'] > 0: #detection failed, do install via prehook_deps + env['CM_GENERIC_SYS_UTIL_INSTALL_NEEDED'] = "yes" + return {'return': 0} + else: #detection is successful, no need to install + env['CM_SYS_UTIL_INSTALL_CMD'] = "" + return {'return': 0} + else: #No detction command available, just install + env['CM_GENERIC_SYS_UTIL_INSTALL_NEEDED'] = "yes" + return {'return': 0} + + # Only "install" mode reaches here pm = env.get('CM_HOST_OS_PACKAGE_MANAGER') if os_info['platform'] == 'windows': @@ -73,3 +93,51 @@ def preprocess(i): env['CM_SYS_UTIL_INSTALL_CMD'] = f"""{env['CM_SYS_UTIL_CHECK_CMD']} || {env['CM_SYS_UTIL_INSTALL_CMD']}""" return {'return':0} + + +def detect_version(i): + env = i['env'] + version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION" + version_check_re = env.get('CM_SYS_UTIL_VERSION_RE', '') + + if version_check_re == '' or not os.path.exists("tmp-ver.out"): + version = "undetected" + + else: + r = i['automation'].parse_version({'match_text': version_check_re, + 'group_number': 1, + 'env_key': version_env_key, + 'which_env': env}) + if r['return'] >0: return r + + version = r['version'] + + print (i['recursion_spaces'] + ' Detected version: {}'.format(version)) + return {'return':0, 'version':version} + +def postprocess(i): + env = i['env'] + + version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION" + + if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') : + automation = i['automation'] + r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) + if r['return'] > 0: + return r + + r = detect_version(i) + + if r['return'] >0: return r + + version = r['version'] + + env[version_env_key] = version + + #Not used now + env['CM_GENERIC_SYS_UTIL_'+env['CM_SYS_UTIL_NAME'].upper()+'_CACHE_TAGS'] = 'version-'+version + + if env.get(version_env_key, '') == '': + env[version_env_key] = "undetected" + + return {'return':0, 'version': env[version_env_key]} diff --git a/script/get-generic-sys-util/detect.sh b/script/get-generic-sys-util/detect.sh new file mode 100644 index 000000000..6f2e3759b --- /dev/null +++ b/script/get-generic-sys-util/detect.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +if [[ -n "${CM_SYS_UTIL_VERSION_CMD}" ]]; then + ${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out +fi + +test $? -eq 0 || exit $?