diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/llm_bench.json b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/llm_bench.json new file mode 100644 index 00000000000000..5ad25c48ec2ad9 --- /dev/null +++ b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/llm_bench.json @@ -0,0 +1,72 @@ +{ + "appCmd":"source {venvName}/bin/activate && cd {appPath} && python3.10 -m pip install --upgrade pip && python3.10 -m pip install openvino=={wheelVersion} openvino_genai=={wheelVersion} openvino_tokenizers=={wheelVersion} --find-links={precommitPath}wheels/ && python3.10 -m pip install -r requirements.txt && {cmd}", + "appPath" : "{appPath}", + "venvCfg":{ + "venvEnabled":true, + "venvDir":"{workPath}/venv/", + "venvName":"tempVenv" + }, + "commandList":[ + + ], + "runConfig":{ + "mode":"llmBench", + "traversal":"firstFailedVersion", + "perfAppropriateDeviation" : 0.05, + "commitList" : { + "getCommitListCmd" : "git log {c1}..{c2} --boundary --pretty=\"%h\"" + } + }, + "dlbConfig":{ + "launchedAsJob":false, + "toolName":"", + "wheelVersionsMap":{ + + }, + "commonPath":"{commitPath}", + "subPath":"{subPath}", + "appPath":"", + "appCmd":"" + }, + "cachedPathConfig":{ + "enabled":true, + "scheme":"mandatory", + "passCmdList":false, + "changeAppPath":false, + "commonPath":"{commitPath}", + "subPath":"{subPath}", + "cashMap":{ + + } + }, + "substitutionRules":[ + { + "name":"precommitPath", + "enabled":true, + "type":"map", + "placeholder":"precommitPath", + "from":"$.cachedPathConfig.cashMap", + "to":"$.appCmd" + }, + { + "name":"wheelVersion", + "enabled":true, + "type":"map", + "placeholder":"wheelVersion", + "from":"$.dlbConfig.wheelVersionsMap", + "to":"$.appCmd" + } + ], + "subscriptions":[ + { + "name":"wheelPathsMap", + "enabled":true + }, + { + "name":"wheelVersionsMap", + "enabled":true + } + ], + "verboseOutput" : true, + "checkIfBordersDiffer" : true + } \ No newline at end of file diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/modes.py b/src/plugins/intel_cpu/tools/commit_slider/utils/modes.py index 4162f5cd271709..6a5b35137d8d4b 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/utils/modes.py +++ b/src/plugins/intel_cpu/tools/commit_slider/utils/modes.py @@ -341,32 +341,6 @@ def specifyMetric(self, metric: str = "First token latency"): def preliminaryCheck(self, list, cfg): # # model path checking - todo is necessary ? - # if cfg["preliminaryCheckCfg"]["checkBenchmarkModelPath"]: - # cmdStr = cfg["appCmd"] - # matcher = re.search( - # "benchmark.*-m[\s*]([^\S]*)", - # cmdStr, - # flags=re.MULTILINE - # ) - # if matcher is not None: - # # pass if app is not openvino benchmark_app - # try: - # modelPath = extractModelPath(cmdStr) - # if not os.path.isfile(modelPath): - # raise PreliminaryAnalysisError( - # "path {modelPath} does not exist, check config".format( - # modelPath=modelPath - # ), - # PreliminaryAnalysisError.PreliminaryErrType.WRONG_COMMANDLINE - # ) - # except (IndexError, ValueError): - # raise PreliminaryAnalysisError( - # "commandline '{cmdStr}' is not correct, check config".format( - # cmdStr=cmdStr - # ), - # PreliminaryAnalysisError.PreliminaryErrType.WRONG_COMMANDLINE - # ) - # common if-degradation-exists check super().preliminaryCheck(list, cfg)