From 3b03e37fcc4c59e98ce4378278704f3296664d6e Mon Sep 17 00:00:00 2001 From: Matthew Brady Date: Mon, 22 Apr 2024 17:58:12 +0100 Subject: [PATCH] v1.0.6 - further changes --- detect_advisor/global_values.py | 5 +++-- detect_advisor/messages.py | 35 +++++++++++++++++++++++---------- detect_advisor/output.py | 20 +++++++++---------- detect_advisor/process.py | 8 +++++--- pyproject.toml | 2 +- 5 files changed, 44 insertions(+), 26 deletions(-) diff --git a/detect_advisor/global_values.py b/detect_advisor/global_values.py index c91396a..09c2a05 100644 --- a/detect_advisor/global_values.py +++ b/detect_advisor/global_values.py @@ -716,5 +716,6 @@ 'emit-db': 3, } -exclude_dirs = [ -] \ No newline at end of file +exclude_dirs = [] + +message_list = [] \ No newline at end of file diff --git a/detect_advisor/messages.py b/detect_advisor/messages.py index 049ff93..550a2cf 100644 --- a/detect_advisor/messages.py +++ b/detect_advisor/messages.py @@ -86,8 +86,10 @@ 'impact': 'Local license text may need to be scanned', 'action': 'Add options --detect.blackduck.signature.scanner.license.search=true and optionally --detect.blackduck.signature.scanner.upload.source.mode=true', 'cli': 'lic', - 'cli_search': 'detect.blackduck.signature.scanner.upload.source.mode', - 'cli_text': '--detect.blackduck.signature.scanner.upload.source.mode=true (CAUTION - will upload local source files)', + 'cli_search': 'detect.blackduck.signature.scanner.license.search', + 'cli_text': '--detect.blackduck.signature.scanner.license.search=true\n' + '--detect.blackduck.signature.scanner.upload.source.mode=true (CAUTION - will upload local source files)', + }, 'FILES3': { @@ -100,6 +102,16 @@ 'cli_text': '--detect.blackduck.signature.scanner.snippet.matching=SNIPPET_MATCHING (To search for copied OSS source files and functions within source files)', }, + 'FILES4': { + 'level': 'imp', + 'desc': 'Folders included in scan which should probably be excluded', + 'impact': 'Scan size may be larger than required and additional components identified', + 'action': 'Rerun with --full option to see list of folders to exclude, then use the --detect.excluded.directories option to exclude folders', + 'cli': 'scan', + 'cli_search': 'detect.excluded.directories', + 'cli_text': '--detect.excluded.directories=XXX with --detect.excluded.directories.search.depth=X (search for and exclude folders from scan - see list of folders for exclusion using --full option)', + }, + 'SCAN1': { 'level': 'crit', 'desc': 'Overall scan size {:>,d} MB) is too large (default max scan size 5GB)', @@ -139,7 +151,7 @@ 'level': 'imp', 'desc': 'Large amount of data ({:>,d} MB) in {} binary files found', 'impact': 'Binary files not analysed by standard scan, will increase scan size and impact Capacity license usage', - 'action': 'Remove files or ignore folders (using .bdignore files), also consider using Binary scan', + 'action': 'See list of binary files using --full option, then remove files or ignore folders (using .bdignore files), also consider using Binary scan', 'cli': 'scan', 'cli_search': 'detect.binary.scan.file.name.patterns', 'cli_text': '--detect.binary.scan.file.name.patterns=exe,bin,dll (for example) and --detect.binary.scan.search.depth=X (folder depth to search for binaries)' @@ -166,14 +178,14 @@ 'level': 'crit', 'desc': 'Required package manager programs ({}) missing for dependency scan in invocation folder', 'impact': 'Scan will fail', - 'action': 'Install required package manager programs', + 'action': 'See list of affected PMs using --full option, then install required package manager programs', }, 'PACKAGES4': { 'level': 'imp', 'desc': 'Required package manager programs ({}) missing for dependency scan in sub-folders', 'impact': 'The scan will fail if the scan depth is modified from the default level 0', - 'action': 'Install required package manager programs', + 'action': 'See list of affected PMs using --full option, , then install required package manager programs', }, 'PACKAGES5': { @@ -186,8 +198,8 @@ 'PACKAGES6': { 'level': 'crit', 'desc': 'Missing lockfiles/PMs for package manager files in invocation folder', - 'impact': 'Dependency scan will fail unless lockfiles created, PMs installed or --detect.accuracy.required=NONE specified', - 'action': 'Either install required package manager programs, create lockfiles or specify --detect.accuracy.required=NONE (reduced accuracy scan)', + 'impact': 'Dependency scan will fail', + 'action': 'See list of affected PMs using --full option, then either install required package manager programs, create lockfiles or specify --detect.accuracy.required=NONE (reduced accuracy scan)', 'cli': 'reqd', 'cli_search': 'detect.accuracy.required', 'cli_text': '--detect.accuracy.required=NONE (OR specify --detect.XXXX.path= where XXX is package manager OR install package managers OR create lockfiles)', @@ -196,8 +208,8 @@ 'PACKAGES7': { 'level': 'imp', 'desc': 'Missing lockfiles/PMs will cause scan to fail for package manager files in sub-folders', - 'impact': 'Dependency scan will fail if scan depth > 0 unless lockfiles created, PMs installed or --detect.accuracy.required=NONE specified', - 'action': 'Either install required package manager programs, create lockfiles or specify --detect.accuracy.required=NONE (reduced accuracy scan)', + 'impact': 'Dependency scan will fail if scan depth > 0', + 'action': 'See list of affected PMs using --full option, then either install required package manager programs, create lockfiles or specify --detect.accuracy.required=NONE (reduced accuracy scan)', 'cli': 'reqd', 'cli_search': 'detect.accuracy.required', 'cli_text': '--detect.accuracy.required=NONE (OR specify --detect.XXXX.path= where XXX is package manager OR install package managers)', @@ -233,7 +245,7 @@ # 'cli_search': 'detect.accuracy.required', # 'cli_text': '--detect.accuracy.required=NONE (OR install JS packages)', # }, - # + 'PACKAGES12': { 'level': 'imp', 'desc': 'Lockfile(s) required for dependency scan missing in invocation folder', @@ -260,7 +272,10 @@ def message(id, val1='', val2=''): + if id in global_values.message_list: + return if id in messages_dict: + global_values.message_list.append(id) if val2 != '': mtext = f"- {levtexts[messages_dict[id]['level']]}: " + messages_dict[id]['desc'].format(val1, val2) + '\n' elif val1 != '': diff --git a/detect_advisor/output.py b/detect_advisor/output.py index 9bb072f..0d8558b 100644 --- a/detect_advisor/output.py +++ b/detect_advisor/output.py @@ -193,10 +193,10 @@ def output_cli(critical_only, reportfile): if global_values.recs_msgs_dict['crit']: output += "Note that scan will probably fail - see CRITICAL recommendations above\n\n" - output += " DETECT COMMAND:\n" - output += re.sub(r"^", " ", global_values.cli_msgs_dict['detect'], flags=re.MULTILINE) - output += "\n MINIMUM REQUIRED OPTIONS:\n" - output += re.sub(r"^", " ", global_values.cli_msgs_dict['reqd'], flags=re.MULTILINE) + output += "DETECT COMMAND:\n" + output += global_values.cli_msgs_dict['detect'] + output += "\nMINIMUM REQUIRED OPTIONS:\n" + output += global_values.cli_msgs_dict['reqd'] # if len(bdignore_list) > 0: # if report: @@ -208,8 +208,11 @@ def output_cli(critical_only, reportfile): if not critical_only: output += '\n' + if global_values.cli_msgs_dict['proj'] != '': + output += "\nPROJECT OPTIONS:\n" + global_values.cli_msgs_dict['proj'] + "\n" + if global_values.cli_msgs_dict['scan'] != '': - output += "\nOPTIONS TO IMPROVE SCAN COVERAGE:\n" + global_values.cli_msgs_dict['scan'] + "\n" + output += "\nOPTIONS TO IMPROVE SCAN COVERAGE/ACCURACY:\n" + global_values.cli_msgs_dict['scan'] + "\n" if global_values.cli_msgs_dict['size'] != '': output += "\nOPTIONS TO REDUCE SIGNATURE SCAN SIZE:\n" + global_values.cli_msgs_dict['size'] + "\n" @@ -220,9 +223,6 @@ def output_cli(critical_only, reportfile): if global_values.cli_msgs_dict['lic'] != '': output += "\nOPTIONS TO IMPROVE LICENSE COMPLIANCE ANALYSIS:\n" + global_values.cli_msgs_dict['lic'] + "\n" - if global_values.cli_msgs_dict['proj'] != '': - output += "\nPROJECT OPTIONS:\n" + global_values.cli_msgs_dict['proj'] + "\n" - if global_values.cli_msgs_dict['rep'] != '': output += "\nREPORTING OPTIONS:\n" + global_values.cli_msgs_dict['rep'] + "\n" @@ -241,11 +241,11 @@ def output_config(projdir): "# Uncomment and update required options\n#\n#\n" + \ "# DETECT COMMAND TO RUN:\n#\n" + global_values.cli_msgs_dict['detect'] + "\n" + \ "# MINIMUM REQUIRED OPTIONS:\n#\n" + global_values.cli_msgs_dict['reqd'] + "\n" + \ - "# OPTIONS TO IMPROVE SCAN COVERAGE:\n#\n" + global_values.cli_msgs_dict['scan'] + "\n" + \ + "# PROJECT OPTIONS:\n#\n" + global_values.cli_msgs_dict['proj'] + "\n" + \ + "# OPTIONS TO IMPROVE SCAN COVERAGE/ACCURACY:\n#\n" + global_values.cli_msgs_dict['scan'] + "\n" + \ "# OPTIONS TO REDUCE SIGNATURE SCAN SIZE:\n#\n" + global_values.cli_msgs_dict['size'] + "\n" + \ "# OPTIONS TO CONFIGURE DEPENDENCY SCAN:\n#\n" + global_values.cli_msgs_dict['dep'] + "\n" + \ "# OPTIONS TO IMPROVE LICENSE COMPLIANCE ANALYSIS:\n#\n" + global_values.cli_msgs_dict['lic'] + "\n" + \ - "# PROJECT OPTIONS:\n#\n" + global_values.cli_msgs_dict['proj'] + "\n" + \ "# REPORTING OPTIONS:\n#\n" + global_values.cli_msgs_dict['rep'] + "\n" config = re.sub("=", ": ", config) diff --git a/detect_advisor/process.py b/detect_advisor/process.py index 471d184..c1621eb 100644 --- a/detect_advisor/process.py +++ b/detect_advisor/process.py @@ -221,12 +221,11 @@ def det_excluded(dir): break return excluded - def process_dir(path, dirdepth): + from . import messages dir_size = 0 dir_entries = 0 filenames_string = "" - # global global_values.messages if sig_excluded(path): return 0 @@ -245,6 +244,7 @@ def process_dir(path, dirdepth): dir_size += this_size if entry in global_values.exclude_dirs: global_values.file_list['exclude_dirs'].append(entry.path) + messages.message('FILES4') else: ftype = checkfile(entry.name, entry.path, entry.stat(follow_symlinks=False).st_size, 0, dirdepth, False) @@ -264,6 +264,7 @@ def process_dir(path, dirdepth): # check depth to exclude if global_values.exclude_files[entry.name] == 1: global_values.file_list['exclude_dirs'].append(os.path.dirname(entry.path)) + messages.message('FILES4') else: p = Path(entry.path) count = global_values.exclude_files[entry.name] @@ -271,6 +272,7 @@ def process_dir(path, dirdepth): p = p.parent count -= 1 global_values.file_list['exclude_dirs'].append(str(p)) + messages.message('FILES4') except OSError: global_values.messages += "ERROR: Unable to open folder {}\n".format(path) @@ -601,7 +603,7 @@ def pm_getter(item): global_values.cli_msgs_dict['crit'] += ( f"For {pm}:\n" + global_values.pm_dict[pm]['cli_reqd'] + '\n') - print(" Done") + print(" Done\n") return diff --git a/pyproject.toml b/pyproject.toml index f82315b..172f35d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "detect_advisor" -version = "1.0.5" +version = "1.0.6" authors = [ { name="Matthew Brady", email="mbrad@synopsys.com" }, ]