v1.0.6 - further changes

Author: matthewb66

detect_advisor/global_values.py

@@ -716,5 +716,6 @@
     'emit-db': 3,
 }
 
-exclude_dirs = [
-]
+exclude_dirs = []
+
+message_list = []

detect_advisor/messages.py

@@ -86,8 +86,10 @@
         'impact': 'Local license text may need to be scanned',
         'action': 'Add options --detect.blackduck.signature.scanner.license.search=true and optionally --detect.blackduck.signature.scanner.upload.source.mode=true',
         'cli': 'lic',
-        'cli_search': 'detect.blackduck.signature.scanner.upload.source.mode',
-        'cli_text': '--detect.blackduck.signature.scanner.upload.source.mode=true (CAUTION - will upload local source files)',
+        'cli_search': 'detect.blackduck.signature.scanner.license.search',
+        'cli_text': '--detect.blackduck.signature.scanner.license.search=true\n'
+                    '--detect.blackduck.signature.scanner.upload.source.mode=true (CAUTION - will upload local source files)',
+
     },
 
     'FILES3': {
@@ -100,6 +102,16 @@
         'cli_text': '--detect.blackduck.signature.scanner.snippet.matching=SNIPPET_MATCHING (To search for copied OSS source files and functions within source files)',
     },
 
+    'FILES4': {
+        'level': 'imp',
+        'desc': 'Folders included in scan which should probably be excluded',
+        'impact': 'Scan size may be larger than required and additional components identified',
+        'action': 'Rerun with --full option to see list of folders to exclude, then use the --detect.excluded.directories option to exclude folders',
+        'cli': 'scan',
+        'cli_search': 'detect.excluded.directories',
+        'cli_text': '--detect.excluded.directories=XXX with --detect.excluded.directories.search.depth=X (search for and exclude folders from scan - see list of folders for exclusion using --full option)',
+    },
+
     'SCAN1': {
         'level': 'crit',
         'desc': 'Overall scan size {:>,d} MB) is too large (default max scan size 5GB)',
@@ -139,7 +151,7 @@
         'level': 'imp',
         'desc': 'Large amount of data ({:>,d} MB) in {} binary files found',
         'impact': 'Binary files not analysed by standard scan, will increase scan size and impact Capacity license usage',
-        'action': 'Remove files or ignore folders (using .bdignore files), also consider using Binary scan',
+        'action': 'See list of binary files using --full option, then remove files or ignore folders (using .bdignore files), also consider using Binary scan',
         'cli': 'scan',
         'cli_search': 'detect.binary.scan.file.name.patterns',
         'cli_text': '--detect.binary.scan.file.name.patterns=exe,bin,dll (for example) and --detect.binary.scan.search.depth=X (folder depth to search for binaries)'
@@ -166,14 +178,14 @@
         'level': 'crit',
         'desc': 'Required package manager programs ({}) missing for dependency scan in invocation folder',
         'impact': 'Scan will fail',
-        'action': 'Install required package manager programs',
+        'action': 'See list of affected PMs using --full option, then install required package manager programs',
     },
 
     'PACKAGES4': {
         'level': 'imp',
         'desc': 'Required package manager programs ({}) missing for dependency scan in sub-folders',
         'impact': 'The scan will fail if the scan depth is modified from the default level 0',
-        'action': 'Install required package manager programs',
+        'action': 'See list of affected PMs using --full option, , then install required package manager programs',
     },
 
     'PACKAGES5': {
@@ -186,8 +198,8 @@
     'PACKAGES6': {
         'level': 'crit',
         'desc': 'Missing lockfiles/PMs for package manager files in invocation folder',
-        'impact': 'Dependency scan will fail unless lockfiles created, PMs installed or --detect.accuracy.required=NONE specified',
-        'action': 'Either install required package manager programs, create lockfiles or specify --detect.accuracy.required=NONE (reduced accuracy scan)',
+        'impact': 'Dependency scan will fail',
+        'action': 'See list of affected PMs using --full option, then either install required package manager programs, create lockfiles or specify --detect.accuracy.required=NONE (reduced accuracy scan)',
         'cli': 'reqd',
         'cli_search': 'detect.accuracy.required',
         'cli_text': '--detect.accuracy.required=NONE (OR specify --detect.XXXX.path=<LOCATION> where XXX is package manager OR install package managers OR create lockfiles)',
@@ -196,8 +208,8 @@
     'PACKAGES7': {
         'level': 'imp',
         'desc': 'Missing lockfiles/PMs will cause scan to fail for package manager files in sub-folders',
-        'impact': 'Dependency scan will fail if scan depth > 0 unless lockfiles created, PMs installed or --detect.accuracy.required=NONE specified',
-        'action': 'Either install required package manager programs, create lockfiles or specify --detect.accuracy.required=NONE (reduced accuracy scan)',
+        'impact': 'Dependency scan will fail if scan depth > 0',
+        'action': 'See list of affected PMs using --full option, then either install required package manager programs, create lockfiles or specify --detect.accuracy.required=NONE (reduced accuracy scan)',
         'cli': 'reqd',
         'cli_search': 'detect.accuracy.required',
         'cli_text': '--detect.accuracy.required=NONE (OR specify --detect.XXXX.path=<LOCATION> where XXX is package manager OR install package managers)',
@@ -233,7 +245,7 @@
     #     'cli_search': 'detect.accuracy.required',
     #     'cli_text': '--detect.accuracy.required=NONE (OR install JS packages)',
     # },
-    #
+
     'PACKAGES12': {
         'level': 'imp',
         'desc': 'Lockfile(s) required for dependency scan missing in invocation folder',
@@ -260,7 +272,10 @@
 
 
 def message(id, val1='', val2=''):
+    if id in global_values.message_list:
+        return
     if id in messages_dict:
+        global_values.message_list.append(id)
         if val2 != '':
             mtext = f"- {levtexts[messages_dict[id]['level']]}: " + messages_dict[id]['desc'].format(val1, val2) + '\n'
         elif val1 != '':

detect_advisor/output.py

@@ -193,10 +193,10 @@ def output_cli(critical_only, reportfile):
     if global_values.recs_msgs_dict['crit']:
         output += "Note that scan will probably fail - see CRITICAL recommendations above\n\n"
 
-    output += "    DETECT COMMAND:\n"
-    output += re.sub(r"^", "    ", global_values.cli_msgs_dict['detect'], flags=re.MULTILINE)
-    output += "\n    MINIMUM REQUIRED OPTIONS:\n"
-    output += re.sub(r"^", "    ", global_values.cli_msgs_dict['reqd'], flags=re.MULTILINE)
+    output += "DETECT COMMAND:\n"
+    output += global_values.cli_msgs_dict['detect']
+    output += "\nMINIMUM REQUIRED OPTIONS:\n"
+    output += global_values.cli_msgs_dict['reqd']
 
     # if len(bdignore_list) > 0:
     # 	if report:
@@ -208,8 +208,11 @@ def output_cli(critical_only, reportfile):
 
     if not critical_only:
         output += '\n'
+        if global_values.cli_msgs_dict['proj'] != '':
+            output += "\nPROJECT OPTIONS:\n" + global_values.cli_msgs_dict['proj'] + "\n"
+
         if global_values.cli_msgs_dict['scan'] != '':
-            output += "\nOPTIONS TO IMPROVE SCAN COVERAGE:\n" + global_values.cli_msgs_dict['scan'] + "\n"
+            output += "\nOPTIONS TO IMPROVE SCAN COVERAGE/ACCURACY:\n" + global_values.cli_msgs_dict['scan'] + "\n"
 
         if global_values.cli_msgs_dict['size'] != '':
             output += "\nOPTIONS TO REDUCE SIGNATURE SCAN SIZE:\n" + global_values.cli_msgs_dict['size'] + "\n"
@@ -220,9 +223,6 @@ def output_cli(critical_only, reportfile):
         if global_values.cli_msgs_dict['lic'] != '':
             output += "\nOPTIONS TO IMPROVE LICENSE COMPLIANCE ANALYSIS:\n" + global_values.cli_msgs_dict['lic'] + "\n"
 
-        if global_values.cli_msgs_dict['proj'] != '':
-            output += "\nPROJECT OPTIONS:\n" + global_values.cli_msgs_dict['proj'] + "\n"
-
         if global_values.cli_msgs_dict['rep'] != '':
             output += "\nREPORTING OPTIONS:\n" + global_values.cli_msgs_dict['rep'] + "\n"
 
@@ -241,11 +241,11 @@ def output_config(projdir):
                  "# Uncomment and update required options\n#\n#\n" + \
                  "# DETECT COMMAND TO RUN:\n#\n" + global_values.cli_msgs_dict['detect'] + "\n" + \
                  "# MINIMUM REQUIRED OPTIONS:\n#\n" + global_values.cli_msgs_dict['reqd'] + "\n" + \
-                 "# OPTIONS TO IMPROVE SCAN COVERAGE:\n#\n" + global_values.cli_msgs_dict['scan'] + "\n" + \
+                 "# PROJECT OPTIONS:\n#\n" + global_values.cli_msgs_dict['proj'] + "\n" + \
+                 "# OPTIONS TO IMPROVE SCAN COVERAGE/ACCURACY:\n#\n" + global_values.cli_msgs_dict['scan'] + "\n" + \
                  "# OPTIONS TO REDUCE SIGNATURE SCAN SIZE:\n#\n" + global_values.cli_msgs_dict['size'] + "\n" + \
                  "# OPTIONS TO CONFIGURE DEPENDENCY SCAN:\n#\n" + global_values.cli_msgs_dict['dep'] + "\n" + \
                  "# OPTIONS TO IMPROVE LICENSE COMPLIANCE ANALYSIS:\n#\n" + global_values.cli_msgs_dict['lic'] + "\n" + \
-                 "# PROJECT OPTIONS:\n#\n" + global_values.cli_msgs_dict['proj'] + "\n" + \
                  "# REPORTING OPTIONS:\n#\n" + global_values.cli_msgs_dict['rep'] + "\n"
 
         config = re.sub("=", ": ", config)

detect_advisor/process.py

@@ -221,12 +221,11 @@ def det_excluded(dir):
             break
     return excluded
 
-
 def process_dir(path, dirdepth):
+    from . import messages
     dir_size = 0
     dir_entries = 0
     filenames_string = ""
-    # global global_values.messages
 
     if sig_excluded(path):
         return 0
@@ -245,6 +244,7 @@ def process_dir(path, dirdepth):
                 dir_size += this_size
                 if entry in global_values.exclude_dirs:
                     global_values.file_list['exclude_dirs'].append(entry.path)
+                    messages.message('FILES4')
             else:
                 ftype = checkfile(entry.name, entry.path, entry.stat(follow_symlinks=False).st_size, 0,
                                   dirdepth, False)
@@ -264,13 +264,15 @@ def process_dir(path, dirdepth):
                     # check depth to exclude
                     if global_values.exclude_files[entry.name] == 1:
                         global_values.file_list['exclude_dirs'].append(os.path.dirname(entry.path))
+                        messages.message('FILES4')
                     else:
                         p = Path(entry.path)
                         count = global_values.exclude_files[entry.name]
                         while count > 0:
                             p = p.parent
                             count -= 1
                         global_values.file_list['exclude_dirs'].append(str(p))
+                        messages.message('FILES4')
 
     except OSError:
         global_values.messages += "ERROR: Unable to open folder {}\n".format(path)
@@ -601,7 +603,7 @@ def pm_getter(item):
             global_values.cli_msgs_dict['crit'] += (
                     f"For {pm}:\n" + global_values.pm_dict[pm]['cli_reqd'] + '\n')
 
-    print(" Done")
+    print(" Done\n")
 
     return
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "detect_advisor"
-version = "1.0.5"
+version = "1.0.6"
 authors = [
   { name="Matthew Brady", email="[email protected]" },
 ]