Skip to content

Commit

Permalink
Merge pull request #7 from matthewb66/dev
Browse files Browse the repository at this point in the history 
Dev
  • Loading branch information
@matthewb66
matthewb66 authored Apr 22, 2024
2 parents 34df3b7 + 3b03e37 commit 57de388
Show file tree
Hide file tree
Showing 5 changed files with 113 additions and 51 deletions.
14 changes: 12 additions & 2 deletions detect_advisor/global_values.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#
# Constants
advisor_version = "1.0.5"
advisor_version = "1.0.6"
detect_version = "9.X.0"

ext_list = {
Expand Down Expand Up @@ -688,12 +688,13 @@
'js_single': [],
'arcs_pm': [],
'det': [],
'exclude_dirs': [],
}

files_dict = {
'bin_large': {},
'det': {},
'large': {}
'large': {},
}

detectors_list = []
Expand All @@ -709,3 +710,12 @@
rep = ""
full_rep = ""
messages = ""

exclude_files = {
'pyvenv.cfg': 1,
'emit-db': 3,
}

exclude_dirs = []

message_list = []
65 changes: 40 additions & 25 deletions detect_advisor/messages.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,10 @@
'impact': 'Local license text may need to be scanned',
'action': 'Add options --detect.blackduck.signature.scanner.license.search=true and optionally --detect.blackduck.signature.scanner.upload.source.mode=true',
'cli': 'lic',
'cli_search': 'detect.blackduck.signature.scanner.upload.source.mode',
'cli_text': '--detect.blackduck.signature.scanner.upload.source.mode=true (CAUTION - will upload local source files)',
'cli_search': 'detect.blackduck.signature.scanner.license.search',
'cli_text': '--detect.blackduck.signature.scanner.license.search=true\n'
'--detect.blackduck.signature.scanner.upload.source.mode=true (CAUTION - will upload local source files)',

},

'FILES3': {
Expand All @@ -100,6 +102,16 @@
'cli_text': '--detect.blackduck.signature.scanner.snippet.matching=SNIPPET_MATCHING (To search for copied OSS source files and functions within source files)',
},

'FILES4': {
'level': 'imp',
'desc': 'Folders included in scan which should probably be excluded',
'impact': 'Scan size may be larger than required and additional components identified',
'action': 'Rerun with --full option to see list of folders to exclude, then use the --detect.excluded.directories option to exclude folders',
'cli': 'scan',
'cli_search': 'detect.excluded.directories',
'cli_text': '--detect.excluded.directories=XXX with --detect.excluded.directories.search.depth=X (search for and exclude folders from scan - see list of folders for exclusion using --full option)',
},

'SCAN1': {
'level': 'crit',
'desc': 'Overall scan size {:>,d} MB) is too large (default max scan size 5GB)',
Expand Down Expand Up @@ -139,7 +151,7 @@
'level': 'imp',
'desc': 'Large amount of data ({:>,d} MB) in {} binary files found',
'impact': 'Binary files not analysed by standard scan, will increase scan size and impact Capacity license usage',
'action': 'Remove files or ignore folders (using .bdignore files), also consider using Binary scan',
'action': 'See list of binary files using --full option, then remove files or ignore folders (using .bdignore files), also consider using Binary scan',
'cli': 'scan',
'cli_search': 'detect.binary.scan.file.name.patterns',
'cli_text': '--detect.binary.scan.file.name.patterns=exe,bin,dll (for example) and --detect.binary.scan.search.depth=X (folder depth to search for binaries)'
Expand All @@ -166,14 +178,14 @@
'level': 'crit',
'desc': 'Required package manager programs ({}) missing for dependency scan in invocation folder',
'impact': 'Scan will fail',
'action': 'Install required package manager programs',
'action': 'See list of affected PMs using --full option, then install required package manager programs',
},

'PACKAGES4': {
'level': 'imp',
'desc': 'Required package manager programs ({}) missing for dependency scan in sub-folders',
'impact': 'The scan will fail if the scan depth is modified from the default level 0',
'action': 'Install required package manager programs',
'action': 'See list of affected PMs using --full option, , then install required package manager programs',
},

'PACKAGES5': {
Expand All @@ -186,8 +198,8 @@
'PACKAGES6': {
'level': 'crit',
'desc': 'Missing lockfiles/PMs for package manager files in invocation folder',
'impact': 'Dependency scan will fail unless lockfiles created, PMs installed or --detect.accuracy.required=NONE specified',
'action': 'Either install required package manager programs, create lockfiles or specify --detect.accuracy.required=NONE (reduced accuracy scan)',
'impact': 'Dependency scan will fail',
'action': 'See list of affected PMs using --full option, then either install required package manager programs, create lockfiles or specify --detect.accuracy.required=NONE (reduced accuracy scan)',
'cli': 'reqd',
'cli_search': 'detect.accuracy.required',
'cli_text': '--detect.accuracy.required=NONE (OR specify --detect.XXXX.path=<LOCATION> where XXX is package manager OR install package managers OR create lockfiles)',
Expand All @@ -196,8 +208,8 @@
'PACKAGES7': {
'level': 'imp',
'desc': 'Missing lockfiles/PMs will cause scan to fail for package manager files in sub-folders',
'impact': 'Dependency scan will fail if scan depth > 0 unless lockfiles created, PMs installed or --detect.accuracy.required=NONE specified',
'action': 'Either install required package manager programs, create lockfiles or specify --detect.accuracy.required=NONE (reduced accuracy scan)',
'impact': 'Dependency scan will fail if scan depth > 0',
'action': 'See list of affected PMs using --full option, then either install required package manager programs, create lockfiles or specify --detect.accuracy.required=NONE (reduced accuracy scan)',
'cli': 'reqd',
'cli_search': 'detect.accuracy.required',
'cli_text': '--detect.accuracy.required=NONE (OR specify --detect.XXXX.path=<LOCATION> where XXX is package manager OR install package managers)',
Expand All @@ -217,22 +229,22 @@
'action': 'Invoke scan from within required virtualenv',
},

'PACKAGES10': {
'level': 'crit',
'desc': 'JS packages must be installed for accurate dependency scanning for projects in invocation folder',
'impact': 'JS packages will not be identified correctly',
'action': "Run 'npm install' prior to scanning, or consider specifying --detect.accuracy.required=NONE (reduced accuracy scan)",
},

'PACKAGES11': {
'level': 'imp',
'desc': 'JS packages must be installed for accurate dependency scanning for JS projects below invocation folder',
'impact': 'JS packages will not be identified correctly',
'action': "Run 'npm install' prior to scanning, or consider specifying --detect.accuracy.required=NONE (reduced accuracy scan)",
'cli': 'reqd',
'cli_search': 'detect.accuracy.required',
'cli_text': '--detect.accuracy.required=NONE (OR install JS packages)',
},
# 'PACKAGES10': {
# 'level': 'crit',
# 'desc': 'JS packages must be installed for accurate dependency scanning for projects in invocation folder',
# 'impact': 'JS packages will not be identified correctly',
# 'action': "Run 'npm install' prior to scanning, or consider specifying --detect.accuracy.required=NONE (reduced accuracy scan)",
# },
#
# 'PACKAGES11': {
# 'level': 'imp',
# 'desc': 'JS packages must be installed for accurate dependency scanning for JS projects below invocation folder',
# 'impact': 'JS packages will not be identified correctly',
# 'action': "Run 'npm install' prior to scanning, or consider specifying --detect.accuracy.required=NONE (reduced accuracy scan)",
# 'cli': 'reqd',
# 'cli_search': 'detect.accuracy.required',
# 'cli_text': '--detect.accuracy.required=NONE (OR install JS packages)',
# },

'PACKAGES12': {
'level': 'imp',
Expand Down Expand Up @@ -260,7 +272,10 @@


def message(id, val1='', val2=''):
if id in global_values.message_list:
return
if id in messages_dict:
global_values.message_list.append(id)
if val2 != '':
mtext = f"- {levtexts[messages_dict[id]['level']]}: " + messages_dict[id]['desc'].format(val1, val2) + '\n'
elif val1 != '':
Expand Down
23 changes: 12 additions & 11 deletions detect_advisor/output.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,8 @@ def output_full_rep(reportfile):
'huge': "HUGE FILES (> {}MB):".format(trunc(global_values.hugesize / 1000000)),
'js_single': 'SINGLETON JS FILES:',
'arcs_pm': 'ARCHIVES CONTAINING PACKAGE MANAGER CONFIGS:',
'bin': 'BINARY FILES:'
'bin': 'BINARY FILES:',
'exclude_dirs': 'FOLDERS WHICH SHOULD BE EXCLUDED:'
}
for ftype in desc.keys():
rep += desc[ftype] + '\n' + "\n".join(global_values.file_list[ftype]) + '\n\n'
Expand Down Expand Up @@ -192,10 +193,10 @@ def output_cli(critical_only, reportfile):
if global_values.recs_msgs_dict['crit']:
output += "Note that scan will probably fail - see CRITICAL recommendations above\n\n"

output += " DETECT COMMAND:\n"
output += re.sub(r"^", " ", global_values.cli_msgs_dict['detect'], flags=re.MULTILINE)
output += "\n MINIMUM REQUIRED OPTIONS:\n"
output += re.sub(r"^", " ", global_values.cli_msgs_dict['reqd'], flags=re.MULTILINE)
output += "DETECT COMMAND:\n"
output += global_values.cli_msgs_dict['detect']
output += "\nMINIMUM REQUIRED OPTIONS:\n"
output += global_values.cli_msgs_dict['reqd']

# if len(bdignore_list) > 0:
# if report:
Expand All @@ -207,8 +208,11 @@ def output_cli(critical_only, reportfile):

if not critical_only:
output += '\n'
if global_values.cli_msgs_dict['proj'] != '':
output += "\nPROJECT OPTIONS:\n" + global_values.cli_msgs_dict['proj'] + "\n"

if global_values.cli_msgs_dict['scan'] != '':
output += "\nOPTIONS TO IMPROVE SCAN COVERAGE:\n" + global_values.cli_msgs_dict['scan'] + "\n"
output += "\nOPTIONS TO IMPROVE SCAN COVERAGE/ACCURACY:\n" + global_values.cli_msgs_dict['scan'] + "\n"

if global_values.cli_msgs_dict['size'] != '':
output += "\nOPTIONS TO REDUCE SIGNATURE SCAN SIZE:\n" + global_values.cli_msgs_dict['size'] + "\n"
Expand All @@ -219,9 +223,6 @@ def output_cli(critical_only, reportfile):
if global_values.cli_msgs_dict['lic'] != '':
output += "\nOPTIONS TO IMPROVE LICENSE COMPLIANCE ANALYSIS:\n" + global_values.cli_msgs_dict['lic'] + "\n"

if global_values.cli_msgs_dict['proj'] != '':
output += "\nPROJECT OPTIONS:\n" + global_values.cli_msgs_dict['proj'] + "\n"

if global_values.cli_msgs_dict['rep'] != '':
output += "\nREPORTING OPTIONS:\n" + global_values.cli_msgs_dict['rep'] + "\n"

Expand All @@ -240,11 +241,11 @@ def output_config(projdir):
"# Uncomment and update required options\n#\n#\n" + \
"# DETECT COMMAND TO RUN:\n#\n" + global_values.cli_msgs_dict['detect'] + "\n" + \
"# MINIMUM REQUIRED OPTIONS:\n#\n" + global_values.cli_msgs_dict['reqd'] + "\n" + \
"# OPTIONS TO IMPROVE SCAN COVERAGE:\n#\n" + global_values.cli_msgs_dict['scan'] + "\n" + \
"# PROJECT OPTIONS:\n#\n" + global_values.cli_msgs_dict['proj'] + "\n" + \
"# OPTIONS TO IMPROVE SCAN COVERAGE/ACCURACY:\n#\n" + global_values.cli_msgs_dict['scan'] + "\n" + \
"# OPTIONS TO REDUCE SIGNATURE SCAN SIZE:\n#\n" + global_values.cli_msgs_dict['size'] + "\n" + \
"# OPTIONS TO CONFIGURE DEPENDENCY SCAN:\n#\n" + global_values.cli_msgs_dict['dep'] + "\n" + \
"# OPTIONS TO IMPROVE LICENSE COMPLIANCE ANALYSIS:\n#\n" + global_values.cli_msgs_dict['lic'] + "\n" + \
"# PROJECT OPTIONS:\n#\n" + global_values.cli_msgs_dict['proj'] + "\n" + \
"# REPORTING OPTIONS:\n#\n" + global_values.cli_msgs_dict['rep'] + "\n"

config = re.sub("=", ": ", config)
Expand Down
60 changes: 48 additions & 12 deletions detect_advisor/process.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from math import trunc
import platform
import hashlib
from pathlib import Path

from . import global_values
from . import messages
Expand Down Expand Up @@ -220,12 +221,11 @@ def det_excluded(dir):
break
return excluded


def process_dir(path, dirdepth):
from . import messages
dir_size = 0
dir_entries = 0
filenames_string = ""
# global global_values.messages

if sig_excluded(path):
return 0
Expand All @@ -242,6 +242,9 @@ def process_dir(path, dirdepth):
global_values.counts['dir'][global_values.notinarc] += 1
this_size = process_dir(entry.path, dirdepth)
dir_size += this_size
if entry in global_values.exclude_dirs:
global_values.file_list['exclude_dirs'].append(entry.path)
messages.message('FILES4')
else:
ftype = checkfile(entry.name, entry.path, entry.stat(follow_symlinks=False).st_size, 0,
dirdepth, False)
Expand All @@ -250,11 +253,27 @@ def process_dir(path, dirdepth):
# all_bin = True
# else:
# all_bin = False

ext = os.path.splitext(entry.name)[1]
if ext in global_values.ext_list['zip']:
process_zip(entry.path, 0, dirdepth)

dir_size += entry.stat(follow_symlinks=False).st_size

if entry.name in global_values.exclude_files.keys():
# check depth to exclude
if global_values.exclude_files[entry.name] == 1:
global_values.file_list['exclude_dirs'].append(os.path.dirname(entry.path))
messages.message('FILES4')
else:
p = Path(entry.path)
count = global_values.exclude_files[entry.name]
while count > 0:
p = p.parent
count -= 1
global_values.file_list['exclude_dirs'].append(str(p))
messages.message('FILES4')

except OSError:
global_values.messages += "ERROR: Unable to open folder {}\n".format(path)
return 0
Expand Down Expand Up @@ -518,9 +537,26 @@ def pm_getter(item):
messages.message('PACKAGES4', ','.join(exes))

if global_values.pm_dict[pm]['accuracy'] == 'LOW':
if (global_values.pm_dict[pm]['exec_reqd'] and item[1]['exes_missing'] and
not item[1]['lockfound'] and global_values.pm_dict[pm]['lockfile_reqd']):
info += " - LOW accuracy scan due to missing PM/lockfiles"
exec_files_bool = (len(global_values.pm_dict[pm]['exec_files'])>0 and item[1]['exes_missing'] and
not global_values.pm_dict[pm]['execs_reqd'])
lock_files_bool = (len(global_values.pm_dict[pm]['lock_files'])>0 and not item[1]['lockfound'] and
not global_values.pm_dict[pm]['lockfile_reqd'])
exec_list_zero = len(global_values.pm_dict[pm]['exec_files']) == 0
lock_list_zero = len(global_values.pm_dict[pm]['lock_files']) == 0

# Calculate drop-through scenarios (LOW accuracy):
# if (exec_files_bool and lock_files_bool)
# OR
# (exec_list_zero and lock_list_zero)
# OR
# (exec_file_bool and lock_list_zero)
# OR
# (exes_list_zero and lock_files_bool)

if ((exec_files_bool and lock_files_bool) or
(exec_list_zero and lock_list_zero) or
(lock_files_bool and lock_list_zero) or
(exec_list_zero and lock_files_bool)):
if item[1]['mindepth'] == 1:
messages.message('PACKAGES6', ','.join(exes))
else:
Expand All @@ -532,14 +568,14 @@ def pm_getter(item):
else:
messages.message('PLATFORM4', pm)

if pm in ['PIP', 'PYTHON']:
if pm in ['PIP']:
check_python_venv(item[1]['mindepth'])

if pm in ['NPM', 'YARN', 'LERNA', 'PNPM']:
if item[1]['mindepth'] == 1:
messages.message('PACKAGES10')
else:
messages.message('PACKAGES11')
# if pm in ['NPM', 'YARN', 'LERNA', 'PNPM']:
# if item[1]['mindepth'] == 1:
# messages.message('PACKAGES10')
# else:
# messages.message('PACKAGES11')

global_values.rep += \
" - {:11} {:>8d} {:>8d} {:>5,d} {}\n".format(
Expand Down Expand Up @@ -567,7 +603,7 @@ def pm_getter(item):
global_values.cli_msgs_dict['crit'] += (
f"For {pm}:\n" + global_values.pm_dict[pm]['cli_reqd'] + '\n')

print(" Done")
print(" Done\n")

return

Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

[project]
name = "detect_advisor"
version = "1.0.5"
version = "1.0.6"
authors = [
{ name="Matthew Brady", email="[email protected]" },
]
Expand Down

0 comments on commit 57de388

Please sign in to comment.