Skip to content

Commit

Permalink
Dedupe output, group module warnings and errors
Browse files Browse the repository at this point in the history
  • Loading branch information
@antfie
antfie committed Mar 13, 2023
1 parent 8672561 commit daebbc3
Show file tree
Hide file tree
Showing 6 changed files with 114 additions and 47 deletions.
45 changes: 40 additions & 5 deletions analyze_files.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ func (data Data) analyzeUploadedFiles() {
}

detectSensitiveFiles(data, &report, files)
detectTestArtefacts(data, &report, files)
detectNodeModules(data, &report)
detectRoslyn(data, &report, files)
detectGit(data, &report, files)
Expand All @@ -36,7 +37,9 @@ func (data Data) analyzeUploadedFiles() {
detectUnwantedFiles(data, &report, files, ".sln", ".NET solution file", []string{"Do not upload C# source code. They will not be scanned", "Veracode requires the .NET application to be compiled"})
detectUnwantedFiles(data, &report, files, ".csproj", "C# project file", []string{"Do not upload C# source code. They will not be scanned", "Veracode requires the .NET application to be compiled"})
detectUnwantedFiles(data, &report, files, ".c", "C source code file", []string{"Do not upload C source code. They will not be scanned", "Veracode requires the application to be compiled with debug symbols"})
detectUnwantedFiles(data, &report, files, ".test.dll", "test artifact", []string{"Do not upload any test code"})
detectUnwantedFiles(data, &report, files, ".test.dll", "test artifact", []string{"Do not upload any testing artefacts"})
detectUnwantedFiles(data, &report, files, ".unittests.dll", "test artifact", []string{"Do not upload any testing artefacts"})
detectUnwantedFiles(data, &report, files, ".unittest.dll", "test artifact", []string{"Do not upload any testing artefacts"})
detectUnwantedFiles(data, &report, files, ".coffee", "CoffeeScript file", []string{"CoffeeScript source code files will not be scanned", "Review the JavaScript/TypeScript packaging cheatsheet: https://nhinv11.github.io/#/JavaScript%20/%20TypeScript", "Consider using the unofficial JavaScript/TypeScript packaging tool: https://github.com/fw10/veracode-javascript-packager"})

if report.Len() > 0 {
Expand All @@ -50,7 +53,9 @@ func detectSensitiveFiles(data Data, report *strings.Builder, files []string) {

for _, fileName := range files {
if isFileNameInFancyList(fileName, secretFileNames) {
foundFiles = append(foundFiles, fileName)
if !isStringInStringArray(fileName, foundFiles) {
foundFiles = append(foundFiles, fileName)
}
}
}

Expand All @@ -67,18 +72,46 @@ func detectSensitiveFiles(data Data, report *strings.Builder, files []string) {
data.makeRecommendation("Do not upload any secrets, certificates or key files")
}

func detectTestArtefacts(data Data, report *strings.Builder, files []string) {
var foundFiles []string

for _, fileName := range files {
if isFileNameInFancyList(fileName, testFileNames) {
if !isStringInStringArray(fileName, foundFiles) {
foundFiles = append(foundFiles, fileName)
}
}
}

if len(foundFiles) == 0 {
return
}

report.WriteString(fmt.Sprintf(
"❌ %d test artefact%s were found: %s\n",
len(foundFiles),
pluralise(len(foundFiles)),
top5StringList(foundFiles)))

data.makeRecommendation("Do not upload any testing artefacts")
}

func detectNodeModules(data Data, report *strings.Builder) {
var foundFiles []string

for _, file := range data.PrescanFileList.Files {
if strings.Contains(strings.ToLower(file.Name), "_nodemodule_") {
foundFiles = append(foundFiles, file.Name)
if !isStringInStringArray(file.Name, foundFiles) {
foundFiles = append(foundFiles, file.Name)
}
}
}

for _, module := range data.DetailedReport.StaticAnalysis.Modules {
if strings.Contains(strings.ToLower(module.Name), "_nodemodule_") {
foundFiles = append(foundFiles, module.Name)
if !isStringInStringArray(module.Name, foundFiles) {
foundFiles = append(foundFiles, module.Name)
}
}
}

Expand Down Expand Up @@ -134,7 +167,9 @@ func detectUnwantedFiles(data Data, report *strings.Builder, files []string, suf

for _, fileName := range files {
if strings.HasSuffix(strings.ToLower(fileName), suffix) && !isStringInStringArray(fileName, foundFiles) {
foundFiles = append(foundFiles, fileName)
if !isStringInStringArray(fileName, foundFiles) {
foundFiles = append(foundFiles, fileName)
}
}
}

Expand Down
96 changes: 60 additions & 36 deletions analyze_modules.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,16 @@ package main
import (
"fmt"
"strings"

"github.com/dustin/go-humanize"
)

func (data Data) analyzeModules() {
var report strings.Builder

if data.PrescanModuleList.TotalSize > 1e+9 {
report.WriteString(fmt.Sprintf(
"⚠️ The size of the modules was %s. This is a very large scan and will likely take a long time to run\n",
humanize.Bytes(uint64(data.PrescanModuleList.TotalSize))))
}
// if data.PrescanModuleList.TotalSize > 1e+9 {
// report.WriteString(fmt.Sprintf(
// "⚠️ The size of the modules was %s. This is a very large scan and will likely take a long time to run\n",
// humanize.Bytes(uint64(data.PrescanModuleList.TotalSize))))
// }

if len(data.PrescanModuleList.Modules) > 1000 {
report.WriteString(fmt.Sprintf(
Expand Down Expand Up @@ -66,20 +64,28 @@ func (data Data) analyzeModules() {
func (data Data) analyzeModuleFatalErrors() {
var report strings.Builder

var errors []string
errors := make(map[string][]string)

for _, module := range data.PrescanModuleList.Modules {
if module.HasFatalErrors {
formattedError := fmt.Sprintf("\"%s\": %s", module.Name, module.Status)
reason := module.getFatalReason()

if _, isReasonInMap := errors[reason]; !isReasonInMap {
errors[reason] = []string{}
}

if !isStringInStringArray(formattedError, errors) {
errors = append(errors, formattedError)
if !isStringInStringArray(module.Name, errors[reason]) {
errors[reason] = append(errors[reason], module.Name)
}
}
}

for _, errors := range errors {
report.WriteString(fmt.Sprintf("⚠️ %s\n", errors))
for errorMessage, affectedModules := range errors {
report.WriteString(fmt.Sprintf(
"❌ %d %s: %s\n",
len(affectedModules),
errorMessage,
top5StringList(affectedModules)))
}

if report.Len() > 0 {
Expand All @@ -91,9 +97,13 @@ func (data Data) analyzeModuleFatalErrors() {
func (data Data) analyzeModuleWarnings() {
var report strings.Builder

var warnings []string
warnings := make(map[string][]string)

for _, module := range data.PrescanModuleList.Modules {
if module.HasFatalErrors {
continue
}

if module.IsThirdParty {
continue
}
Expand Down Expand Up @@ -127,39 +137,53 @@ func (data Data) analyzeModuleWarnings() {
data.makeRecommendation("Ensure you include PDB files for all 1st and 2nd party .NET components. This enables Veracode to accurately report line numbers for any found flaws")
}

formattedIssue := fmt.Sprintf("\"%s\": %s", module.Name, issue.Details)
if _, isMessageInMap := warnings[issue.Details]; !isMessageInMap {
warnings[issue.Details] = []string{}
}

if !isStringInStringArray(formattedIssue, warnings) {
warnings = append(warnings, formattedIssue)
if !isStringInStringArray(module.Name, warnings[issue.Details]) {
warnings[issue.Details] = append(warnings[issue.Details], module.Name)
}
}

for _, statusMessage := range strings.Split(module.Status, ",") {
if module.Status == "OK" {
continue
}
// for _, statusMessage := range strings.Split(module.Status, ",") {
// if module.Status == "OK" {
// continue
// }

formattedStatusMessage := strings.TrimSpace(statusMessage)
// formattedStatusMessage := strings.TrimSpace(statusMessage)

if strings.HasPrefix(formattedStatusMessage, "Unsupported Framework") {
// These are captured under the issue details
continue
}
// if strings.HasPrefix(formattedStatusMessage, "Unsupported Framework") {
// // These are captured under the issue details
// continue
// }

formattedIssue := fmt.Sprintf("\"%s\": %s", module.Name, formattedStatusMessage)
// if _, isMessageInMap := warnings[formattedStatusMessage]; !isMessageInMap {
// warnings[formattedStatusMessage] = []string{}
// }

if !isStringInStringArray(formattedIssue, warnings) {
warnings = append(warnings, formattedIssue)
// if !isStringInStringArray(module.Name, warnings[formattedStatusMessage]) {
// warnings[formattedStatusMessage] = append(warnings[formattedStatusMessage], module.Name)
// }

if strings.Contains(formattedStatusMessage, "Missing Supporting Files") {
data.makeRecommendation("Be sure to include all the components that make up the application within the upload. Do not omit any 2nd or third party libraries from the upload")
}
}
}
// formattedIssue := fmt.Sprintf("\"%s\": %s", module.Name, formattedStatusMessage)

// if !isStringInStringArray(formattedIssue, warnings) {
// warnings = append(warnings, formattedIssue)

// if strings.Contains(formattedStatusMessage, "Missing Supporting Files") {
// data.makeRecommendation("Be sure to include all the components that make up the application within the upload. Do not omit any 2nd or third party libraries from the upload")
// }
// }
// }
}

for _, warning := range warnings {
report.WriteString(fmt.Sprintf("⚠️ %s\n", warning))
for warningMessage, affectedModules := range warnings {
report.WriteString(fmt.Sprintf(
"⚠️ %d %s: %s\n",
len(affectedModules),
warningMessage,
top5StringList(affectedModules)))
}

if report.Len() > 0 {
Expand Down
10 changes: 5 additions & 5 deletions prescan_module_list.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,13 +62,13 @@ func (api API) getPrescanModuleList(appId, buildId int) PrescanModuleList {

func calculateModuleSize(size string) int {
var totalModuleSize = 0
totalModuleSize += foo(size, "GB", 1e+9)
totalModuleSize += foo(size, "MB", 1e+6)
totalModuleSize += foo(size, "KB", 1000)
totalModuleSize += convertSize(size, "GB", 1e+9)
totalModuleSize += convertSize(size, "MB", 1e+6)
totalModuleSize += convertSize(size, "KB", 1000)
return totalModuleSize
}

func foo(size, measurement string, multiplier int) int {
func convertSize(size, measurement string, multiplier int) int {
if !strings.HasSuffix(size, measurement) {
return 0
}
Expand Down Expand Up @@ -97,7 +97,7 @@ func (moduleList PrescanModuleList) getFromName(moduleName string) PrescanModule
func (module PrescanModule) getFatalReason() string {
for _, issue := range strings.Split(module.Status, ",") {
if strings.HasPrefix(issue, "(Fatal)") {
return strings.Replace(issue, "(Fatal)", ": ", 1)
return strings.Replace(strings.Replace(issue, "(Fatal)", "", 1), " - 1 File", "", 1)
}
}

Expand Down
2 changes: 1 addition & 1 deletion release.sh
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# !/usr/bin/env sh

ESCAPE=$'\e'
export VERSION="1.10"
export VERSION="1.11"

./build.sh && \

Expand Down
6 changes: 6 additions & 0 deletions test_file_names.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package main

var testFileNames = []string{
"nunit.framework.dll",
"Moq.dll",
}
2 changes: 2 additions & 0 deletions third_party_module_names.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,6 @@ var thirdPartyModules = []string{
"system.*.dll",
"Telerik.*.dll",
"WebGrease.dll",
"phantomjs.exe",
"Moq.dll",
}

0 comments on commit daebbc3

Please sign in to comment.