From a486242410bb7d0f6ef8b9cdca1fd187a2fe8f4c Mon Sep 17 00:00:00 2001 From: Sid Dange Date: Sat, 29 Jul 2023 23:10:34 -0400 Subject: [PATCH] Create blocklist to filter out unwanted matched files --- internal/cmd/submit/submit.go | 143 +++++++++++++++++- internal/cmd/submit/submit_test.go | 24 +-- .../{coverage-files => coverage}/report-2.xml | 0 .../{coverage-files => coverage}/report.xml | 0 .../vendor/simplecov/coverage_statistic.go | 7 + 5 files changed, 158 insertions(+), 16 deletions(-) rename internal/cmd/submit/testdata/example-reports-dir/{coverage-files => coverage}/report-2.xml (100%) rename internal/cmd/submit/testdata/example-reports-dir/{coverage-files => coverage}/report.xml (100%) create mode 100644 internal/cmd/submit/testdata/example-reports-dir/vendor/simplecov/coverage_statistic.go diff --git a/internal/cmd/submit/submit.go b/internal/cmd/submit/submit.go index 84253369..688685e3 100644 --- a/internal/cmd/submit/submit.go +++ b/internal/cmd/submit/submit.go @@ -289,7 +289,7 @@ func (s *Submit) bundle() (string, error) { // if coverage file paths are not provided, we infer them var coveragePaths = s.coveragePaths if len(coveragePaths) == 0 { - coveragePaths, err = coveragePathsInferred() + coveragePaths, err = s.coveragePathsInferred() } if err == nil && len(coveragePaths) > 0 { @@ -452,11 +452,15 @@ func xmlPathsFromArgs(args []string) ([]string, error) { return paths, nil } -func globPatternFromPattern(pattern string) string { - return fmt.Sprintf("./**/%s", pattern) +func globPatternFromPattern(repositoryPath string, pattern string) string { + if len(repositoryPath) == 0 { + repositoryPath = "." + } + + return fmt.Sprintf("%s/**/%s", repositoryPath, pattern) } -func coveragePathsInferred() ([]string, error) { +func (s *Submit) coveragePathsInferred() ([]string, error) { coverageFileTypes := []string{ "*coverage*.*", "nosetests.xml", @@ -481,9 +485,120 @@ func coveragePathsInferred() ([]string, error) { "test_cov.xml", } + fileBlocklistMatchers := []string{ + `__pycache__`, + `node_modules/.*`, + `vendor`, + `\.circleci`, + `\.git`, + `\.gitignore`, + `\.nvmrc`, + `\.nyc_output`, + `\.tox`, + `.*\.am$`, + `.*\.bash$`, + `.*\.bat$`, + `.*\.bw$`, + `.*\.cfg$`, + `.*\.class$`, + `.*\.cmake$`, + `.*\.cmake$`, + `.*\.conf$`, + `.*\.coverage$`, + `.*\.cp$`, + `.*\.cpp$`, + `.*\.crt$`, + `.*\.css$`, + `.*\.csv$`, + `.*\.csv$`, + `.*\.data$`, + `.*\.db$`, + `.*\.dox$`, + `.*\.ec$`, + `.*\.ec$`, + `.*\.egg$`, + `.*\.egg-info$`, + `.*\.el$`, + `.*\.env$`, + `.*\.erb$`, + `.*\.exe$`, + `.*\.ftl$`, + `.*\.gif$`, + `.*\.go$`, + `.*\.gradle$`, + `.*\.gz$`, + `.*\.h$`, + `.*\.html$`, + `.*\.in$`, + `.*\.jade$`, + `.*\.jar.*$`, + `.*\.jpeg$`, + `.*\.jpg$`, + `.*\.js$`, + `.*\.less$`, + `.*\.log$`, + `.*\.m4$`, + `.*\.mak.*$`, + `.*\.map$`, + `.*\.marker$`, + `.*\.md$`, + `.*\.o$`, + `.*\.p12$`, + `.*\.pem$`, + `.*\.png$`, + `.*\.pom.*$`, + `.*\.profdata$`, + `.*\.proto$`, + `.*\.ps1$`, + `.*\.pth$`, + `.*\.py$`, + `.*\.pyc$`, + `.*\.pyo$`, + `.*\.rb$`, + `.*\.rsp$`, + `.*\.rst$`, + `.*\.ru$`, + `.*\.sbt$`, + `.*\.scss$`, + `.*\.scss$`, + `.*\.serialized$`, + `.*\.sh$`, + `.*\.snapshot$`, + `.*\.sql$`, + `.*\.svg$`, + `.*\.tar\.tz$`, + `.*\.template$`, + `.*\.ts$`, + `.*\.whl$`, + `.*\.xcconfig$`, + `.*\.xcoverage\..*$`, + `.*/classycle/report\.xml$`, + `.*codecov\.yml$`, + `.*~$`, + `.*\.coveragerc$`, + `\.coverage.*$`, + `codecov\.SHA256SUM$`, + `codecov\.SHA256SUM\.sig$`, + `coverage-summary\.json$`, + `createdFiles\.lst$`, + `fullLocaleNames\.lst$`, + `include\.lst$`, + `inputFiles\.lst$`, + `phpunit-code-coverage\.xml$`, + `phpunit-coverage\.xml$`, + `remapInstanbul\.coverage.*\.json$`, + `scoverage\.measurements\..*$`, + `test-result-.*-codecoverage\.json$`, + `test_.*_coverage\.txt$`, + `testrunner-coverage.*$`, + `.*\..*js$`, + `\.yarn$`, + `.*\.zip$`, + } + filePaths := []string{} for _, filePattern := range coverageFileTypes { - fullPattern := globPatternFromPattern(filePattern) + fullPattern := globPatternFromPattern(s.repositoryPath, filePattern) candidates, err := filepathx.Glob(fullPattern) if err != nil { @@ -493,7 +608,23 @@ func coveragePathsInferred() ([]string, error) { filePaths = append(filePaths, candidates...) } - return filePaths, nil + sanitized := []string{} + for _, filepath := range filePaths { + var matched = false + for _, blocklistPattern := range fileBlocklistMatchers { + regex, _ := regexp.Compile(blocklistPattern) + if regex.MatchString(filepath) { + matched = true + break + } + } + + if !matched { + sanitized = append(sanitized, filepath) + } + } + + return sanitized, nil } // xmlPathsFromDir returns a list of all the XML files in the given directory diff --git a/internal/cmd/submit/submit_test.go b/internal/cmd/submit/submit_test.go index 521255eb..dc088aa3 100644 --- a/internal/cmd/submit/submit_test.go +++ b/internal/cmd/submit/submit_test.go @@ -448,7 +448,7 @@ func Test_bundle(t *testing.T) { commitResolver: metadata.NewStaticCommitResolver(&metadata.Commit{TreeSHA: "ccccccccccccccccccccdddddddddddddddddddd"}, log), envs: envs, paths: []string{"testdata/example-reports-dir/example-1.xml"}, - coveragePaths: []string{"testdata/example-reports-dir/coverage-files/report.xml", "testdata/example-reports-dir/coverage-files/report-2.xml"}, + coveragePaths: []string{"testdata/example-reports-dir/coverage/report.xml", "testdata/example-reports-dir/coverage/report-2.xml"}, bucket: "buildpulse-uploads", accountID: 42, repositoryID: 8675309, @@ -477,13 +477,13 @@ func Test_bundle(t *testing.T) { // Verify coverage files are present and contains expected content assertEqualContent(t, - "testdata/example-reports-dir/coverage-files/report.xml", - filepath.Join(unzipDir, "coverage/testdata/example-reports-dir/coverage-files/report.xml"), + "testdata/example-reports-dir/coverage/report.xml", + filepath.Join(unzipDir, "coverage/testdata/example-reports-dir/coverage/report.xml"), ) assertEqualContent(t, - "testdata/example-reports-dir/coverage-files/report-2.xml", - filepath.Join(unzipDir, "coverage/testdata/example-reports-dir/coverage-files/report-2.xml"), + "testdata/example-reports-dir/coverage/report-2.xml", + filepath.Join(unzipDir, "coverage/testdata/example-reports-dir/coverage/report-2.xml"), ) // Verify buildpulse.log is present and contains expected content @@ -533,14 +533,18 @@ func Test_bundle(t *testing.T) { // Verify coverage file is present and contains expected content assertEqualContent(t, - "testdata/example-reports-dir/coverage-files/report.xml", - filepath.Join(unzipDir, "coverage/testdata/example-reports-dir/coverage-files/report.xml"), + "testdata/example-reports-dir/coverage/report.xml", + filepath.Join(unzipDir, "coverage/testdata/example-reports-dir/coverage/report.xml"), ) - ignoredCoverageReportPath := filepath.Join(unzipDir, "coverage/testdata/example-reports-dir/coverage-files/report-2.xml") + ignoredCoverageReportPath := filepath.Join(unzipDir, "coverage/testdata/example-reports-dir/coverage/report-2.xml") _, err = os.Stat(ignoredCoverageReportPath) assert.True(t, os.IsNotExist(err)) + ignoredSourceFilePath := filepath.Join(unzipDir, "coverage/testdata/example-reports-dir/vendor/simplecov/coverage_statistic.go") + _, err = os.Stat(ignoredSourceFilePath) + assert.True(t, os.IsNotExist(err)) + // Verify buildpulse.log is present and contains expected content logdata, err := os.ReadFile(filepath.Join(unzipDir, "buildpulse.log")) require.NoError(t, err) @@ -662,8 +666,8 @@ func Test_xmlPathsFromDir(t *testing.T) { name: "DirectoryWithFilesAtRootAndInSubDirectories", path: "testdata/example-reports-dir", want: []string{ - "testdata/example-reports-dir/coverage-files/report.xml", - "testdata/example-reports-dir/coverage-files/report-2.xml", + "testdata/example-reports-dir/coverage/report.xml", + "testdata/example-reports-dir/coverage/report-2.xml", "testdata/example-reports-dir/example-1.xml", "testdata/example-reports-dir/example-2.XML", "testdata/example-reports-dir/dir-with-xml-files/browserstack/example-1.xml", diff --git a/internal/cmd/submit/testdata/example-reports-dir/coverage-files/report-2.xml b/internal/cmd/submit/testdata/example-reports-dir/coverage/report-2.xml similarity index 100% rename from internal/cmd/submit/testdata/example-reports-dir/coverage-files/report-2.xml rename to internal/cmd/submit/testdata/example-reports-dir/coverage/report-2.xml diff --git a/internal/cmd/submit/testdata/example-reports-dir/coverage-files/report.xml b/internal/cmd/submit/testdata/example-reports-dir/coverage/report.xml similarity index 100% rename from internal/cmd/submit/testdata/example-reports-dir/coverage-files/report.xml rename to internal/cmd/submit/testdata/example-reports-dir/coverage/report.xml diff --git a/internal/cmd/submit/testdata/example-reports-dir/vendor/simplecov/coverage_statistic.go b/internal/cmd/submit/testdata/example-reports-dir/vendor/simplecov/coverage_statistic.go new file mode 100644 index 00000000..a3dd973f --- /dev/null +++ b/internal/cmd/submit/testdata/example-reports-dir/vendor/simplecov/coverage_statistic.go @@ -0,0 +1,7 @@ +package main + +import "fmt" + +func main() { + fmt.Println("Hello, World!") +}