From 4be4b03279e6be0e8eaa5ebce4cc7ed641bbdb39 Mon Sep 17 00:00:00 2001 From: Xueqin Cui <72771658+cuixq@users.noreply.github.com> Date: Wed, 12 Feb 2025 15:49:11 +1100 Subject: [PATCH] Revert "chore: Revert "refactor: use pomxmlnet extractor from scalibr" (#1613)" This reverts commit f9bcb9ce60d83955c1aee7d1131584655fe1a117. --- .../licensematcher/licensematcher.go | 2 +- .../language/java/pomxmlnet/extractor.go | 190 --------- .../language/java/pomxmlnet/extractor_test.go | 367 ------------------ .../java/pomxmlnet/testdata/maven/empty.xml | 7 - .../testdata/maven/interpolation.xml | 37 -- .../testdata/maven/invalid-syntax.xml | 13 - .../java/pomxmlnet/testdata/maven/not-pom.txt | 1 - .../pomxmlnet/testdata/maven/one-package.xml | 17 - .../pomxmlnet/testdata/maven/parent/pom.xml | 21 - .../pomxmlnet/testdata/maven/transitive.xml | 33 -- .../pomxmlnet/testdata/maven/two-packages.xml | 22 -- .../maven/with-dependency-management.xml | 37 -- .../pomxmlnet/testdata/maven/with-parent.xml | 54 --- .../pomxmlnet/testdata/maven/with-scope.xml | 14 - .../testdata/universe/basic-universe.yaml | 60 --- .../internal/scanners/extractorbuilder.go | 10 +- pkg/osvscanner/internal/scanners/lockfile.go | 68 ++-- .../internal/scanners/lockfile_test.go | 9 +- pkg/osvscanner/osvscanner.go | 12 +- 19 files changed, 52 insertions(+), 922 deletions(-) delete mode 100644 internal/scalibrextract/language/java/pomxmlnet/extractor.go delete mode 100644 internal/scalibrextract/language/java/pomxmlnet/extractor_test.go delete mode 100644 internal/scalibrextract/language/java/pomxmlnet/testdata/maven/empty.xml delete mode 100644 internal/scalibrextract/language/java/pomxmlnet/testdata/maven/interpolation.xml delete mode 100644 internal/scalibrextract/language/java/pomxmlnet/testdata/maven/invalid-syntax.xml delete mode 100644 internal/scalibrextract/language/java/pomxmlnet/testdata/maven/not-pom.txt delete mode 100644 internal/scalibrextract/language/java/pomxmlnet/testdata/maven/one-package.xml delete mode 100644 internal/scalibrextract/language/java/pomxmlnet/testdata/maven/parent/pom.xml delete mode 100644 internal/scalibrextract/language/java/pomxmlnet/testdata/maven/transitive.xml delete mode 100644 internal/scalibrextract/language/java/pomxmlnet/testdata/maven/two-packages.xml delete mode 100644 internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-dependency-management.xml delete mode 100644 internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-parent.xml delete mode 100644 internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-scope.xml delete mode 100644 internal/scalibrextract/language/java/pomxmlnet/testdata/universe/basic-universe.yaml diff --git a/internal/clients/clientimpl/licensematcher/licensematcher.go b/internal/clients/clientimpl/licensematcher/licensematcher.go index 2ade202696a..a59a8632877 100644 --- a/internal/clients/clientimpl/licensematcher/licensematcher.go +++ b/internal/clients/clientimpl/licensematcher/licensematcher.go @@ -4,7 +4,7 @@ import ( "context" depsdevpb "deps.dev/api/v3" - "github.com/google/osv-scanner/v2/internal/datasource" + "github.com/google/osv-scalibr/clients/datasource" "github.com/google/osv-scanner/v2/internal/depsdev" "github.com/google/osv-scanner/v2/internal/imodels" "github.com/google/osv-scanner/v2/pkg/models" diff --git a/internal/scalibrextract/language/java/pomxmlnet/extractor.go b/internal/scalibrextract/language/java/pomxmlnet/extractor.go deleted file mode 100644 index 4e1224e772d..00000000000 --- a/internal/scalibrextract/language/java/pomxmlnet/extractor.go +++ /dev/null @@ -1,190 +0,0 @@ -// Package pomxmlnet extracts Maven's pom.xml format with transitive dependency resolution. -package pomxmlnet - -import ( - "context" - "fmt" - "path/filepath" - - "golang.org/x/exp/maps" - - mavenresolve "deps.dev/util/resolve/maven" - mavenutil "github.com/google/osv-scanner/v2/internal/utility/maven" - - "deps.dev/util/maven" - "deps.dev/util/resolve" - "deps.dev/util/resolve/dep" - "github.com/google/osv-scalibr/extractor" - "github.com/google/osv-scalibr/extractor/filesystem" - "github.com/google/osv-scalibr/extractor/filesystem/osv" - "github.com/google/osv-scalibr/plugin" - "github.com/google/osv-scalibr/purl" - "github.com/google/osv-scanner/v2/internal/datasource" - "github.com/google/osv-scanner/v2/internal/resolution/client" -) - -// Extractor extracts Maven packages with transitive dependency resolution. -// TODO: Use the virtual filesystem rather than the real filesystem. -type Extractor struct { - client.DependencyClient - *datasource.MavenRegistryAPIClient -} - -// Name of the extractor. -func (e Extractor) Name() string { return "java/pomxml" } - -// Version of the extractor. -func (e Extractor) Version() int { return 0 } - -// Requirements of the extractor. -func (e Extractor) Requirements() *plugin.Capabilities { - return &plugin.Capabilities{ - Network: true, - } -} - -// FileRequired never returns true, as this is for the osv-scanner json output. -func (e Extractor) FileRequired(fapi filesystem.FileAPI) bool { - return filepath.Base(fapi.Path()) == "pom.xml" -} - -// Extract extracts packages from yarn.lock files passed through the scan input. -func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]*extractor.Inventory, error) { - var project maven.Project - if err := datasource.NewMavenDecoder(input.Reader).Decode(&project); err != nil { - return nil, fmt.Errorf("could not extract from %s: %w", input.Path, err) - } - // Empty JDK and ActivationOS indicates merging the default profiles. - if err := project.MergeProfiles("", maven.ActivationOS{}); err != nil { - return nil, fmt.Errorf("failed to merge profiles: %w", err) - } - for _, repo := range project.Repositories { - if err := e.MavenRegistryAPIClient.AddRegistry(datasource.MavenRegistry{ - URL: string(repo.URL), - ID: string(repo.ID), - ReleasesEnabled: repo.Releases.Enabled.Boolean(), - SnapshotsEnabled: repo.Snapshots.Enabled.Boolean(), - }); err != nil { - return nil, fmt.Errorf("failed to add registry %s: %w", repo.URL, err) - } - } - // Merging parents data by parsing local parent pom.xml or fetching from upstream. - if err := mavenutil.MergeParents(ctx, e.MavenRegistryAPIClient, &project, project.Parent, 1, filepath.Join(input.Root, input.Path), true); err != nil { - return nil, fmt.Errorf("failed to merge parents: %w", err) - } - // Process the dependencies: - // - dedupe dependencies and dependency management - // - import dependency management - // - fill in missing dependency version requirement - project.ProcessDependencies(func(groupID, artifactID, version maven.String) (maven.DependencyManagement, error) { - return mavenutil.GetDependencyManagement(ctx, e.MavenRegistryAPIClient, groupID, artifactID, version) - }) - - if registries := e.MavenRegistryAPIClient.GetRegistries(); len(registries) > 0 { - clientRegs := make([]client.Registry, len(registries)) - for i, reg := range registries { - clientRegs[i] = reg - } - if err := e.DependencyClient.AddRegistries(clientRegs); err != nil { - return nil, err - } - } - - overrideClient := client.NewOverrideClient(e.DependencyClient) - resolver := mavenresolve.NewResolver(overrideClient) - - // Resolve the dependencies. - root := resolve.Version{ - VersionKey: resolve.VersionKey{ - PackageKey: resolve.PackageKey{ - System: resolve.Maven, - Name: project.ProjectKey.Name(), - }, - VersionType: resolve.Concrete, - Version: string(project.Version), - }} - reqs := make([]resolve.RequirementVersion, len(project.Dependencies)+len(project.DependencyManagement.Dependencies)) - for i, d := range project.Dependencies { - reqs[i] = resolve.RequirementVersion{ - VersionKey: resolve.VersionKey{ - PackageKey: resolve.PackageKey{ - System: resolve.Maven, - Name: d.Name(), - }, - VersionType: resolve.Requirement, - Version: string(d.Version), - }, - Type: resolve.MavenDepType(d, ""), - } - } - for i, d := range project.DependencyManagement.Dependencies { - reqs[len(project.Dependencies)+i] = resolve.RequirementVersion{ - VersionKey: resolve.VersionKey{ - PackageKey: resolve.PackageKey{ - System: resolve.Maven, - Name: d.Name(), - }, - VersionType: resolve.Requirement, - Version: string(d.Version), - }, - Type: resolve.MavenDepType(d, mavenutil.OriginManagement), - } - } - overrideClient.AddVersion(root, reqs) - - client.PreFetch(ctx, overrideClient, reqs, filepath.Join(input.Root, input.Path)) - g, err := resolver.Resolve(ctx, root.VersionKey) - if err != nil { - return nil, fmt.Errorf("failed resolving %v: %w", root, err) - } - for i, e := range g.Edges { - e.Type = dep.Type{} - g.Edges[i] = e - } - - details := map[string]*extractor.Inventory{} - for i := 1; i < len(g.Nodes); i++ { - // Ignore the first node which is the root. - node := g.Nodes[i] - depGroups := []string{} - inventory := extractor.Inventory{ - Name: node.Version.Name, - Version: node.Version.Version, - // TODO(rexpan): Add merged paths in here as well - Locations: []string{input.Path}, - } - // We are only able to know dependency groups of direct dependencies but - // not transitive dependencies because the nodes in the resolve graph does - // not have the scope information. - for _, dep := range project.Dependencies { - if dep.Name() != inventory.Name { - continue - } - if dep.Scope != "" && dep.Scope != "compile" { - depGroups = append(depGroups, string(dep.Scope)) - } - } - inventory.Metadata = osv.DepGroupMetadata{ - DepGroupVals: depGroups, - } - details[inventory.Name] = &inventory - } - - return maps.Values(details), nil -} - -// ToPURL converts an inventory created by this extractor into a PURL. -func (e Extractor) ToPURL(i *extractor.Inventory) *purl.PackageURL { - return &purl.PackageURL{ - Type: purl.TypeMaven, - Name: i.Name, - Version: i.Version, - } -} - -// Ecosystem returns the OSV ecosystem ('npm') of the software extracted by this extractor. -func (e Extractor) Ecosystem(_ *extractor.Inventory) string { - return "Maven" -} - -var _ filesystem.Extractor = Extractor{} diff --git a/internal/scalibrextract/language/java/pomxmlnet/extractor_test.go b/internal/scalibrextract/language/java/pomxmlnet/extractor_test.go deleted file mode 100644 index 8c1e7c4c51e..00000000000 --- a/internal/scalibrextract/language/java/pomxmlnet/extractor_test.go +++ /dev/null @@ -1,367 +0,0 @@ -package pomxmlnet_test - -import ( - "context" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/google/osv-scalibr/extractor" - "github.com/google/osv-scalibr/extractor/filesystem/osv" - "github.com/google/osv-scalibr/extractor/filesystem/simplefileapi" - "github.com/google/osv-scalibr/testing/extracttest" - "github.com/google/osv-scanner/v2/internal/datasource" - "github.com/google/osv-scanner/v2/internal/resolution/clienttest" - "github.com/google/osv-scanner/v2/internal/scalibrextract/language/java/pomxmlnet" - "github.com/google/osv-scanner/v2/internal/testutility" -) - -func TestMavenResolverExtractor_FileRequired(t *testing.T) { - t.Parallel() - - tests := []struct { - path string - want bool - }{ - { - path: "", - want: false, - }, - { - path: "pom.xml", - want: true, - }, - { - path: "path/to/my/pom.xml", - want: true, - }, - { - path: "path/to/my/pom.xml/file", - want: false, - }, - { - path: "path/to/my/pom.xml.file", - want: false, - }, - { - path: "path.to.my.pom.xml", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.path, func(t *testing.T) { - t.Parallel() - e := pomxmlnet.Extractor{} - got := e.FileRequired(simplefileapi.New(tt.path, nil)) - if got != tt.want { - t.Errorf("Extract() got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestExtractor_Extract(t *testing.T) { - t.Parallel() - - tests := []extracttest.TestTableEntry{ - { - Name: "Not a pom file", - InputConfig: extracttest.ScanInputMockConfig{ - Path: "testdata/maven/not-pom.txt", - }, - WantErr: extracttest.ContainsErrStr{Str: "could not extract from"}, - }, - { - Name: "invalid xml syntax", - InputConfig: extracttest.ScanInputMockConfig{ - Path: "testdata/maven/invalid-syntax.xml", - }, - WantErr: extracttest.ContainsErrStr{Str: "XML syntax error"}, - }, - { - Name: "empty", - InputConfig: extracttest.ScanInputMockConfig{ - Path: "testdata/maven/empty.xml", - }, - WantInventory: []*extractor.Inventory{}, - }, - { - Name: "one package", - InputConfig: extracttest.ScanInputMockConfig{ - Path: "testdata/maven/one-package.xml", - }, - WantInventory: []*extractor.Inventory{ - { - Name: "org.apache.maven:maven-artifact", - Version: "1.0.0", - Locations: []string{"testdata/maven/one-package.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - }, - }, - { - Name: "two packages", - InputConfig: extracttest.ScanInputMockConfig{ - Path: "testdata/maven/two-packages.xml", - }, - WantInventory: []*extractor.Inventory{ - { - Name: "io.netty:netty-all", - Version: "4.1.42.Final", - Locations: []string{"testdata/maven/two-packages.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - { - Name: "org.slf4j:slf4j-log4j12", - Version: "1.7.25", - Locations: []string{"testdata/maven/two-packages.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - }, - }, - { - Name: "with dependency management", - InputConfig: extracttest.ScanInputMockConfig{ - Path: "testdata/maven/with-dependency-management.xml", - }, - WantInventory: []*extractor.Inventory{ - { - Name: "io.netty:netty-all", - Version: "4.1.9", - Locations: []string{"testdata/maven/with-dependency-management.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - { - Name: "org.slf4j:slf4j-log4j12", - Version: "1.7.25", - Locations: []string{"testdata/maven/with-dependency-management.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - }, - }, - { - Name: "interpolation", - InputConfig: extracttest.ScanInputMockConfig{ - Path: "testdata/maven/interpolation.xml", - }, - WantInventory: []*extractor.Inventory{ - { - Name: "org.mine:mypackage", - Version: "1.0.0", - Locations: []string{"testdata/maven/interpolation.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - { - Name: "org.mine:my.package", - Version: "2.3.4", - Locations: []string{"testdata/maven/interpolation.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - { - Name: "org.mine:ranged-package", - Version: "9.4.37", - Locations: []string{"testdata/maven/interpolation.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - }, - }, - { - Name: "with scope / dep groups", - InputConfig: extracttest.ScanInputMockConfig{ - Path: "testdata/maven/with-scope.xml", - }, - WantInventory: []*extractor.Inventory{ - { - Name: "junit:junit", - Version: "4.12", - Locations: []string{"testdata/maven/with-scope.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{"runtime"}}, - }, - }, - }, - { - Name: "transitive dependencies", - InputConfig: extracttest.ScanInputMockConfig{ - Path: "testdata/maven/transitive.xml", - }, - WantInventory: []*extractor.Inventory{ - { - Name: "org.direct:alice", - Version: "1.0.0", - Locations: []string{"testdata/maven/transitive.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - { - Name: "org.direct:bob", - Version: "2.0.0", - Locations: []string{"testdata/maven/transitive.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - { - Name: "org.direct:chris", - Version: "3.0.0", - Locations: []string{"testdata/maven/transitive.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - { - Name: "org.transitive:chuck", - Version: "1.1.1", - Locations: []string{"testdata/maven/transitive.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - { - Name: "org.transitive:dave", - Version: "2.2.2", - Locations: []string{"testdata/maven/transitive.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - { - Name: "org.transitive:eve", - Version: "3.3.3", - Locations: []string{"testdata/maven/transitive.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - { - Name: "org.transitive:frank", - Version: "4.4.4", - Locations: []string{"testdata/maven/transitive.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.Name, func(t *testing.T) { - t.Parallel() - - resolutionClient := clienttest.NewMockResolutionClient(t, "testdata/universe/basic-universe.yaml") - extr := pomxmlnet.Extractor{ - DependencyClient: resolutionClient, - MavenRegistryAPIClient: &datasource.MavenRegistryAPIClient{}, - } - - scanInput := extracttest.GenerateScanInputMock(t, tt.InputConfig) - defer extracttest.CloseTestScanInput(t, scanInput) - - got, err := extr.Extract(context.Background(), &scanInput) - - if diff := cmp.Diff(tt.WantErr, err, cmpopts.EquateErrors()); diff != "" { - t.Errorf("%s.Extract(%q) error diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) - return - } - - if diff := cmp.Diff(tt.WantInventory, got, cmpopts.SortSlices(extracttest.InventoryCmpLess)); diff != "" { - t.Errorf("%s.Extract(%q) diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) - } - }) - } -} - -func TestExtractor_Extract_WithMockServer(t *testing.T) { - t.Parallel() - - tt := extracttest.TestTableEntry{ - // Name: "with parent", - InputConfig: extracttest.ScanInputMockConfig{ - Path: "testdata/maven/with-parent.xml", - }, - WantInventory: []*extractor.Inventory{ - { - Name: "org.alice:alice", - Version: "1.0.0", - Locations: []string{"testdata/maven/with-parent.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - { - Name: "org.bob:bob", - Version: "2.0.0", - Locations: []string{"testdata/maven/with-parent.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - { - Name: "org.chuck:chuck", - Version: "3.0.0", - Locations: []string{"testdata/maven/with-parent.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - { - Name: "org.dave:dave", - Version: "4.0.0", - Locations: []string{"testdata/maven/with-parent.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - { - Name: "org.eve:eve", - Version: "5.0.0", - Locations: []string{"testdata/maven/with-parent.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - { - Name: "org.frank:frank", - Version: "6.0.0", - Locations: []string{"testdata/maven/with-parent.xml"}, - Metadata: osv.DepGroupMetadata{DepGroupVals: []string{}}, - }, - }, - } - - srv := testutility.NewMockHTTPServer(t) - srv.SetResponse(t, "org/upstream/parent-pom/1.0/parent-pom-1.0.pom", []byte(` - - org.upstream - parent-pom - 1.0 - pom - - - org.eve - eve - 5.0.0 - - - - `)) - srv.SetResponse(t, "org/import/import/1.2.3/import-1.2.3.pom", []byte(` - - org.import - import - 1.2.3 - pom - - - - org.frank - frank - 6.0.0 - - - - - `)) - - apiClient, err := datasource.NewMavenRegistryAPIClient(datasource.MavenRegistry{URL: srv.URL, ReleasesEnabled: true}) - if err != nil { - t.Fatalf("%v", err) - } - - resolutionClient := clienttest.NewMockResolutionClient(t, "testdata/universe/basic-universe.yaml") - extr := pomxmlnet.Extractor{ - DependencyClient: resolutionClient, - MavenRegistryAPIClient: apiClient, - } - - scanInput := extracttest.GenerateScanInputMock(t, tt.InputConfig) - defer extracttest.CloseTestScanInput(t, scanInput) - - got, err := extr.Extract(context.Background(), &scanInput) - - if diff := cmp.Diff(tt.WantErr, err, cmpopts.EquateErrors()); diff != "" { - t.Errorf("%s.Extract(%q) error diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) - return - } - - if diff := cmp.Diff(tt.WantInventory, got, cmpopts.SortSlices(extracttest.InventoryCmpLess)); diff != "" { - t.Errorf("%s.Extract(%q) diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) - } -} diff --git a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/empty.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/empty.xml deleted file mode 100644 index 8cfeebaaa4d..00000000000 --- a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/empty.xml +++ /dev/null @@ -1,7 +0,0 @@ - - 4.0.0 - - com.mycompany.app - my-app - 1 - diff --git a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/interpolation.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/interpolation.xml deleted file mode 100644 index 6b7f761afc6..00000000000 --- a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/interpolation.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - 4.0.0 - - io.library - my-library - 1.0-SNAPSHOT - jar - - - 1.0.0 - 2.3.4 - [9.4.35.v20201120,9.5) - - - - - org.mine - mypackage - ${mypackageVersion} - - - - org.mine - my.package - ${my.package.version} - - - - org.mine - ranged-package - ${version-range} - - - - diff --git a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/invalid-syntax.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/invalid-syntax.xml deleted file mode 100644 index 761a32c1abb..00000000000 --- a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/invalid-syntax.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - <${Id}.version>${project.version} - - - - - io.netty - netty-all - 4.1.42.Final - - - diff --git a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/not-pom.txt b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/not-pom.txt deleted file mode 100644 index f9df712bcb2..00000000000 --- a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/not-pom.txt +++ /dev/null @@ -1 +0,0 @@ -this is not a pom.xml file! diff --git a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/one-package.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/one-package.xml deleted file mode 100644 index bbb1359e9d5..00000000000 --- a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/one-package.xml +++ /dev/null @@ -1,17 +0,0 @@ - - com.mycompany.app - my-app - 1.0 - - - 3.0 - - - - - org.apache.maven - maven-artifact - 1.0.0 - - - diff --git a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/parent/pom.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/parent/pom.xml deleted file mode 100644 index 3751df6be32..00000000000 --- a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/parent/pom.xml +++ /dev/null @@ -1,21 +0,0 @@ - - org.local - parent-pom - 1.0 - - pom - - - org.upstream - parent-pom - 1.0 - - - - - org.dave - dave - 4.0.0 - - - diff --git a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/transitive.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/transitive.xml deleted file mode 100644 index 52e416a0bcd..00000000000 --- a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/transitive.xml +++ /dev/null @@ -1,33 +0,0 @@ - - com.mycompany.app - my-app - 1.0 - - - - - org.transitive - frank - 4.4.4 - - - - - - - org.direct - alice - 1.0.0 - - - org.direct - bob - 2.0.0 - - - org.direct - chris - 3.0.0 - - - diff --git a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/two-packages.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/two-packages.xml deleted file mode 100644 index 897f648a1e4..00000000000 --- a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/two-packages.xml +++ /dev/null @@ -1,22 +0,0 @@ - - com.mycompany.app - my-app - 1.0 - - - 3.0 - - - - - io.netty - netty-all - 4.1.42.Final - - - org.slf4j - slf4j-log4j12 - 1.7.25 - - - diff --git a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-dependency-management.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-dependency-management.xml deleted file mode 100644 index 1928688e949..00000000000 --- a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-dependency-management.xml +++ /dev/null @@ -1,37 +0,0 @@ - - com.mycompany.app - my-app - 1.0 - - - 3.0 - - - - - io.netty - netty-all - 4.1.9 - - - org.slf4j - slf4j-log4j12 - 1.7.25 - - - - - - - io.netty - netty-all - 4.1.42.Final - - - com.google.code.findbugs - jsr305 - 3.0.2 - - - - diff --git a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-parent.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-parent.xml deleted file mode 100644 index 602b8b877f1..00000000000 --- a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-parent.xml +++ /dev/null @@ -1,54 +0,0 @@ - - com.mycompany.app - my-app - 1.0 - - - org.local - parent-pom - 1.0 - ./parent/pom.xml - - - - 2.0.0 - - - - - org.alice - alice - 1.0.0 - - - org.bob - bob - ${bob.version} - - - org.chuck - chuck - - - org.frank - frank - - - - - - - org.chuck - chuck - 3.0.0 - - - org.import - import - 1.2.3 - pom - import - - - - diff --git a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-scope.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-scope.xml deleted file mode 100644 index 688c6bb7bc2..00000000000 --- a/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-scope.xml +++ /dev/null @@ -1,14 +0,0 @@ - - com.mycompany.app - my-app - 1.0 - - - - junit - junit - 4.12 - runtime - - - diff --git a/internal/scalibrextract/language/java/pomxmlnet/testdata/universe/basic-universe.yaml b/internal/scalibrextract/language/java/pomxmlnet/testdata/universe/basic-universe.yaml deleted file mode 100644 index 2bf2b32724a..00000000000 --- a/internal/scalibrextract/language/java/pomxmlnet/testdata/universe/basic-universe.yaml +++ /dev/null @@ -1,60 +0,0 @@ -system: maven -schema: | - com.google.code.findbugs:jsr305 - 3.0.2 - io.netty:netty-all - 4.1.9 - 4.1.42.Final - junit:junit - 4.12 - org.alice:alice - 1.0.0 - org.apache.maven:maven-artifact - 1.0.0 - org.bob:bob - 2.0.0 - org.chuck:chuck - 3.0.0 - org.dave:dave - 4.0.0 - org.direct:alice - 1.0.0 - org.transitive:chuck@1.1.1 - org.transitive:dave@2.2.2 - org.direct:bob - 2.0.0 - org.transitive:eve@3.3.3 - org.direct:chris - 3.0.0 - org.transitive:frank@3.3.3 - org.eve:eve - 5.0.0 - org.frank:frank - 6.0.0 - org.mine:my.package - 2.3.4 - org.mine:mypackage - 1.0.0 - org.mine:ranged-package - 9.4.35 - 9.4.36 - 9.4.37 - 9.5 - org.slf4j:slf4j-log4j12 - 1.7.25 - org.transitive:chuck - 1.1.1 - 2.2.2 - org.transitive:eve@2.2.2 - 3.3.3 - org.transitive:dave - 1.1.1 - 2.2.2 - 3.3.3 - org.transitive:eve - 1.1.1 - 2.2.2 - 3.3.3 - org.transitive:frank - 3.3.3 - 4.4.4 diff --git a/pkg/osvscanner/internal/scanners/extractorbuilder.go b/pkg/osvscanner/internal/scanners/extractorbuilder.go index 8217138a77a..8786fa40574 100644 --- a/pkg/osvscanner/internal/scanners/extractorbuilder.go +++ b/pkg/osvscanner/internal/scanners/extractorbuilder.go @@ -1,6 +1,8 @@ package scanners import ( + "github.com/google/osv-scalibr/clients/datasource" + "github.com/google/osv-scalibr/clients/resolution" "github.com/google/osv-scalibr/extractor/filesystem" "github.com/google/osv-scalibr/extractor/filesystem/language/cpp/conanlock" "github.com/google/osv-scalibr/extractor/filesystem/language/dart/pubspec" @@ -14,6 +16,7 @@ import ( "github.com/google/osv-scalibr/extractor/filesystem/language/java/gradlelockfile" "github.com/google/osv-scalibr/extractor/filesystem/language/java/gradleverificationmetadataxml" "github.com/google/osv-scalibr/extractor/filesystem/language/java/pomxml" + "github.com/google/osv-scalibr/extractor/filesystem/language/java/pomxmlnet" "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/bunlock" "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/packagelockjson" "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/pnpmlock" @@ -32,11 +35,8 @@ import ( "github.com/google/osv-scalibr/extractor/filesystem/os/dpkg" "github.com/google/osv-scalibr/extractor/filesystem/sbom/cdx" "github.com/google/osv-scalibr/extractor/filesystem/sbom/spdx" - "github.com/google/osv-scanner/v2/internal/datasource" "github.com/google/osv-scanner/v2/internal/osvdev" - "github.com/google/osv-scanner/v2/internal/resolution/client" "github.com/google/osv-scanner/v2/internal/scalibrextract/filesystem/vendored" - "github.com/google/osv-scanner/v2/internal/scalibrextract/language/java/pomxmlnet" "github.com/google/osv-scanner/v2/internal/scalibrextract/language/javascript/nodemodules" "github.com/google/osv-scanner/v2/internal/scalibrextract/vcs/gitrepo" "github.com/ossf/osv-schema/bindings/go/osvschema" @@ -103,7 +103,7 @@ var lockfileExtractors = []filesystem.Extractor{ // BuildLockfileExtractors returns all relevant extractors for lockfile scanning given the required clients // All clients can be nil, and if nil the extractors requiring those clients will not be returned. -func BuildLockfileExtractors(dependencyClients map[osvschema.Ecosystem]client.DependencyClient, mavenAPIClient *datasource.MavenRegistryAPIClient) []filesystem.Extractor { +func BuildLockfileExtractors(dependencyClients map[osvschema.Ecosystem]resolution.DependencyClient, mavenAPIClient *datasource.MavenRegistryAPIClient) []filesystem.Extractor { extractorsToUse := lockfileExtractors if dependencyClients[osvschema.EcosystemMaven] != nil && mavenAPIClient != nil { @@ -128,7 +128,7 @@ func BuildSBOMExtractors() []filesystem.Extractor { func BuildWalkerExtractors( includeRootGit bool, osvdevClient *osvdev.OSVClient, - dependencyClients map[osvschema.Ecosystem]client.DependencyClient, + dependencyClients map[osvschema.Ecosystem]resolution.DependencyClient, mavenAPIClient *datasource.MavenRegistryAPIClient) []filesystem.Extractor { relevantExtractors := []filesystem.Extractor{} diff --git a/pkg/osvscanner/internal/scanners/lockfile.go b/pkg/osvscanner/internal/scanners/lockfile.go index 9884443675c..e7e8426b1d7 100644 --- a/pkg/osvscanner/internal/scanners/lockfile.go +++ b/pkg/osvscanner/internal/scanners/lockfile.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "path/filepath" + "slices" "strings" "github.com/google/osv-scalibr/extractor" @@ -16,34 +17,32 @@ import ( "github.com/google/osv-scanner/v2/pkg/reporter" ) -var lockfileExtractorMapping = map[string]string{ - "pubspec.lock": "dart/pubspec", - "pnpm-lock.yaml": "javascript/pnpmlock", - "yarn.lock": "javascript/yarnlock", - "package-lock.json": "javascript/packagelockjson", - // This translation works for both the transitive scanning and non transitive scanning - // As both extractors have the same name - "pom.xml": "java/pomxml", - "buildscript-gradle.lockfile": "java/gradlelockfile", - "gradle.lockfile": "java/gradlelockfile", - "verification-metadata.xml": "java/gradleverificationmetadataxml", - "poetry.lock": "python/poetrylock", - "Pipfile.lock": "python/Pipfilelock", - "pdm.lock": "python/pdmlock", - "requirements.txt": "python/requirements", - "uv.lock": "python/uvlock", - "Cargo.lock": "rust/Cargolock", - "composer.lock": "php/composerlock", - "mix.lock": "erlang/mixlock", - "renv.lock": "r/renvlock", - "deps.json": "dotnet/depsjson", - "packages.lock.json": "dotnet/packageslockjson", - "conan.lock": "cpp/conanlock", - "go.mod": "go/gomod", - "bun.lock": "javascript/bunlock", - "Gemfile.lock": "ruby/gemfilelock", - "cabal.project.freeze": "haskell/cabal", - "stack.yaml.lock": "haskell/stacklock", +var lockfileExtractorMapping = map[string][]string{ + "pubspec.lock": {"dart/pubspec"}, + "pnpm-lock.yaml": {"javascript/pnpmlock"}, + "yarn.lock": {"javascript/yarnlock"}, + "package-lock.json": {"javascript/packagelockjson"}, + "pom.xml": {"java/pomxmlnet", "java/pomxml"}, + "buildscript-gradle.lockfile": {"java/gradlelockfile"}, + "gradle.lockfile": {"java/gradlelockfile"}, + "verification-metadata.xml": {"java/gradleverificationmetadataxml"}, + "poetry.lock": {"python/poetrylock"}, + "Pipfile.lock": {"python/Pipfilelock"}, + "pdm.lock": {"python/pdmlock"}, + "requirements.txt": {"python/requirements"}, + "uv.lock": {"python/uvlock"}, + "Cargo.lock": {"rust/Cargolock"}, + "composer.lock": {"php/composerlock"}, + "mix.lock": {"erlang/mixlock"}, + "renv.lock": {"r/renvlock"}, + "deps.json": {"dotnet/depsjson"}, + "packages.lock.json": {"dotnet/packageslockjson"}, + "conan.lock": {"cpp/conanlock"}, + "go.mod": {"go/gomod"}, + "bun.lock": {"javascript/bunlock"}, + "Gemfile.lock": {"ruby/gemfilelock"}, + "cabal.project.freeze": {"haskell/cabal"}, + "stack.yaml.lock": {"haskell/stacklock"}, // "Package.resolved": "swift/packageresolved", } @@ -103,13 +102,14 @@ func ScanSingleFileWithMapping(r reporter.Reporter, scanPath string, extractorsT inventories, err = scalibrextract.ExtractWithExtractors(context.Background(), path, extractorsToUse) default: // A specific parseAs without a special case is selected // Find and extract with the extractor of parseAs - if name, ok := lockfileExtractorMapping[parseAs]; ok { - for _, ext := range extractorsToUse { - if name == ext.Name() { - inventories, err = scalibrextract.ExtractWithExtractor(context.Background(), path, ext) - break - } + if names, ok := lockfileExtractorMapping[parseAs]; ok && len(names) > 0 { + i := slices.IndexFunc(extractorsToUse, func(ext filesystem.Extractor) bool { + return slices.Contains(names, ext.Name()) + }) + if i < 0 { + return nil, fmt.Errorf("could not determine extractor, requested %s", parseAs) } + inventories, err = scalibrextract.ExtractWithExtractor(context.Background(), path, extractorsToUse[i]) } else { return nil, fmt.Errorf("could not determine extractor, requested %s", parseAs) } diff --git a/pkg/osvscanner/internal/scanners/lockfile_test.go b/pkg/osvscanner/internal/scanners/lockfile_test.go index c80769ecbf4..2169cdba0af 100644 --- a/pkg/osvscanner/internal/scanners/lockfile_test.go +++ b/pkg/osvscanner/internal/scanners/lockfile_test.go @@ -1,6 +1,9 @@ package scanners -import "testing" +import ( + "slices" + "testing" +) func TestLockfileScalibrMappingExists(t *testing.T) { t.Parallel() @@ -10,8 +13,8 @@ func TestLockfileScalibrMappingExists(t *testing.T) { // and not present in lockfileExtractors for _, target := range lockfileExtractors { found := false - for _, val := range lockfileExtractorMapping { - if target.Name() == val { + for _, names := range lockfileExtractorMapping { + if slices.Contains(names, target.Name()) { found = true break } diff --git a/pkg/osvscanner/osvscanner.go b/pkg/osvscanner/osvscanner.go index 4fdd48d867a..d415b996e88 100644 --- a/pkg/osvscanner/osvscanner.go +++ b/pkg/osvscanner/osvscanner.go @@ -11,6 +11,8 @@ import ( scalibr "github.com/google/osv-scalibr" "github.com/google/osv-scalibr/artifact/image/layerscanning/image" + "github.com/google/osv-scalibr/clients/datasource" + "github.com/google/osv-scalibr/clients/resolution" "github.com/google/osv-scalibr/extractor" "github.com/google/osv-scanner/v2/internal/clients/clientimpl/baseimagematcher" "github.com/google/osv-scanner/v2/internal/clients/clientimpl/licensematcher" @@ -18,13 +20,11 @@ import ( "github.com/google/osv-scanner/v2/internal/clients/clientimpl/osvmatcher" "github.com/google/osv-scanner/v2/internal/clients/clientinterfaces" "github.com/google/osv-scanner/v2/internal/config" - "github.com/google/osv-scanner/v2/internal/datasource" "github.com/google/osv-scanner/v2/internal/depsdev" "github.com/google/osv-scanner/v2/internal/imodels" "github.com/google/osv-scanner/v2/internal/imodels/results" "github.com/google/osv-scanner/v2/internal/osvdev" "github.com/google/osv-scanner/v2/internal/output" - "github.com/google/osv-scanner/v2/internal/resolution/client" "github.com/google/osv-scanner/v2/internal/version" "github.com/google/osv-scanner/v2/pkg/models" "github.com/google/osv-scanner/v2/pkg/osvscanner/internal/imagehelpers" @@ -80,7 +80,7 @@ type ExternalAccessors struct { // DependencyClients is a map of implementations of DependencyClient // for each ecosystem, the following is currently implemented: // - [osvschema.EcosystemMaven] required for pomxmlnet Extractor - DependencyClients map[osvschema.Ecosystem]client.DependencyClient + DependencyClients map[osvschema.Ecosystem]resolution.DependencyClient } // ErrNoPackagesFound for when no packages are found during a scan. @@ -96,7 +96,7 @@ var ErrAPIFailed = errors.New("API query failed") func initializeExternalAccessors(r reporter.Reporter, actions ScannerActions) (ExternalAccessors, error) { externalAccessors := ExternalAccessors{ - DependencyClients: map[osvschema.Ecosystem]client.DependencyClient{}, + DependencyClients: map[osvschema.Ecosystem]resolution.DependencyClient{}, } var err error @@ -161,9 +161,9 @@ func initializeExternalAccessors(r reporter.Reporter, actions ScannerActions) (E } if !actions.TransitiveScanningActions.NativeDataSource { - externalAccessors.DependencyClients[osvschema.EcosystemMaven], err = client.NewDepsDevClient(depsdev.DepsdevAPI, "osv-scanner_scan/"+version.OSVVersion) + externalAccessors.DependencyClients[osvschema.EcosystemMaven], err = resolution.NewDepsDevClient(depsdev.DepsdevAPI, "osv-scanner_scan/"+version.OSVVersion) } else { - externalAccessors.DependencyClients[osvschema.EcosystemMaven], err = client.NewMavenRegistryClient(actions.TransitiveScanningActions.MavenRegistry) + externalAccessors.DependencyClients[osvschema.EcosystemMaven], err = resolution.NewMavenRegistryClient(actions.TransitiveScanningActions.MavenRegistry) } if err != nil {