Guided Remediation: Add manifest resolution (#757)

Starting to make guided remediation public #352 🎉  

This PR has the code used to resolve the dependency graph of a manifest
(i.e. a `package.json`) using the deps.dev resolvers and find
vulnerabilities within it.

Doesn't include the code to actually parse/write `package.json` files -
will probably add that in the next PR.

Much of this has been reviewed internally already, but I've made some
significant changes/refactoring to `dependency_chain.go` and the
`computeVulns()` function in `resolve.go`, so please take a more careful
look at those.

go.mod

@@ -4,6 +4,7 @@ go 1.21.5
 
 require (
 	deps.dev/api/v3alpha v0.0.0-20240109042716-00b51ef52ece
+	deps.dev/util/resolve v0.0.0-20240109042716-00b51ef52ece
 	github.com/BurntSushi/toml v1.3.2
 	github.com/CycloneDX/cyclonedx-go v0.8.0
 	github.com/gkampitakis/go-snaps v0.4.12
@@ -23,12 +24,13 @@ require (
 	golang.org/x/term v0.16.0
 	golang.org/x/vuln v1.0.1
 	google.golang.org/grpc v1.60.1
+	google.golang.org/protobuf v1.31.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	// Vanity URL for https://github.com/imdario/mergo
 	dario.cat/mergo v1.0.0 // indirect
+	deps.dev/util/semver v0.0.0-20240109040450-1e316b822bc4 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c // indirect
 	github.com/anchore/go-struct-converter v0.0.0-20230627203149-c72ef8859ca9 // indirect
@@ -66,6 +68,5 @@ require (
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.17.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 )

go.sum

@@ -2,6 +2,10 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 deps.dev/api/v3alpha v0.0.0-20240109042716-00b51ef52ece h1:jvq1tMp7Xx0oD43DFxG7Eiawkc3UzAaEv6inEylcuc8=
 deps.dev/api/v3alpha v0.0.0-20240109042716-00b51ef52ece/go.mod h1:uRN72FJn1F0FD/2ZYUOqdyFMu8VUsyHxvmZAMW30/DA=
+deps.dev/util/resolve v0.0.0-20240109042716-00b51ef52ece h1:qVMTb2x3WSlxepBTrOB6YCwMFZ4bHjIUkvuIQTvCvTw=
+deps.dev/util/resolve v0.0.0-20240109042716-00b51ef52ece/go.mod h1:jf1QVEA+0Tj8gSiKyKabwsx4M5zK1LC49xjphwNP5ko=
+deps.dev/util/semver v0.0.0-20240109040450-1e316b822bc4 h1:RDmJe2F67jB7ovkbd28Pdpw3vEYUi2tWV5RlOHlxByk=
+deps.dev/util/semver v0.0.0-20240109040450-1e316b822bc4/go.mod h1:jkcH+k02gWHBiZ7G4OnUOkSZ6WDq54Pt5DrOA8FN8Uo=
 github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/CycloneDX/cyclonedx-go v0.8.0 h1:FyWVj6x6hoJrui5uRQdYZcSievw3Z32Z88uYzG/0D6M=

internal/resolution/client/depsdev_client.go

@@ -0,0 +1,96 @@
+package client
+
+import (
+	"context"
+	"encoding/gob"
+	"os"
+
+	pb "deps.dev/api/v3alpha"
+	"deps.dev/util/resolve"
+	"github.com/google/osv-scanner/internal/resolution/datasource"
+)
+
+const depsDevCacheExt = ".resolve.deps"
+
+// DepsDevClient is a ResolutionClient wrapping the official resolve.APIClient
+type DepsDevClient struct {
+	resolve.APIClient
+	c *datasource.DepsDevAPIClient
+}
+
+func NewDepsDevClient(addr string) (*DepsDevClient, error) {
+	c, err := datasource.NewDepsDevAPIClient(addr)
+	if err != nil {
+		return nil, err
+	}
+
+	return &DepsDevClient{APIClient: *resolve.NewAPIClient(c), c: c}, nil
+}
+
+func (d *DepsDevClient) PreFetch(ctx context.Context, requirements []resolve.RequirementVersion, manifestPath string) {
+	// It doesn't matter if loading the cache fails
+	_ = d.LoadCache(manifestPath)
+
+	// Use the deps.dev client to fetch complete dependency graphs of the direct requirements
+	for _, im := range requirements {
+		// Get the preferred version of the import requirement
+		vks, err := d.MatchingVersions(ctx, im.VersionKey)
+		if err != nil || len(vks) == 0 {
+			continue
+		}
+
+		vk := vks[len(vks)-1]
+
+		// Make a request for the precomputed dependency tree
+		resp, err := d.c.GetDependencies(ctx, &pb.GetDependenciesRequest{
+			VersionKey: &pb.VersionKey{
+				System:  pb.System(vk.System),
+				Name:    vk.Name,
+				Version: vk.Version,
+			},
+		})
+		if err != nil {
+			continue
+		}
+
+		// Send off queries to cache the packages in the dependency tree
+		for _, node := range resp.GetNodes() {
+			pbvk := node.GetVersionKey()
+
+			pk := resolve.PackageKey{
+				System: resolve.System(pbvk.GetSystem()),
+				Name:   pbvk.GetName(),
+			}
+			go d.Versions(ctx, pk) //nolint:errcheck
+
+			vk := resolve.VersionKey{
+				PackageKey:  pk,
+				Version:     pbvk.GetVersion(),
+				VersionType: resolve.Concrete,
+			}
+			go d.Requirements(ctx, vk) //nolint:errcheck
+			go d.Version(ctx, vk)      //nolint:errcheck
+		}
+	}
+	// Don't bother waiting for these goroutines to finish.
+}
+
+func (d *DepsDevClient) WriteCache(path string) error {
+	f, err := os.Create(path + depsDevCacheExt)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	return gob.NewEncoder(f).Encode(d.c)
+}
+
+func (d *DepsDevClient) LoadCache(path string) error {
+	f, err := os.Open(path + depsDevCacheExt)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	return gob.NewDecoder(f).Decode(&d.c)
+}

internal/resolution/client/override_client.go

@@ -0,0 +1,74 @@
+package client
+
+import (
+	"context"
+	"slices"
+
+	"deps.dev/util/resolve"
+)
+
+// OvverideClient wraps a resolve.Client, allowing for custom packages & versions to be added
+type OverrideClient struct {
+	c resolve.Client
+
+	// Can't quite reuse resolve.LocalClient because it automatically creates dependencies
+	pkgVers map[resolve.PackageKey][]resolve.Version            // versions of a package
+	verDeps map[resolve.VersionKey][]resolve.RequirementVersion // dependencies of a version
+}
+
+func NewOverrideClient(c resolve.Client) *OverrideClient {
+	return &OverrideClient{
+		c:       c,
+		pkgVers: make(map[resolve.PackageKey][]resolve.Version),
+		verDeps: make(map[resolve.VersionKey][]resolve.RequirementVersion),
+	}
+}
+
+func (c *OverrideClient) AddVersion(v resolve.Version, deps []resolve.RequirementVersion) {
+	// TODO: Inserting multiple co-dependent requirements may not work, depending on order
+	versions := c.pkgVers[v.PackageKey]
+	sem := v.Semver()
+	// Only add it to the versions if not already there (and keep versions sorted)
+	idx, ok := slices.BinarySearchFunc(versions, v, func(a, b resolve.Version) int {
+		return sem.Compare(a.Version, b.Version)
+	})
+	if !ok {
+		versions = slices.Insert(versions, idx, v)
+	}
+	c.pkgVers[v.PackageKey] = versions
+	c.verDeps[v.VersionKey] = slices.Clone(deps) // overwrites dependencies if called multiple times with same version
+}
+
+func (c *OverrideClient) Version(ctx context.Context, vk resolve.VersionKey) (resolve.Version, error) {
+	for _, v := range c.pkgVers[vk.PackageKey] {
+		if v.VersionKey == vk {
+			return v, nil
+		}
+	}
+
+	return c.c.Version(ctx, vk)
+}
+
+func (c *OverrideClient) Versions(ctx context.Context, pk resolve.PackageKey) ([]resolve.Version, error) {
+	if vers, ok := c.pkgVers[pk]; ok {
+		return vers, nil
+	}
+
+	return c.c.Versions(ctx, pk)
+}
+
+func (c *OverrideClient) Requirements(ctx context.Context, vk resolve.VersionKey) ([]resolve.RequirementVersion, error) {
+	if deps, ok := c.verDeps[vk]; ok {
+		return deps, nil
+	}
+
+	return c.c.Requirements(ctx, vk)
+}
+
+func (c *OverrideClient) MatchingVersions(ctx context.Context, vk resolve.VersionKey) ([]resolve.Version, error) {
+	if vs, ok := c.pkgVers[vk.PackageKey]; ok {
+		return resolve.MatchRequirement(vk, vs), nil
+	}
+
+	return c.c.MatchingVersions(ctx, vk)
+}

internal/resolution/client/resolution_client.go

@@ -0,0 +1,17 @@
+package client
+
+import (
+	"context"
+
+	"deps.dev/util/resolve"
+)
+
+type ResolutionClient interface {
+	resolve.Client
+	// WriteCache writes a manifest-specific resolution cache.
+	WriteCache(filepath string) error
+	// LoadCache loads a manifest-specific resolution cache.
+	LoadCache(filepath string) error
+	// PreFetch loads cache, then makes and caches likely queries needed for resolving a package with a list of requirements
+	PreFetch(ctx context.Context, requirements []resolve.RequirementVersion, manifestPath string)
+}

internal/resolution/datasource/cache.go

@@ -0,0 +1,26 @@
+package datasource
+
+import (
+	"bytes"
+	"encoding/gob"
+	"time"
+)
+
+const cacheExpiry = 6 * time.Hour
+
+func gobMarshal(v any) ([]byte, error) {
+	var b bytes.Buffer
+	enc := gob.NewEncoder(&b)
+
+	err := enc.Encode(v)
+	if err != nil {
+		return nil, err
+	}
+
+	return b.Bytes(), nil
+}
+
+func gobUnmarshal(b []byte, v any) error {
+	dec := gob.NewDecoder(bytes.NewReader(b))
+	return dec.Decode(v)
+}

internal/resolution/datasource/depsdev_api.go

@@ -0,0 +1,129 @@
+package datasource
+
+import (
+	"context"
+	"crypto/x509"
+	"fmt"
+	"sync"
+	"time"
+
+	pb "deps.dev/api/v3alpha"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+)
+
+// DepsDevAPIClient is a wrapper for InsightsClient that caches requests.
+type DepsDevAPIClient struct {
+	pb.InsightsClient
+
+	// cache fields
+	mu                sync.Mutex
+	cacheTimestamp    *time.Time
+	packageCache      map[packageKey]*pb.Package
+	versionCache      map[versionKey]*pb.Version
+	requirementsCache map[versionKey]*pb.Requirements
+}
+
+// Comparable types to use as map keys for cache.
+type packageKey struct {
+	System pb.System
+	Name   string
+}
+
+func makePackageKey(k *pb.PackageKey) packageKey {
+	return packageKey{
+		System: k.GetSystem(),
+		Name:   k.GetName(),
+	}
+}
+
+type versionKey struct {
+	System  pb.System
+	Name    string
+	Version string
+}
+
+func makeVersionKey(k *pb.VersionKey) versionKey {
+	return versionKey{
+		System:  k.GetSystem(),
+		Name:    k.GetName(),
+		Version: k.GetVersion(),
+	}
+}
+
+func NewDepsDevAPIClient(addr string) (*DepsDevAPIClient, error) {
+	certPool, err := x509.SystemCertPool()
+	if err != nil {
+		return nil, fmt.Errorf("getting system cert pool: %w", err)
+	}
+	creds := credentials.NewClientTLSFromCert(certPool, "")
+	conn, err := grpc.Dial(addr, grpc.WithTransportCredentials(creds))
+	if err != nil {
+		return nil, fmt.Errorf("dialling %q: %w", addr, err)
+	}
+	c := pb.NewInsightsClient(conn)
+
+	return &DepsDevAPIClient{
+		InsightsClient:    c,
+		packageCache:      make(map[packageKey]*pb.Package),
+		versionCache:      make(map[versionKey]*pb.Version),
+		requirementsCache: make(map[versionKey]*pb.Requirements),
+	}, nil
+}
+
+func (c *DepsDevAPIClient) GetPackage(ctx context.Context, in *pb.GetPackageRequest, opts ...grpc.CallOption) (*pb.Package, error) {
+	key := makePackageKey(in.GetPackageKey())
+	c.mu.Lock()
+	pkg, ok := c.packageCache[key]
+	c.mu.Unlock()
+	if ok {
+		return pkg, nil
+	}
+	// TODO: avoid sending the same request multiple times if called multiple times before the cache is filled
+	pkg, err := c.InsightsClient.GetPackage(ctx, in, opts...)
+	if err == nil {
+		c.mu.Lock()
+		c.packageCache[key] = pkg
+		c.mu.Unlock()
+	}
+
+	return pkg, err
+}
+
+func (c *DepsDevAPIClient) GetVersion(ctx context.Context, in *pb.GetVersionRequest, opts ...grpc.CallOption) (*pb.Version, error) {
+	key := makeVersionKey(in.GetVersionKey())
+	c.mu.Lock()
+	ver, ok := c.versionCache[key]
+	c.mu.Unlock()
+	if ok {
+		return ver, nil
+	}
+	// TODO: avoid sending the same request multiple times if called multiple times before the cache is filled
+	ver, err := c.InsightsClient.GetVersion(ctx, in, opts...)
+	if err == nil {
+		c.mu.Lock()
+		c.versionCache[key] = ver
+		c.mu.Unlock()
+	}
+
+	return ver, err
+}
+
+func (c *DepsDevAPIClient) GetRequirements(ctx context.Context, in *pb.GetRequirementsRequest, opts ...grpc.CallOption) (*pb.Requirements, error) {
+	key := makeVersionKey(in.GetVersionKey())
+	c.mu.Lock()
+	req, ok := c.requirementsCache[key]
+	c.mu.Unlock()
+	if ok {
+		return req, nil
+	}
+	// TODO: avoid sending the same request multiple times if called multiple times before the cache is filled
+	req, err := c.InsightsClient.GetRequirements(ctx, in, opts...)
+	if err == nil {
+		c.mu.Lock()
+		c.requirementsCache[key] = req
+		c.mu.Unlock()
+	}
+
+	return req, err
+}