diff --git a/internal/testing/cmd/ingest/cmd/example.go b/internal/testing/cmd/ingest/cmd/example.go index abdd283a6f..d094c519fc 100644 --- a/internal/testing/cmd/ingest/cmd/example.go +++ b/internal/testing/cmd/ingest/cmd/example.go @@ -37,7 +37,7 @@ type options struct { graphqlEndpoint string } -func ingestExample(cmd *cobra.Command, args []string) { +func ingestExample(cmd *cobra.Command, args []string, gqlClient graphql.Client) { ctx := logging.WithLogger(context.Background()) logger := logging.FromContext(ctx) @@ -59,7 +59,7 @@ func ingestExample(cmd *cobra.Command, args []string) { var inputs []assembler.IngestPredicates for _, doc := range docs { // This is a test example, so we will ignore calling out to a collectsub service - input, _, err := parser.ParseDocumentTree(ctx, doc) + input, _, err := parser.ParseDocumentTree(ctx, doc, gqlClient) if err != nil { logger.Fatalf("unable to parse document: %v", err) } diff --git a/internal/testing/cmd/ingest/cmd/root.go b/internal/testing/cmd/ingest/cmd/root.go index 279511727b..bc091f8b65 100644 --- a/internal/testing/cmd/ingest/cmd/root.go +++ b/internal/testing/cmd/ingest/cmd/root.go @@ -18,6 +18,8 @@ package cmd import ( "context" "fmt" + "github.com/Khan/genqlient/graphql" + "net/http" "os" "strings" @@ -85,7 +87,9 @@ var rootCmd = &cobra.Command{ Use: "ingest", Short: "example ingestor for ingesting a set of example documents and populating a graph for GUAC", Run: func(cmd *cobra.Command, args []string) { - ingestExample(cmd, args) + httpClient := http.Client{} + gqlClient := graphql.NewClient(flags.graphqlEndpoint, &httpClient) + ingestExample(cmd, args, gqlClient) }, } diff --git a/pkg/assembler/clients/generated/operations.go b/pkg/assembler/clients/generated/operations.go index a336313086..6f47901455 100644 --- a/pkg/assembler/clients/generated/operations.go +++ b/pkg/assembler/clients/generated/operations.go @@ -5247,6 +5247,7 @@ func (v *AllPkgTreeNamespacesPackageNamespaceNamesPackageName) GetVersions() []A // the trie. type AllPkgTreeNamespacesPackageNamespaceNamesPackageNameVersionsPackageVersion struct { Id string `json:"id"` + Purl string `json:"purl"` Version string `json:"version"` Qualifiers []AllPkgTreeNamespacesPackageNamespaceNamesPackageNameVersionsPackageVersionQualifiersPackageQualifier `json:"qualifiers"` Subpath string `json:"subpath"` @@ -5257,6 +5258,11 @@ func (v *AllPkgTreeNamespacesPackageNamespaceNamesPackageNameVersionsPackageVers return v.Id } +// GetPurl returns AllPkgTreeNamespacesPackageNamespaceNamesPackageNameVersionsPackageVersion.Purl, and is useful for accessing the field via an interface. +func (v *AllPkgTreeNamespacesPackageNamespaceNamesPackageNameVersionsPackageVersion) GetPurl() string { + return v.Purl +} + // GetVersion returns AllPkgTreeNamespacesPackageNamespaceNamesPackageNameVersionsPackageVersion.Version, and is useful for accessing the field via an interface. func (v *AllPkgTreeNamespacesPackageNamespaceNamesPackageNameVersionsPackageVersion) GetVersion() string { return v.Version @@ -24212,6 +24218,7 @@ fragment AllPkgTree on Package { name versions { id + purl version qualifiers { key @@ -24312,6 +24319,7 @@ fragment AllPkgTree on Package { name versions { id + purl version qualifiers { key @@ -24402,6 +24410,7 @@ fragment AllPkgTree on Package { name versions { id + purl version qualifiers { key @@ -24467,6 +24476,7 @@ fragment AllPkgTree on Package { name versions { id + purl version qualifiers { key @@ -24581,6 +24591,7 @@ fragment AllPkgTree on Package { name versions { id + purl version qualifiers { key @@ -27296,6 +27307,7 @@ fragment AllPkgTree on Package { name versions { id + purl version qualifiers { key @@ -27484,6 +27496,7 @@ fragment AllPkgTree on Package { name versions { id + purl version qualifiers { key @@ -27961,6 +27974,7 @@ fragment AllPkgTree on Package { name versions { id + purl version qualifiers { key @@ -28436,6 +28450,7 @@ fragment AllPkgTree on Package { name versions { id + purl version qualifiers { key @@ -29012,6 +29027,7 @@ fragment AllPkgTree on Package { name versions { id + purl version qualifiers { key @@ -29140,6 +29156,7 @@ fragment AllPkgTree on Package { name versions { id + purl version qualifiers { key diff --git a/pkg/assembler/clients/operations/trees.graphql b/pkg/assembler/clients/operations/trees.graphql index c9aeba03a3..6ecb3683eb 100644 --- a/pkg/assembler/clients/operations/trees.graphql +++ b/pkg/assembler/clients/operations/trees.graphql @@ -30,6 +30,7 @@ fragment AllPkgTree on Package { name versions { id + purl version qualifiers { key diff --git a/pkg/ingestor/ingestor.go b/pkg/ingestor/ingestor.go index df3848be1e..f6e0d62c40 100644 --- a/pkg/ingestor/ingestor.go +++ b/pkg/ingestor/ingestor.go @@ -43,10 +43,13 @@ func Ingest( transport http.RoundTripper, csubClient csub_client.Client, ) error { + httpClient := http.Client{} + gqlclient := graphql.NewClient(graphqlEndpoint, &httpClient) + logger := d.ChildLogger // Get pipeline of components processorFunc := GetProcessor(ctx) - ingestorFunc := GetIngestor(ctx) + ingestorFunc := GetIngestor(ctx, gqlclient) collectSubEmitFunc := GetCollectSubEmit(ctx, csubClient) assemblerFunc := GetAssembler(ctx, d.ChildLogger, graphqlEndpoint, transport) @@ -83,10 +86,13 @@ func MergedIngest( transport http.RoundTripper, csubClient csub_client.Client, ) error { + httpClient := http.Client{} + gqlclient := graphql.NewClient(graphqlEndpoint, &httpClient) + logger := logging.FromContext(ctx) // Get pipeline of components processorFunc := GetProcessor(ctx) - ingestorFunc := GetIngestor(ctx) + ingestorFunc := GetIngestor(ctx, gqlclient) collectSubEmitFunc := GetCollectSubEmit(ctx, csubClient) assemblerFunc := GetAssembler(ctx, logger, graphqlEndpoint, transport) @@ -159,9 +165,9 @@ func GetProcessor(ctx context.Context) func(*processor.Document) (processor.Docu } } -func GetIngestor(ctx context.Context) func(processor.DocumentTree) ([]assembler.IngestPredicates, []*parser_common.IdentifierStrings, error) { +func GetIngestor(ctx context.Context, gqlClient graphql.Client) func(processor.DocumentTree) ([]assembler.IngestPredicates, []*parser_common.IdentifierStrings, error) { return func(doc processor.DocumentTree) ([]assembler.IngestPredicates, []*parser_common.IdentifierStrings, error) { - return parser.ParseDocumentTree(ctx, doc) + return parser.ParseDocumentTree(ctx, doc, gqlClient) } } diff --git a/pkg/ingestor/parser/csaf/parser_csaf.go b/pkg/ingestor/parser/csaf/parser_csaf.go index fe79a806d0..d60919e223 100644 --- a/pkg/ingestor/parser/csaf/parser_csaf.go +++ b/pkg/ingestor/parser/csaf/parser_csaf.go @@ -18,6 +18,7 @@ package csaf import ( "context" "fmt" + "github.com/Khan/genqlient/graphql" "strconv" jsoniter "github.com/json-iterator/go" @@ -68,7 +69,7 @@ type visitedProductRef struct { category string } -func NewCsafParser() common.DocumentParser { +func NewCsafParser(gqlClient graphql.Client) common.DocumentParser { return &csafParser{ identifierStrings: &common.IdentifierStrings{}, } diff --git a/pkg/ingestor/parser/csaf/parser_csaf_test.go b/pkg/ingestor/parser/csaf/parser_csaf_test.go index 1757c4b441..4aa2a6ad04 100644 --- a/pkg/ingestor/parser/csaf/parser_csaf_test.go +++ b/pkg/ingestor/parser/csaf/parser_csaf_test.go @@ -57,7 +57,7 @@ func Test_csafParser(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := NewCsafParser() + s := NewCsafParser(nil) err := s.Parse(ctx, tt.doc) if (err != nil) != tt.wantErr { t.Errorf("csafParse.Parse() error = %v, wantErr %v", err, tt.wantErr) @@ -295,7 +295,7 @@ func Test_csafParser_GetIdentifiers(t *testing.T) { }, } - c := NewCsafParser() + c := NewCsafParser(nil) err := c.Parse(test.ctx, test.fields.doc) if err != nil { diff --git a/pkg/ingestor/parser/cyclonedx/parser_cyclonedx.go b/pkg/ingestor/parser/cyclonedx/parser_cyclonedx.go index 4dfd90113a..76a8a5f0a2 100644 --- a/pkg/ingestor/parser/cyclonedx/parser_cyclonedx.go +++ b/pkg/ingestor/parser/cyclonedx/parser_cyclonedx.go @@ -19,7 +19,9 @@ import ( "context" "encoding/xml" "fmt" + "github.com/Khan/genqlient/graphql" "reflect" + "regexp" "strings" "time" @@ -57,6 +59,7 @@ type cyclonedxParser struct { identifierStrings *common.IdentifierStrings cdxBom *cdx.BOM vulnData vulnData + gqlClient graphql.Client } type vulnData struct { @@ -65,11 +68,22 @@ type vulnData struct { vex []assembler.VexIngest } -func NewCycloneDXParser() common.DocumentParser { +type VersionMatchObject struct { + MinVersion string + MaxVersion string + MinInclusive bool + MaxInclusive bool +} + +var getPackages func(ctx context.Context, gqlClient graphql.Client, filter model.PkgSpec) (*model.PackagesResponse, error) + +func NewCycloneDXParser(gqlClient graphql.Client) common.DocumentParser { + getPackages = model.Packages return &cyclonedxParser{ packagePackages: map[string][]*model.PkgInputSpec{}, packageArtifacts: map[string][]*model.ArtifactInputSpec{}, identifierStrings: &common.IdentifierStrings{}, + gqlClient: gqlClient, } } @@ -410,7 +424,6 @@ func (c *cyclonedxParser) getVulnerabilities(ctx context.Context) error { // Get package name and range versions to create package input spec for the affected packages. func (c *cyclonedxParser) getAffectedPackages(ctx context.Context, vulnInput *model.VulnerabilityInputSpec, vexData model.VexStatementInputSpec, affectsObj cdx.Affects) (*[]assembler.VexIngest, error) { - logger := logging.FromContext(ctx) pkgRef := affectsObj.Ref var foundVexIngest []assembler.VexIngest @@ -430,15 +443,15 @@ func (c *cyclonedxParser) getAffectedPackages(ctx context.Context, vulnInput *mo if len(pkgRefInfo) != 2 { return nil, fmt.Errorf("malformed affected-package reference: %q", affectsObj.Ref) } - pkdIdentifier := pkgRefInfo[1] + pkgIdentifier := pkgRefInfo[1] // check whether the ref contains a purl - if strings.Contains(pkdIdentifier, "pkg:") { - pkg, err := asmhelpers.PurlToPkg(pkdIdentifier) + if strings.Contains(pkgIdentifier, "pkg:") { + pkg, err := asmhelpers.PurlToPkg(pkgIdentifier) if err != nil { return nil, fmt.Errorf("unable to create package input spec: %v", err) } - c.identifierStrings.PurlStrings = append(c.identifierStrings.PurlStrings, pkdIdentifier) + c.identifierStrings.PurlStrings = append(c.identifierStrings.PurlStrings, pkgIdentifier) return &[]assembler.VexIngest{{VexData: &vexData, Vulnerability: vulnInput, Pkg: pkg}}, nil } @@ -448,33 +461,186 @@ func (c *cyclonedxParser) getAffectedPackages(ctx context.Context, vulnInput *mo var viList []assembler.VexIngest for _, affect := range *affectsObj.Range { - // TODO: Handle package range versions (see - https://github.com/CycloneDX/bom-examples/blob/master/VEX/CISA-Use-Cases/Case-8/vex.json#L42) + // Handles package range versions like https://github.com/CycloneDX/bom-examples/blob/master/VEX/CISA-Use-Cases/Case-8/vex.json#L42 if affect.Range != "" { - logger.Debugf("[cdx vex] package range versions not supported yet: %q", affect.Range) + purls, err := c.findCDXPkgVersionIDs(ctx, pkgIdentifier, affect.Range) + if err != nil { + return nil, fmt.Errorf("error searching version ranges for %s: %v", affect.Range, err) + } + + for _, purl := range purls { + var addVulnDataErr error + viList, addVulnDataErr = c.AddVulnerabilityIngestionData(purl, vexData, vulnInput, viList) + if addVulnDataErr != nil { + return nil, fmt.Errorf("could not add Vulnerability information for %s: %v", pkgIdentifier, err) + } + } + continue } if affect.Version == "" { return nil, fmt.Errorf("no version found for package ref %q", pkgRef) } - vi := &assembler.VexIngest{ - VexData: &vexData, - Vulnerability: vulnInput, - } // create guac specific identifier string using affected package name and version. - pkgID := guacCDXPkgPurl(pkdIdentifier, affect.Version, "", false) - pkg, err := asmhelpers.PurlToPkg(pkgID) + purl := guacCDXPkgPurl(pkgIdentifier, affect.Version, "", false) + var err error + viList, err = c.AddVulnerabilityIngestionData(purl, vexData, vulnInput, viList) if err != nil { - return nil, fmt.Errorf("unable to create package input spec from guac pkg purl: %v", err) + return nil, fmt.Errorf("could not add Vulnerability information for %s: %v", pkgIdentifier, err) } - vi.Pkg = pkg - viList = append(viList, *vi) - c.identifierStrings.PurlStrings = append(c.identifierStrings.PurlStrings, pkgID) } return &viList, nil } +func (c *cyclonedxParser) AddVulnerabilityIngestionData(purl string, vexData model.VexStatementInputSpec, vulnInput *model.VulnerabilityInputSpec, viList []assembler.VexIngest) ([]assembler.VexIngest, error) { + pkg, err := asmhelpers.PurlToPkg(purl) + if err != nil { + return nil, fmt.Errorf("unable to create package input spec from guac pkg purl: %v", err) + } + vi := &assembler.VexIngest{ + VexData: &vexData, + Vulnerability: vulnInput, + Pkg: pkg, + } + viList = append(viList, *vi) + c.identifierStrings.PurlStrings = append(c.identifierStrings.PurlStrings, purl) + return viList, nil +} + +func (c *cyclonedxParser) findCDXPkgVersionIDs(ctx context.Context, pkgIdentifier string, versionRange string) ([]string, error) { + var matchingVersionPurls []string + + // search for all packages that have a matching type, namespace, and name. + typeGUAC, namespace := "guac", "pkg" + pkgResponse, err := getPackages(ctx, c.gqlClient, model.PkgSpec{ + Type: &typeGUAC, + Namespace: &namespace, + Name: &pkgIdentifier, + }) + if err != nil { + return nil, fmt.Errorf("error querying for packages: %w", err) + } + + pkgVersionsMap := map[string]string{} + var pkgVersions []string + for _, depPkgVersion := range pkgResponse.Packages[0].Namespaces[0].Names[0].Versions { + pkgVersions = append(pkgVersions, depPkgVersion.Version) + pkgVersionsMap[depPkgVersion.Version] = depPkgVersion.Purl + } + + matchingDepPkgVersions, err := WhichVersionMatches(pkgVersions, versionRange) + if err != nil { + return nil, fmt.Errorf("error determining dependent version matches: %w", err) + } + + for matchingDepPkgVersion := range matchingDepPkgVersions { + matchingVersionPurls = append(matchingVersionPurls, pkgVersionsMap[matchingDepPkgVersion]) + } + + return matchingVersionPurls, nil +} + +func WhichVersionMatches(versions []string, versionRange string) (map[string]bool, error) { + matchedVersions := make(map[string]bool) + versionMatchObjects, excludedVersions, err := parseVersionRange(versionRange) + if err != nil { + return nil, fmt.Errorf("unable to parse version range: %w", err) + } + + for _, version := range versions { + for _, vmo := range versionMatchObjects { + if vmo.Match(version, excludedVersions) { + matchedVersions[version] = true + } + } + } + + return matchedVersions, nil +} + +// Match checks if the given version is within the range specified by the VersionMatchObject. +func (vmo VersionMatchObject) Match(version string, excludedVersions []string) bool { + withinRange := ((vmo.MinInclusive && vmo.MinVersion <= version) || (vmo.MinVersion < version)) && + ((vmo.MaxInclusive && vmo.MaxVersion >= version) || (vmo.MaxVersion > version)) + + if !withinRange { + return false + } + for _, excludedVersion := range excludedVersions { + if version == excludedVersion { + return false // Version is explicitly excluded + } + } + return true +} + +func parseVersionRange(rangeStr string) ([]VersionMatchObject, []string, error) { + // Split the range string on the logical OR operator `|` so that we can find different comparisons + ranges := strings.Split(rangeStr, "|") + + var versionMatchObjects []VersionMatchObject + var excludedVersions []string + + for _, r := range ranges { + // Check for implicit equals by detecting an absence of range operators. + if !strings.ContainsAny(r, "><=") { + versionMatchObjects = append(versionMatchObjects, VersionMatchObject{ + MinVersion: r, + MaxVersion: r, + MinInclusive: true, + MaxInclusive: true, + }) + continue + } + + // Regular expression to capture version comparison operators and version numbers + // This is for cases like: "vers:generic/>=2.9|<=4.1" and ">=2.0.0 <3.0.0 | != 2.3.0". + // This doesn't check for implicit equality + re := regexp.MustCompile(`(!=|[><]=?|=)?\s*v?(\d+(\.\d+)?(\.\d+)?)`) + + // Find all matches for version constraints within r. + matches := re.FindAllStringSubmatch(r, -1) + if len(matches) == 0 { + return nil, nil, fmt.Errorf("no version constraints found in: %s", r) + } + + vmo := VersionMatchObject{} + + for _, match := range matches { + operator := match[1] + version := match[2] + + switch operator { + case ">": + vmo.MinVersion = version + vmo.MinInclusive = false + case ">=": + vmo.MinVersion = version + vmo.MinInclusive = true + case "<": + vmo.MaxVersion = version + vmo.MaxInclusive = false + case "<=": + vmo.MaxVersion = version + vmo.MaxInclusive = true + case "=": + vmo.MinVersion = version + vmo.MaxVersion = version + vmo.MinInclusive = true + vmo.MaxInclusive = true + case "!=": + excludedVersions = append(excludedVersions, version) + } + } + + versionMatchObjects = append(versionMatchObjects, vmo) + } + + return versionMatchObjects, excludedVersions, nil +} + func (c *cyclonedxParser) getPackageElement(elementID string) []*model.PkgInputSpec { if packNode, ok := c.packagePackages[elementID]; ok { return packNode diff --git a/pkg/ingestor/parser/cyclonedx/parser_cyclonedx_test.go b/pkg/ingestor/parser/cyclonedx/parser_cyclonedx_test.go index 1ad93d88a0..b6f47999a5 100644 --- a/pkg/ingestor/parser/cyclonedx/parser_cyclonedx_test.go +++ b/pkg/ingestor/parser/cyclonedx/parser_cyclonedx_test.go @@ -17,6 +17,7 @@ package cyclonedx import ( "context" + "reflect" "testing" "time" @@ -130,7 +131,7 @@ func Test_cyclonedxParser(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := NewCycloneDXParser() + s := NewCycloneDXParser(nil) err := s.Parse(ctx, tt.doc) if (err != nil) != tt.wantErr { t.Errorf("cyclonedxParser.Parse() error = %v, wantErr %v", err, tt.wantErr) @@ -560,3 +561,184 @@ func noAnalysisVexPredicates() *assembler.IngestPredicates { }, } } + +func Test_parseVersionRange(t *testing.T) { + type args struct { + rangeStr string + } + tests := []struct { + name string + args args + want []VersionMatchObject + want1 []string + wantErr bool + }{ + { + name: "Single Version Implicit Equal", + args: args{ + rangeStr: "1.2.3", + }, + want: []VersionMatchObject{ + { + MinVersion: "1.2.3", + MaxVersion: "1.2.3", + MinInclusive: true, + MaxInclusive: true, + }, + }, + want1: []string{}, + wantErr: false, + }, + { + name: "Single Version Explicit Equal", + args: args{ + rangeStr: "=1.2.3", + }, + want: []VersionMatchObject{ + { + MinVersion: "1.2.3", + MaxVersion: "1.2.3", + MinInclusive: true, + MaxInclusive: true, + }, + }, + want1: []string{}, + wantErr: false, + }, + { + name: "Greater Than", + args: args{ + rangeStr: ">1.2.3", + }, + want: []VersionMatchObject{ + { + MinVersion: "1.2.3", + MaxVersion: "", + MinInclusive: false, + MaxInclusive: false, + }, + }, + want1: []string{}, + wantErr: false, + }, + { + name: "Greater Than or Equal", + args: args{ + rangeStr: ">=1.2.3", + }, + want: []VersionMatchObject{ + { + MinVersion: "1.2.3", + MaxVersion: "", + MinInclusive: true, + MaxInclusive: false, + }, + }, + want1: []string{}, + wantErr: false, + }, + { + name: "Less Than", + args: args{ + rangeStr: "<1.2.3", + }, + want: []VersionMatchObject{ + { + MinVersion: "", + MaxVersion: "1.2.3", + MinInclusive: false, + MaxInclusive: false, + }, + }, + want1: []string{}, + wantErr: false, + }, + { + name: "Less Than or Equal", + args: args{ + rangeStr: "<=1.2.3", + }, + want: []VersionMatchObject{ + { + MinVersion: "", + MaxVersion: "1.2.3", + MinInclusive: false, + MaxInclusive: true, + }, + }, + want1: []string{}, + wantErr: false, + }, + { + name: "Range Inclusive", + args: args{ + rangeStr: ">=1.2.0,<=1.2.3", + }, + want: []VersionMatchObject{ + { + MinVersion: "1.2.0", + MaxVersion: "1.2.3", + MinInclusive: true, + MaxInclusive: true, + }, + }, + want1: []string{}, + wantErr: false, + }, + { + name: "Range Exclusive", + args: args{ + rangeStr: ">1.2.0,<1.2.3", + }, + want: []VersionMatchObject{ + { + MinVersion: "1.2.0", + MaxVersion: "1.2.3", + MinInclusive: false, + MaxInclusive: false, + }, + }, + want1: []string{}, + wantErr: false, + }, + { + name: "Complex Range", + args: args{ + rangeStr: ">=1.0.0,<1.0.5 | >=1.1.0,<1.1.5", + }, + want: []VersionMatchObject{ + { + MinVersion: "1.0.0", + MaxVersion: "1.0.5", + MinInclusive: true, + MaxInclusive: false, + }, + { + MinVersion: "1.1.0", + MaxVersion: "1.1.5", + MinInclusive: true, + MaxInclusive: false, + }, + }, + want1: []string{}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1, err := parseVersionRange(tt.args.rangeStr) + if (err != nil) != tt.wantErr { + t.Errorf("parseVersionRange() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseVersionRange() got = %v, want %v", got, tt.want) + } + if len(got1) == 0 && len(tt.want1) == 0 { + // Both slices are empty, consider this a match. + } else if !reflect.DeepEqual(got1, tt.want1) { + t.Errorf("parseVersionRange() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} diff --git a/pkg/ingestor/parser/deps_dev/deps_dev.go b/pkg/ingestor/parser/deps_dev/deps_dev.go index d141d8adfa..0f9383f19e 100644 --- a/pkg/ingestor/parser/deps_dev/deps_dev.go +++ b/pkg/ingestor/parser/deps_dev/deps_dev.go @@ -18,6 +18,7 @@ package deps_dev import ( "context" "fmt" + "github.com/Khan/genqlient/graphql" "time" jsoniter "github.com/json-iterator/go" @@ -37,7 +38,7 @@ type depsDevParser struct { packComponent *deps_dev.PackageComponent } -func NewDepsDevParser() common.DocumentParser { +func NewDepsDevParser(gqlClient graphql.Client) common.DocumentParser { return &depsDevParser{} } diff --git a/pkg/ingestor/parser/deps_dev/deps_dev_test.go b/pkg/ingestor/parser/deps_dev/deps_dev_test.go index 23b334c0fd..6b8710d39f 100644 --- a/pkg/ingestor/parser/deps_dev/deps_dev_test.go +++ b/pkg/ingestor/parser/deps_dev/deps_dev_test.go @@ -44,7 +44,7 @@ func TestNewDepsDevParser(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := NewDepsDevParser(); !reflect.DeepEqual(got, tt.want) { + if got := NewDepsDevParser(nil); !reflect.DeepEqual(got, tt.want) { t.Errorf("NewDepsDevParser() = %v, want %v", got, tt.want) } }) @@ -522,7 +522,7 @@ func Test_depsDevParser_Parse(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - d := NewDepsDevParser() + d := NewDepsDevParser(nil) if err := d.Parse(ctx, tt.doc); (err != nil) != tt.wantErr { t.Errorf("deps.dev.Parse() error = %v, wantErr %v", err, tt.wantErr) } @@ -567,7 +567,7 @@ func Test_depsDevParser_GetIdentifiers(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - d := NewDepsDevParser() + d := NewDepsDevParser(nil) if err := d.Parse(ctx, tt.doc); (err != nil) != tt.wantErr { t.Errorf("deps.dev.Parse() error = %v, wantErr %v", err, tt.wantErr) } diff --git a/pkg/ingestor/parser/dsse/parser_dsse.go b/pkg/ingestor/parser/dsse/parser_dsse.go index e0ab3c60b1..e4d0428656 100644 --- a/pkg/ingestor/parser/dsse/parser_dsse.go +++ b/pkg/ingestor/parser/dsse/parser_dsse.go @@ -18,6 +18,7 @@ package dsse import ( "context" "fmt" + "github.com/Khan/genqlient/graphql" "github.com/guacsec/guac/pkg/assembler" "github.com/guacsec/guac/pkg/handler/processor" @@ -33,7 +34,7 @@ type dsseParser struct { } // NewDSSEParser initializes the dsseParser -func NewDSSEParser() common.DocumentParser { +func NewDSSEParser(gqlClient graphql.Client) common.DocumentParser { return &dsseParser{ identities: []common.TrustInformation{}, } diff --git a/pkg/ingestor/parser/dsse/parser_dsse_test.go b/pkg/ingestor/parser/dsse/parser_dsse_test.go index 878428def1..d457c69f7b 100644 --- a/pkg/ingestor/parser/dsse/parser_dsse_test.go +++ b/pkg/ingestor/parser/dsse/parser_dsse_test.go @@ -50,7 +50,7 @@ func Test_DsseParser(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - d := NewDSSEParser() + d := NewDSSEParser(nil) err := d.Parse(ctx, tt.doc) if (err != nil) != tt.wantErr { t.Errorf("slsa.Parse() error = %v, wantErr %v", err, tt.wantErr) diff --git a/pkg/ingestor/parser/ingest_predicates/parser_ingest_predicates.go b/pkg/ingestor/parser/ingest_predicates/parser_ingest_predicates.go index 7cfeeda6e3..34565611da 100644 --- a/pkg/ingestor/parser/ingest_predicates/parser_ingest_predicates.go +++ b/pkg/ingestor/parser/ingest_predicates/parser_ingest_predicates.go @@ -21,6 +21,7 @@ package ingest_predicates import ( "context" + "github.com/Khan/genqlient/graphql" jsoniter "github.com/json-iterator/go" @@ -32,12 +33,15 @@ import ( var json = jsoniter.ConfigCompatibleWithStandardLibrary type ingestPredicatesParser struct { - preds assembler.IngestPredicates + preds assembler.IngestPredicates + gqlClient graphql.Client } // NewIngestPredicatesParser initializes the ingestPredicatesParser -func NewIngestPredicatesParser() common.DocumentParser { - return &ingestPredicatesParser{} +func NewIngestPredicatesParser(gqlClient graphql.Client) common.DocumentParser { + return &ingestPredicatesParser{ + gqlClient: gqlClient, + } } // Parse breaks out the document into the graph components diff --git a/pkg/ingestor/parser/ingest_predicates/parser_ingest_predicates_test.go b/pkg/ingestor/parser/ingest_predicates/parser_ingest_predicates_test.go index 31c574f69a..45f276be67 100644 --- a/pkg/ingestor/parser/ingest_predicates/parser_ingest_predicates_test.go +++ b/pkg/ingestor/parser/ingest_predicates/parser_ingest_predicates_test.go @@ -47,7 +47,7 @@ func Test_ingestPredicatesParser(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := NewIngestPredicatesParser() + s := NewIngestPredicatesParser(nil) err := s.Parse(ctx, tt.doc) if (err != nil) != tt.wantErr { t.Errorf("slsa.Parse() error = %v, wantErr %v", err, tt.wantErr) diff --git a/pkg/ingestor/parser/open_vex/parser_open_vex.go b/pkg/ingestor/parser/open_vex/parser_open_vex.go index c74059ddc1..426cfcbc25 100644 --- a/pkg/ingestor/parser/open_vex/parser_open_vex.go +++ b/pkg/ingestor/parser/open_vex/parser_open_vex.go @@ -18,6 +18,7 @@ package open_vex import ( "context" "fmt" + "github.com/Khan/genqlient/graphql" json "github.com/json-iterator/go" "github.com/openvex/go-vex/pkg/vex" @@ -52,7 +53,7 @@ type openVEXParser struct { cvs []assembler.CertifyVulnIngest } -func NewOpenVEXParser() common.DocumentParser { +func NewOpenVEXParser(gqlClient graphql.Client) common.DocumentParser { return &openVEXParser{ identifierStrings: &common.IdentifierStrings{}, } diff --git a/pkg/ingestor/parser/open_vex/parser_open_vex_test.go b/pkg/ingestor/parser/open_vex/parser_open_vex_test.go index e0bd6bd822..18a4786f02 100644 --- a/pkg/ingestor/parser/open_vex/parser_open_vex_test.go +++ b/pkg/ingestor/parser/open_vex/parser_open_vex_test.go @@ -48,7 +48,7 @@ func Test_openVEXParser_Parse(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - newParser := NewOpenVEXParser() + newParser := NewOpenVEXParser(nil) if err := newParser.Parse(tt.args.ctx, tt.args.doc); (err != nil) != tt.wantErr { t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantErr) } @@ -113,7 +113,7 @@ func Test_openVEXParser_GetPredicates(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := NewOpenVEXParser() + c := NewOpenVEXParser(nil) err := c.Parse(tt.args.ctx, tt.fields.doc) if err != nil { @@ -187,7 +187,7 @@ func Test_openVEXParser_GetIdentifiers(t *testing.T) { }, } - c := NewOpenVEXParser() + c := NewOpenVEXParser(nil) err := c.Parse(test.ctx, test.fields.doc) if err != nil { diff --git a/pkg/ingestor/parser/parser.go b/pkg/ingestor/parser/parser.go index 27187e5442..2c37612c79 100644 --- a/pkg/ingestor/parser/parser.go +++ b/pkg/ingestor/parser/parser.go @@ -18,6 +18,7 @@ package parser import ( "context" "fmt" + "github.com/Khan/genqlient/graphql" "github.com/guacsec/guac/pkg/assembler" "github.com/guacsec/guac/pkg/handler/processor" @@ -46,7 +47,7 @@ func init() { } var ( - documentParser = map[processor.DocumentType]func() common.DocumentParser{} + documentParser = map[processor.DocumentType]func(gqlClient graphql.Client) common.DocumentParser{} ) type docTreeBuilder struct { @@ -61,7 +62,7 @@ func newDocTreeBuilder() *docTreeBuilder { } } -func RegisterDocumentParser(p func() common.DocumentParser, d processor.DocumentType) error { +func RegisterDocumentParser(p func(gqlClient graphql.Client) common.DocumentParser, d processor.DocumentType) error { if _, ok := documentParser[d]; ok { documentParser[d] = p return fmt.Errorf("the document parser is being overwritten: %s", d) @@ -71,14 +72,14 @@ func RegisterDocumentParser(p func() common.DocumentParser, d processor.Document } // ParseDocumentTree takes the DocumentTree and create graph inputs (nodes and edges) per document node. -func ParseDocumentTree(ctx context.Context, docTree processor.DocumentTree) ([]assembler.IngestPredicates, []*common.IdentifierStrings, error) { +func ParseDocumentTree(ctx context.Context, docTree processor.DocumentTree, gqlClient graphql.Client) ([]assembler.IngestPredicates, []*common.IdentifierStrings, error) { assemblerInputs := []assembler.IngestPredicates{} identifierStrings := []*common.IdentifierStrings{} logger := docTree.Document.ChildLogger docTreeBuilder := newDocTreeBuilder() logger.Infof("parsing document tree with root type: %v", docTree.Document.Type) - err := docTreeBuilder.parse(ctx, docTree, map[visitedKey]bool{}) + err := docTreeBuilder.parse(ctx, docTree, map[visitedKey]bool{}, gqlClient) if err != nil { return nil, nil, err } @@ -107,8 +108,8 @@ type visitedKey struct { } // The visited map is used to keep track of the document nodes that have already been visited to avoid infinite loops. -func (t *docTreeBuilder) parse(ctx context.Context, root processor.DocumentTree, visited map[visitedKey]bool) error { - builder, err := parseHelper(ctx, root.Document) +func (t *docTreeBuilder) parse(ctx context.Context, root processor.DocumentTree, visited map[visitedKey]bool, gqlClient graphql.Client) error { + builder, err := parseHelper(ctx, root.Document, gqlClient) if err != nil { return err } @@ -123,20 +124,20 @@ func (t *docTreeBuilder) parse(ctx context.Context, root processor.DocumentTree, t.identities = append(t.identities, builder.GetIdentities()...) for _, c := range root.Children { - if err := t.parse(ctx, c, visited); err != nil { + if err := t.parse(ctx, c, visited, gqlClient); err != nil { return err } } return nil } -func parseHelper(ctx context.Context, doc *processor.Document) (*common.GraphBuilder, error) { +func parseHelper(ctx context.Context, doc *processor.Document, gqlClient graphql.Client) (*common.GraphBuilder, error) { pFunc, ok := documentParser[doc.Type] if !ok { return nil, fmt.Errorf("no document parser registered for type: %s", doc.Type) } - p := pFunc() + p := pFunc(gqlClient) err := p.Parse(ctx, doc) if err != nil { return nil, err diff --git a/pkg/ingestor/parser/parser_test.go b/pkg/ingestor/parser/parser_test.go index 819c00fc2c..fb4a4a9506 100644 --- a/pkg/ingestor/parser/parser_test.go +++ b/pkg/ingestor/parser/parser_test.go @@ -21,9 +21,9 @@ import ( "reflect" "testing" - "github.com/guacsec/guac/pkg/logging" - + "github.com/Khan/genqlient/graphql" "github.com/guacsec/guac/pkg/assembler" + "github.com/guacsec/guac/pkg/logging" "github.com/guacsec/guac/internal/testing/mocks" @@ -74,7 +74,7 @@ func TestParserHelper(t *testing.T) { parser := common.DocumentParser(mockDocumentParser) - f := func() common.DocumentParser { + f := func(gqlClient graphql.Client) common.DocumentParser { return parser } @@ -88,7 +88,7 @@ func TestParserHelper(t *testing.T) { _ = RegisterDocumentParser(f, test.registerArgs) // Ignoring error because it is mutating a global variable - if _, err := parseHelper(ctx, test.parseArg); err != nil { // Ignoring the graphBuilder because the mock will always return an empty graphBuilder + if _, err := parseHelper(ctx, test.parseArg, nil); err != nil { // Ignoring the graphBuilder because the mock will always return an empty graphBuilder t.Logf("error parsing document: %v", err) } }) @@ -181,7 +181,7 @@ func Test_docTreeBuilder_parse(t *testing.T) { parser := common.DocumentParser(mockDocumentParser) - f := func() common.DocumentParser { + f := func(gqlClient graphql.Client) common.DocumentParser { return parser } @@ -208,7 +208,7 @@ func Test_docTreeBuilder_parse(t *testing.T) { identities: test.fields.identities, graphBuilders: test.fields.graphBuilders, } - if err := treeBuilder.parse(ctx, test.root, map[visitedKey]bool{}); (err != nil) != test.wantErr { + if err := treeBuilder.parse(ctx, test.root, map[visitedKey]bool{}, nil); (err != nil) != test.wantErr { t1.Errorf("parse() error = %v, wantErr %v", err, test.wantErr) } }) @@ -275,7 +275,7 @@ func TestParseDocumentTree(t *testing.T) { parser := common.DocumentParser(mockDocumentParser) - f := func() common.DocumentParser { + f := func(gqlClient graphql.Client) common.DocumentParser { return parser } @@ -293,7 +293,7 @@ func TestParseDocumentTree(t *testing.T) { _ = RegisterDocumentParser(f, test.registerDocType) // Ignoring error because it is mutating a global variable - got, got1, err := ParseDocumentTree(ctx, test.docTree) + got, got1, err := ParseDocumentTree(ctx, test.docTree, nil) if (err != nil) != test.wantErr { t.Errorf("ParseDocumentTree() error = %v, wantErr %v", err, test.wantErr) diff --git a/pkg/ingestor/parser/scorecard/parser_scorecard.go b/pkg/ingestor/parser/scorecard/parser_scorecard.go index 145e75549c..f102e95600 100644 --- a/pkg/ingestor/parser/scorecard/parser_scorecard.go +++ b/pkg/ingestor/parser/scorecard/parser_scorecard.go @@ -18,6 +18,7 @@ package scorecard import ( "context" "fmt" + "github.com/Khan/genqlient/graphql" "strings" "time" @@ -38,8 +39,8 @@ type scorecardParser struct { vcsStrings []string } -// NewSLSAParser initializes the slsaParser -func NewScorecardParser() common.DocumentParser { +// NewScorecardParser initializes the scorecardParser +func NewScorecardParser(gqlClient graphql.Client) common.DocumentParser { return &scorecardParser{} } diff --git a/pkg/ingestor/parser/scorecard/parser_scorecard_test.go b/pkg/ingestor/parser/scorecard/parser_scorecard_test.go index 45d0758084..9b0d70e440 100644 --- a/pkg/ingestor/parser/scorecard/parser_scorecard_test.go +++ b/pkg/ingestor/parser/scorecard/parser_scorecard_test.go @@ -81,7 +81,7 @@ func Test_scorecardParser(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := NewScorecardParser() + s := NewScorecardParser(nil) if err := s.Parse(ctx, tt.doc); (err != nil) != tt.wantErr { t.Errorf("scorecard.Parse() error = %v, wantErr %v", err, tt.wantErr) } diff --git a/pkg/ingestor/parser/slsa/parser_slsa.go b/pkg/ingestor/parser/slsa/parser_slsa.go index 61a7649221..cf1d119b5b 100644 --- a/pkg/ingestor/parser/slsa/parser_slsa.go +++ b/pkg/ingestor/parser/slsa/parser_slsa.go @@ -19,6 +19,7 @@ import ( "context" "errors" "fmt" + "github.com/Khan/genqlient/graphql" "strings" jsoniter "github.com/json-iterator/go" @@ -68,7 +69,7 @@ type slsaParser struct { } // NewSLSAParser initializes the slsaParser -func NewSLSAParser() common.DocumentParser { +func NewSLSAParser(gqlClient graphql.Client) common.DocumentParser { return &slsaParser{ identifierStrings: &common.IdentifierStrings{}, } diff --git a/pkg/ingestor/parser/slsa/parser_slsa_test.go b/pkg/ingestor/parser/slsa/parser_slsa_test.go index 9289f3e343..2d94afc032 100644 --- a/pkg/ingestor/parser/slsa/parser_slsa_test.go +++ b/pkg/ingestor/parser/slsa/parser_slsa_test.go @@ -55,7 +55,7 @@ func Test_slsaParser(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := NewSLSAParser() + s := NewSLSAParser(nil) err := s.Parse(ctx, tt.doc) if (err != nil) != tt.wantErr { t.Errorf("slsa.Parse() error = %v, wantErr %v", err, tt.wantErr) diff --git a/pkg/ingestor/parser/spdx/parse_spdx.go b/pkg/ingestor/parser/spdx/parse_spdx.go index 0577c79b33..cb5b586b5e 100644 --- a/pkg/ingestor/parser/spdx/parse_spdx.go +++ b/pkg/ingestor/parser/spdx/parse_spdx.go @@ -19,6 +19,7 @@ import ( "bytes" "context" "fmt" + "github.com/Khan/genqlient/graphql" "slices" "strings" "time" @@ -49,7 +50,7 @@ type spdxParser struct { timeScanned time.Time } -func NewSpdxParser() common.DocumentParser { +func NewSpdxParser(gqlClient graphql.Client) common.DocumentParser { return &spdxParser{ packagePackages: map[string][]*model.PkgInputSpec{}, packageArtifacts: map[string][]*model.ArtifactInputSpec{}, diff --git a/pkg/ingestor/parser/spdx/parse_spdx_test.go b/pkg/ingestor/parser/spdx/parse_spdx_test.go index 5c12538730..476e35cc7f 100644 --- a/pkg/ingestor/parser/spdx/parse_spdx_test.go +++ b/pkg/ingestor/parser/spdx/parse_spdx_test.go @@ -1099,7 +1099,7 @@ func Test_spdxParser(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := NewSpdxParser() + s := NewSpdxParser(nil) err := s.Parse(ctx, tt.doc) if (err != nil) != tt.wantErr { t.Errorf("spdxParser.Parse() error = %v, wantErr %v", err, tt.wantErr) diff --git a/pkg/ingestor/parser/vuln/vuln.go b/pkg/ingestor/parser/vuln/vuln.go index 8b80347463..84470aa6ca 100644 --- a/pkg/ingestor/parser/vuln/vuln.go +++ b/pkg/ingestor/parser/vuln/vuln.go @@ -33,6 +33,7 @@ package vuln import ( "context" "fmt" + "github.com/Khan/genqlient/graphql" "strings" jsoniter "github.com/json-iterator/go" @@ -57,7 +58,7 @@ type parser struct { var noVulnInput *generated.VulnerabilityInputSpec = &generated.VulnerabilityInputSpec{Type: "noVuln", VulnerabilityID: ""} // NewVulnCertificationParser initializes the parser -func NewVulnCertificationParser() common.DocumentParser { +func NewVulnCertificationParser(gqlClient graphql.Client) common.DocumentParser { return &parser{} } diff --git a/pkg/ingestor/parser/vuln/vuln_test.go b/pkg/ingestor/parser/vuln/vuln_test.go index de10c3ca64..d61ff277bb 100644 --- a/pkg/ingestor/parser/vuln/vuln_test.go +++ b/pkg/ingestor/parser/vuln/vuln_test.go @@ -289,7 +289,7 @@ func TestParser(t *testing.T) { }) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := NewVulnCertificationParser() + s := NewVulnCertificationParser(nil) err := s.Parse(ctx, tt.doc) if (err != nil) != tt.wantErr { t.Fatalf("parser.Parse() error = %v, wantErr %v", err, tt.wantErr)