Skip to content

Commit

Permalink
feat(scanner): add keppel scanner (#151)
Browse files Browse the repository at this point in the history
* feat(scanner): init keppel scanner
  • Loading branch information
MR2011 committed Aug 22, 2024
1 parent cefb901 commit 946402c
Show file tree
Hide file tree
Showing 17 changed files with 1,494 additions and 3 deletions.
641 changes: 641 additions & 0 deletions scanner/keppel/client/generated.go

Large diffs are not rendered by default.

11 changes: 11 additions & 0 deletions scanner/keppel/client/genqlient.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
# SPDX-License-Identifier: Apache-2.0

# Default genqlient config; for full documentation see:
# https://github.com/Khan/genqlient/blob/main/docs/genqlient.yaml
schema: ../../../internal/api/graphql/graph/schema/*.graphqls
operations:
- ./query/*.graphql
generated: generated.go
package: client
use_struct_references: true
15 changes: 15 additions & 0 deletions scanner/keppel/client/query/addComponentVersionToIssue.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
# SPDX-License-Identifier: Apache-2.0

mutation AddComponentVersionToIssue($issueId: ID!, $componentVersionId: ID!) {
# @genqlient(typename: "Issue")
addComponentVersionToIssue (
issueId: $issueId,
componentVersionId: $componentVersionId
) {
id
primaryName
description
type
}
}
13 changes: 13 additions & 0 deletions scanner/keppel/client/query/createComponent.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
# SPDX-License-Identifier: Apache-2.0

mutation CreateComponent($input: ComponentInput!) {
# @genqlient(typename: "Component")
createComponent (
input: $input
) {
id
name
type
}
}
13 changes: 13 additions & 0 deletions scanner/keppel/client/query/createComponentVersion.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
# SPDX-License-Identifier: Apache-2.0

mutation CreateComponentVersion($input: ComponentVersionInput!) {
# @genqlient(typename: "ComponentVersion")
createComponentVersion (
input: $input
) {
id
version
componentId
}
}
14 changes: 14 additions & 0 deletions scanner/keppel/client/query/createIssue.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
# SPDX-License-Identifier: Apache-2.0

mutation CreateIssue($input: IssueInput!) {
# @genqlient(typename: "Issue")
createIssue (
input: $input
) {
id
primaryName
description
type
}
}
19 changes: 19 additions & 0 deletions scanner/keppel/client/query/listComponentVersions.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
# SPDX-License-Identifier: Apache-2.0

query ListComponentVersions($filter: ComponentVersionFilter, $first: Int) {
# @genqlient(typename: "ComponentVersionConnection")
ComponentVersions (
filter: $filter,
first: $first,
) {
edges {
# @genqlient(typename: "ComponentVersion")
node {
id
version
componentId
}
}
}
}
19 changes: 19 additions & 0 deletions scanner/keppel/client/query/listComponents.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
# SPDX-License-Identifier: Apache-2.0

query ListComponents($filter: ComponentFilter, $first: Int) {
# @genqlient(typename: "ComponentConnection")
Components (
filter: $filter,
first: $first,
) {
edges {
# @genqlient(typename: "Component")
node {
id
name
type
}
}
}
}
20 changes: 20 additions & 0 deletions scanner/keppel/client/query/listIssues.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
# SPDX-License-Identifier: Apache-2.0

query ListIssues($filter: IssueFilter, $first: Int) {
# @genqlient(typename: "IssueConnection")
Issues (
filter: $filter,
first: $first,
) {
edges {
# @genqlient(typename: "Issue")
node {
id
primaryName
description
type
}
}
}
}
34 changes: 33 additions & 1 deletion scanner/keppel/go.mod
Original file line number Diff line number Diff line change
@@ -1,3 +1,35 @@
module github.com/cloudoperators/heureka/scanners/keppel

go 1.22.6
go 1.22.6

require (
github.com/aquasecurity/trivy v0.54.1
github.com/gophercloud/gophercloud v1.14.0
github.com/gophercloud/gophercloud/v2 v2.1.0
github.com/kelseyhightower/envconfig v1.4.0
github.com/pkg/errors v0.9.1
)

require (
github.com/Khan/genqlient v0.7.0 // indirect
github.com/agnivade/levenshtein v1.1.1 // indirect
github.com/alexflint/go-arg v1.4.2 // indirect
github.com/alexflint/go-scalar v1.0.0 // indirect
github.com/vektah/gqlparser/v2 v2.5.11 // indirect
golang.org/x/mod v0.19.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/tools v0.23.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

require (
github.com/aquasecurity/trivy-db v0.0.0-20240718084044-d23a6ca8ba04 // indirect
github.com/google/go-containerregistry v0.20.1 // indirect
github.com/machinebox/graphql v0.2.2
github.com/package-url/packageurl-go v0.1.3 // indirect
github.com/samber/lo v1.46.0 // indirect
github.com/sirupsen/logrus v1.9.3
golang.org/x/text v0.16.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
)
65 changes: 65 additions & 0 deletions scanner/keppel/go.sum
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
github.com/Khan/genqlient v0.7.0 h1:GZ1meyRnzcDTK48EjqB8t3bcfYvHArCUUvgOwpz1D4w=
github.com/Khan/genqlient v0.7.0/go.mod h1:HNyy3wZvuYwmW3Y7mkoQLZsa/R5n5yIRajS1kPBvSFM=
github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8=
github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo=
github.com/alexflint/go-arg v1.4.2 h1:lDWZAXxpAnZUq4qwb86p/3rIJJ2Li81EoMbTMujhVa0=
github.com/alexflint/go-arg v1.4.2/go.mod h1:9iRbDxne7LcR/GSvEr7ma++GLpdIU1zrghf2y2768kM=
github.com/alexflint/go-scalar v1.0.0 h1:NGupf1XV/Xb04wXskDFzS0KWOLH632W/EO4fAFi+A70=
github.com/alexflint/go-scalar v1.0.0/go.mod h1:GpHzbCOZXEKMEcygYQ5n/aa4Aq84zbxjy3MxYW0gjYw=
github.com/aquasecurity/trivy v0.54.1 h1:/uNCF06PfdC69v5n3Zh4fXVf0xmXBml0c/ergf066SQ=
github.com/aquasecurity/trivy v0.54.1/go.mod h1:i5S54WUtOEN9egFF0AHsxq6XT7QD11n9pSmIXhMJV0g=
github.com/aquasecurity/trivy-db v0.0.0-20240718084044-d23a6ca8ba04 h1:6/T8sFdNVG/AwOGoK6X55h7hF7LYqK8bsuPz8iEz8jM=
github.com/aquasecurity/trivy-db v0.0.0-20240718084044-d23a6ca8ba04/go.mod h1:0T6oy2t1Iedt+yi3Ml5cpOYp5FZT4MI1/mx+3p+PIs8=
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
github.com/google/go-containerregistry v0.20.1 h1:eTgx9QNYugV4DN5mz4U8hiAGTi1ybXn0TPi4Smd8du0=
github.com/google/go-containerregistry v0.20.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI=
github.com/gophercloud/gophercloud v1.14.0 h1:Bt9zQDhPrbd4qX7EILGmy+i7GP35cc+AAL2+wIJpUE8=
github.com/gophercloud/gophercloud v1.14.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM=
github.com/gophercloud/gophercloud/v2 v2.1.0 h1:91p6c+uMckXyx39nSIYjDirDBnPVFQq0q1njLNPX+NY=
github.com/gophercloud/gophercloud/v2 v2.1.0/go.mod h1:f2hMRC7Kakbv5vM7wSGHrIPZh6JZR60GVHryJlF/K44=
github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
github.com/machinebox/graphql v0.2.2 h1:dWKpJligYKhYKO5A2gvNhkJdQMNZeChZYyBbrZkBZfo=
github.com/machinebox/graphql v0.2.2/go.mod h1:F+kbVMHuwrQ5tYgU9JXlnskM8nOaFxCAEolaQybkjWA=
github.com/package-url/packageurl-go v0.1.3 h1:4juMED3hHiz0set3Vq3KeQ75KD1avthoXLtmE3I0PLs=
github.com/package-url/packageurl-go v0.1.3/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/samber/lo v1.46.0 h1:w8G+oaCPgz1PoCJztqymCFaKwXt+5cCXn51uPxExFfQ=
github.com/samber/lo v1.46.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/vektah/gqlparser/v2 v2.5.11 h1:JJxLtXIoN7+3x6MBdtIP59TP1RANnY7pXOaDnADQSf8=
github.com/vektah/gqlparser/v2 v2.5.11/go.mod h1:1rCcfwB2ekJofmluGWXMSEnPMZgbxzwj6FaZ/4OT8Cc=
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
143 changes: 141 additions & 2 deletions scanner/keppel/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,148 @@
package main

import (
"fmt"
"os"
"sync"

"github.com/cloudoperators/heureka/scanners/keppel/client"
"github.com/cloudoperators/heureka/scanners/keppel/models"
"github.com/cloudoperators/heureka/scanners/keppel/processor"
"github.com/cloudoperators/heureka/scanners/keppel/scanner"
"github.com/kelseyhightower/envconfig"
log "github.com/sirupsen/logrus"
)

type Config struct {
LogLevel string `envconfig:"LOG_LEVEL" default:"debug" required:"true" json:"-"`
}

func init() {
// Log as JSON instead of the default ASCII formatter.
log.SetFormatter(&log.JSONFormatter{})

// Output to stdout instead of the default stderr
// Can be any io.Writer, see below for File example
log.SetOutput(os.Stdout)

var cfg Config
err := envconfig.Process("heureka", &cfg)
if err != nil {
log.WithError(err).Fatal("Error while reading env config")
}

level, err := log.ParseLevel(cfg.LogLevel)

if err != nil {
log.WithError(err).Fatal("Error while parsing log level")
}

// Only log the warning severity or above.
log.SetLevel(level)
}

func main() {
fmt.Println("This is the Keppel scanner")
var wg sync.WaitGroup
var scannerCfg scanner.Config
err := envconfig.Process("heureka", &scannerCfg)
if err != nil {
log.WithError(err).Fatal("Error while reading env config for scanner")
}

var processorCfg processor.Config
err = envconfig.Process("heureka", &processorCfg)
if err != nil {
log.WithError(err).Fatal("Error while reading env config for processor")
}

keppelScanner := scanner.NewScanner(scannerCfg)
keppelProcessor := processor.NewProcessor(processorCfg)

err = keppelScanner.Setup()
if err != nil {
log.WithError(err).Fatal("Error during scanner setup")
}
accounts, err := keppelScanner.ListAccounts()
if err != nil {
log.WithError(err).Fatal("Error during ListAccounts")
}

wg.Add(len(accounts))

for _, account := range accounts {
go HandleAccount(account, keppelScanner, keppelProcessor, &wg)
}

wg.Wait()
}

func HandleAccount(account models.Account, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor, wg *sync.WaitGroup) error {
defer wg.Done()
repositories, err := keppelScanner.ListRepositories(account.Name)
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
}).WithError(err).Error("Error during ProcessRepository")
return err
}

for _, repository := range repositories {
HandleRepository(account, repository, keppelScanner, keppelProcessor)
}

return nil
}

func HandleRepository(account models.Account, repository models.Repository, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) {
component, err := keppelProcessor.ProcessRepository(repository)
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).WithError(err).Error("Error during ProcessRepository")
component, err = keppelProcessor.GetComponent(repository.Name)
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).WithError(err).Error("Error during GetComponent")
}
}

manifests, err := keppelScanner.ListManifests(account.Name, repository.Name)
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).WithError(err).Error("Error during ListManifests")
return
}
for _, manifest := range manifests {
HandleManifest(account, repository, manifest, component, keppelScanner, keppelProcessor)
}
}

func HandleManifest(account models.Account, repository models.Repository, manifest models.Manifest, component *client.Component, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) {
componentVersion, err := keppelProcessor.ProcessManifest(manifest, component.Id)
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).WithError(err).Error("Error during ProcessManifest")
componentVersion, err = keppelProcessor.GetComponentVersion(manifest.Digest)
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).WithError(err).Error("Error during GetComponentVersion")
}
}
trivyReport, err := keppelScanner.GetTrivyReport(account.Name, repository.Name, manifest.Digest)
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).WithError(err).Error("Error during GetTrivyReport")
return
}
keppelProcessor.ProcessReport(*trivyReport, componentVersion.Id)
}
Loading

0 comments on commit 946402c

Please sign in to comment.