feat(nvd): allow fetching of issues from 2021 to now #247 (#250)

* chore:  WIP for basic matching

* chore: updated eventHandler initialization in tests

* Change EventHandler signature and logic

* Fix tests

* Add more tests regarding concurrent processing

* chore: refactored function for lower coginitive complexity & fixed logic

* First implementation

* Wip

* Fix tests

* Fixing tests part 2

* Implement tests for BuildIssueVariantMap

* Fix tests

* Wip

* chore(deps): bump github.com/prometheus/client_golang (#210)

Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.20.2 to 1.20.3.
- [Release notes](https://github.com/prometheus/client_golang/releases)
- [Changelog](https://github.com/prometheus/client_golang/blob/v1.20.3/CHANGELOG.md)
- [Commits](prometheus/client_golang@v1.20.2...v1.20.3)

---
updated-dependencies:
- dependency-name: github.com/prometheus/client_golang
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* chore(deps): bump golang from 1.23.0 to 1.23.1 (#211)

Bumps golang from 1.23.0 to 1.23.1.

---
updated-dependencies:
- dependency-name: golang
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* Add tests for handling issue repositories with different priorities

* chore: removed changes unrelated to this feature

* fix: fixed imports

* fix: fixed imports

* Improve tests

* resolve issues

* feat(scanner/nvd): Adding possibility to fetch NVD from 2001

* feat: made time window configurable

---------

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: Victor Dorneanu <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Michael Reimsbach <[email protected]>

internal/app/issue_repository/issue_repository_handler_events.go

@@ -4,10 +4,10 @@
 package issue_repository
 
 import (
+	"github.com/sirupsen/logrus"
 	"github.com/cloudoperators/heureka/internal/app/event"
 	"github.com/cloudoperators/heureka/internal/database"
 	"github.com/cloudoperators/heureka/internal/entity"
-	"github.com/sirupsen/logrus"
 )
 
 const (

internal/app/service/service_handler_events.go

@@ -8,7 +8,6 @@ import (
 	"github.com/cloudoperators/heureka/internal/app/event"
 	"github.com/cloudoperators/heureka/internal/database"
 	"github.com/cloudoperators/heureka/internal/entity"
-	"github.com/sirupsen/logrus"
 )
 
 const (

scanner/nvd/main.go

@@ -5,14 +5,13 @@ package main
 
 import (
 	"fmt"
-	"os"
-	"time"
-
+	"github.com/cloudoperators/heureka/scanner/nvd/models"
+	p "github.com/cloudoperators/heureka/scanner/nvd/processor"
+	s "github.com/cloudoperators/heureka/scanner/nvd/scanner"
 	"github.com/kelseyhightower/envconfig"
 	log "github.com/sirupsen/logrus"
-	"github.com/cloudoperators/heureka/scanner/nvd/models"
-	"github.com/cloudoperators/heureka/scanner/nvd/processor"
-	"github.com/cloudoperators/heureka/scanner/nvd/scanner"
+	"os"
+	"time"
 )
 
 func init() {
@@ -30,22 +29,36 @@ func init() {
 	log.SetReportCaller(true)
 }
 
-func main() {
-	var scannerCfg scanner.Config
-	err := envconfig.Process("heureka", &scannerCfg)
+func startTimeWindow(scanner *s.Scanner, processor *p.Processor, config s.Config) error {
+
+	startTime, err := time.Parse("2006-01-02", config.StartDate)
+
+	absoluteEnd := time.Now()
+	if config.EndDate != "" {
+		absoluteEnd, err = time.Parse("2006-01-02", config.EndDate)
+	}
+
 	if err != nil {
-		log.WithFields(log.Fields{
-			"errror": err,
-		}).Warn("Couldn't initialize scanner config")
+		return err
 	}
-	scanner := scanner.NewScanner(scannerCfg)
 
-	t := time.Now()
-	yearToday, monthToday, dayToday := time.Now().Date()
-	today := fmt.Sprintf("%d-%02d-%02dT23:59:59.000", yearToday, monthToday, dayToday)
-	yearYesterday, monthYesterday, dayYesterday := t.AddDate(0, 0, -1).Date()
-	yesterday := fmt.Sprintf("%d-%02d-%02dT00:00:00.000", yearYesterday, monthYesterday, dayYesterday)
+	endTime := startTime.AddDate(0, 2, 0)
 
+	for endTime.Before(absoluteEnd) {
+		startYear, startMonth, startDay := startTime.Date()
+		endYear, endMonth, endDay := endTime.Date()
+		start := fmt.Sprintf("%d-%02d-%02dT23:59:59.000", startYear, startMonth, startDay)
+		end := fmt.Sprintf("%d-%02d-%02dT23:59:59.000", endYear, endMonth, endDay)
+
+		scanAndProcess(scanner, processor, start, end)
+
+		startTime = startTime.AddDate(0, 2, 0)
+		endTime = endTime.AddDate(0, 2, 0)
+	}
+	return nil
+}
+
+func scanAndProcess(scanner *s.Scanner, processor *p.Processor, yesterday string, today string) {
 	filter := models.CveFilter{
 		PubStartDate: yesterday,
 		PubEndDate:   today,
@@ -58,29 +71,60 @@ func main() {
 		}).Error("Couldn't get CVEs")
 	}
 
-	var processorCfg processor.Config
+	for _, cve := range cves {
+		err = processor.Process(&cve.Cve)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error": err,
+				"CVEID": &cve.Cve.Id,
+			}).Warn("Couldn't process CVE")
+		}
+	}
+}
+
+func main() {
+	var err error
+	var scannerCfg s.Config
+	err = envconfig.Process("heureka", &scannerCfg)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error": err,
+		}).Warn("Couldn't initialize scanner config")
+	}
+	scanner := s.NewScanner(scannerCfg)
+
+	var processorCfg p.Config
 	err = envconfig.Process("heureka", &processorCfg)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"error": err,
 		}).Error("Couldn't configure new processor")
 	}
 
-	processor := processor.NewProcessor(processorCfg)
+	processor := p.NewProcessor(processorCfg)
 	err = processor.Setup()
 	if err != nil {
 		log.WithFields(log.Fields{
 			"error": err,
 		}).Error("Couldn't setup new processor")
 	}
 
-	for _, cve := range cves {
-		err = processor.Process(&cve.Cve)
+	if scannerCfg.StartDate != "" {
+		err = startTimeWindow(scanner, processor, scannerCfg)
 		if err != nil {
 			log.WithFields(log.Fields{
 				"error": err,
-				"CVEID": &cve.Cve.Id,
-			}).Warn("Couldn't process CVE")
+			}).Error("Couldn't fetch CVEs for time window")
 		}
+	} else {
+		t := time.Now()
+		yearToday, monthToday, dayToday := time.Now().Date()
+		today := fmt.Sprintf("%d-%02d-%02dT23:59:59.000", yearToday, monthToday, dayToday)
+
+		yearYesterday, monthYesterday, dayYesterday := t.AddDate(0, 0, -2).Date()
+		yesterday := fmt.Sprintf("%d-%02d-%02dT00:00:00.000", yearYesterday, monthYesterday, dayYesterday)
+
+		scanAndProcess(scanner, processor, yesterday, today)
 	}
+
 }

scanner/nvd/processor/processor.go

@@ -60,7 +60,6 @@ func (p *Processor) Setup() error {
 			"issueRepositoryId": p.IssueRepositoryId,
 		}).Info("Created new IssueRepository")
 	} else {
-
 		// Extract IssueRepositoryId
 		for _, ir := range listRepositoriesResp.IssueRepositories.Edges {
 			log.Debugf("nodeId: %s", ir.Node.Id)

scanner/nvd/scanner/config.go

@@ -4,6 +4,8 @@
 package scanner
 
 type Config struct {
+	StartDate string `envconfig:"NVD_START_DATE" default:"" json:"-"`
+	EndDate   string `envconfig:"NVD_END_DATE" default:"" json:"-"`
 	NvdApiUrl string `envconfig:"NVD_API_URL" required:"true" json:"-"`
 	NvdApiKey string `envconfig:"NVD_API_KEY" required:"true" json:"-"`
 	// default value and maximum allowable limit is 2,000