Fix repo crawling

vinted · Aug 7, 2024 · 6a8af5f · 6a8af5f
1 parent 5954c46
commit 6a8af5f
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 16 deletions.
diff --git a/internal/requests.go b/internal/requests.go
@@ -31,6 +31,10 @@ type GetRepositoriesConfig struct {
 }
 
 func NewGetRepositoriesConfig(ctx context.Context, url, username, apiToken string, org string) GetRepositoriesConfig {
+	if org != "" {
+		apiToken = gh.GenerateGithubAppTokenInternal(org)
+	}
+
 	return GetRepositoriesConfig{
 		ctx:                         ctx,
 		URL:                         url,
@@ -139,15 +143,8 @@ func GetRepositories(conf GetRepositoriesConfig) ([]repositoryMapping, error) {
 			return repositories, nil
 		}
 
-		var validRepositories []repositoryMapping
-		for _, r := range repositories {
-			if !r.Archived {
-				validRepositories = append(validRepositories, r)
-			}
-		}
-		return validRepositories, nil
+		return repositories, nil
 	}
-
 	return exponentialBackoff(getRepositories, conf.BackoffPolicy...)
 }
 
@@ -167,7 +164,7 @@ func WalkRepositories(conf GetRepositoriesConfig, callback func(repositoryURLs [
 		query.Set("page", strconv.Itoa(page))
 		endpoint.RawQuery = query.Encode()
 		conf.URL = endpoint.String()
-
+		log.WithField("request github", endpoint.String()).Infof("Getting query for page %d", page)
 		repositories, err = GetRepositories(conf)
 		if err != nil {
 			if regenCount < 1 {
@@ -192,7 +189,18 @@ func WalkRepositories(conf GetRepositoriesConfig, callback func(repositoryURLs [
 			regenCount = 0
 		}
 
-		if len(repositories) == 0 {
+		var validRepositories []repositoryMapping
+		var archivedRepositories []repositoryMapping
+		for _, r := range repositories {
+			if !r.Archived {
+				validRepositories = append(validRepositories, r)
+			} else if r.Archived {
+				archivedRepositories = append(archivedRepositories, r)
+			}
+		}
+
+		if len(repositories) == 0 && len(archivedRepositories) == 0 {
+			log.WithField("request github return", endpoint.String()).Infof("returning with page %d", page)
 			return nil // Done, all repositories have been walked
 		}
 

diff --git a/pkg/bomtools/merge.go b/pkg/bomtools/merge.go
@@ -201,8 +201,17 @@ func mergeAllByPURL(component *cdx.Component, allComponents []*cdx.Component) *c
 			mergedComponent.Properties = &p
 		}
 		if c.Licenses != nil {
-			l := mergeCollection[cdx.LicenseChoice](*c.Licenses, *mergedComponent.Licenses)
-			mergedComponent.Licenses = (*cdx.Licenses)(&l)
+			l := make([]cdx.LicenseChoice, 0)
+			for _, sl := range *c.Licenses {
+				// Check for license ID
+				if sl.License != nil && sl.License.ID != "" {
+					l = append(l, sl)
+				}
+			}
+
+			// Assuming mergedComponent.Licenses is initialized properly earlier
+			mergedLicenses := mergeCollection[cdx.LicenseChoice](l, *mergedComponent.Licenses)
+			mergedComponent.Licenses = (*cdx.Licenses)(&mergedLicenses)
 		}
 		if c.ExternalReferences != nil {
 			e := mergeCollection[cdx.ExternalReference](*c.ExternalReferences, *mergedComponent.ExternalReferences)

diff --git a/pkg/dtrack/http.go b/pkg/dtrack/http.go
@@ -67,11 +67,12 @@ func (d DependencyTrackClient) createProject(ctx context.Context, payload create
 	d.setRequiredHeaders(req)
 
 	resp, err := http.DefaultClient.Do(req)
-	log.WithField("funcType", "createProject").Debugf("CreateProject request response body: %s", resp.Body)
-	log.WithField("funcType", "createProject").Debugf("CreateProject request response status code: %v", resp.StatusCode)
 	if err != nil {
+		log.WithField("funcType", "createProject").Debugf("CreateProject error %v", err.Error())
 		return "", fmt.Errorf(cantPerformHTTPRequest, requestURL, err)
 	}
+	// putting err here incase we get a resp nil
+	log.WithField("funcType", "createProject").Debugf("CreateProject request response status code: %v", resp.StatusCode)
 
 	defer func() {
 		closeErr := resp.Body.Close()
@@ -125,11 +126,11 @@ func (d DependencyTrackClient) updateSBOMs(ctx context.Context, payload updateSB
 	d.setRequiredHeaders(req)
 
 	resp, err := http.DefaultClient.Do(req)
-	log.WithField("funcType", "updateSBOM").Debugf("CreateProject request response body: %s", resp.Body)
-	log.WithField("funcType", "updateSBOM").Debugf("CreateProject request response status code: %v", resp.StatusCode)
 	if err != nil {
 		return fmt.Errorf(cantPerformHTTPRequest, requestURL, err)
 	}
+	log.WithField("funcType", "updateSBOM").Debugf("CreateProject request response body: %s", resp.Body)
+	log.WithField("funcType", "updateSBOM").Debugf("CreateProject request response status code: %v", resp.StatusCode)
 
 	defer func() {
 		closeErr := resp.Body.Close()