Fix Packagist JSON parsing to correctly parse dist fields. (#880)

* Fix Packagist JSON parsing to correctly parse dist fields. Also handle the scenario where a download url is not available. Signed-off-by: Caleb Brown <[email protected]> * Fix an infinite loop and some minor nits. Signed-off-by: Caleb Brown <[email protected]> --------- Signed-off-by: Caleb Brown <[email protected]>
ossf · Sep 13, 2023 · 32b06bc · 32b06bc
1 parent 2677f52
commit 32b06bc
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 13 deletions.
diff --git a/internal/pkgmanager/ecosystem.go b/internal/pkgmanager/ecosystem.go
@@ -1,6 +1,7 @@
 package pkgmanager
 
 import (
+	"errors"
 	"fmt"
 	"path"
 	"path/filepath"
@@ -9,6 +10,8 @@ import (
 	"github.com/ossf/package-analysis/pkg/api/pkgecosystem"
 )
 
+var ErrNoArchiveURL = errors.New("archive URL not found")
+
 // PkgManager represents how packages from a common ecosystem are accessed.
 type PkgManager struct {
 	ecosystem       pkgecosystem.Ecosystem
@@ -102,7 +105,7 @@ func (p *PkgManager) DownloadArchive(name, version, directory string) (string, e
 		return "", err
 	}
 	if downloadURL == "" {
-		return "", fmt.Errorf("no url found for package %s, version %s", name, version)
+		return "", fmt.Errorf("%w: package %s @ %s", ErrNoArchiveURL, name, version)
 	}
 
 	baseFilename := p.archiveFilename(name, version, downloadURL)

diff --git a/internal/pkgmanager/packagist.go b/internal/pkgmanager/packagist.go
@@ -10,19 +10,32 @@ import (
 	"github.com/ossf/package-analysis/pkg/api/pkgecosystem"
 )
 
+type packagistDistJSON struct {
+	URL       string `json:"url"`
+	Type      string `json:"type"`
+	Shasum    string `json:"shasum,omitempty"`
+	Reference string `json:"reference"`
+}
+
+func (d *packagistDistJSON) UnmarshalJSON(data []byte) error {
+	switch string(data) {
+	case "null":
+		return nil
+	case `"__unset"`:
+		return nil
+	}
+	type raw packagistDistJSON
+	return json.Unmarshal(data, (*raw)(d))
+}
+
 type packagistJSON struct {
 	Packages map[string][]struct {
-		Version           string    `json:"version"`
-		VersionNormalized string    `json:"version_normalized"`
-		License           []string  `json:"license,omitempty"`
-		Time              time.Time `json:"time"`
-		Name              string    `json:"name,omitempty"`
-		Dist              struct {
-			URL       string `json:"url"`
-			Type      string `json:"type"`
-			Shasum    string `json:"shasum,omitempty"`
-			Reference string `json:"reference"`
-		} `json:"dist"`
+		Version           string            `json:"version"`
+		VersionNormalized string            `json:"version_normalized"`
+		License           []string          `json:"license,omitempty"`
+		Time              time.Time         `json:"time"`
+		Name              string            `json:"name,omitempty"`
+		Dist              packagistDistJSON `json:"dist"`
 	} `json:"packages"`
 }
 

diff --git a/internal/resultstore/resultstore.go b/internal/resultstore/resultstore.go
@@ -143,7 +143,10 @@ func (rs *ResultStore) SaveTempFilesToZip(ctx context.Context, p Pkg, zipName st
 
 func (rs *ResultStore) SaveAnalyzedPackage(ctx context.Context, manager *pkgmanager.PkgManager, pkg Pkg) error {
 	archivePath, err := manager.DownloadArchive(pkg.Name(), pkg.Version(), "")
-	if err != nil {
+	if errors.Is(err, pkgmanager.ErrNoArchiveURL) {
+		slog.WarnContext(ctx, "unable to download archive", "error", err)
+		return nil
+	} else if err != nil {
 		return err
 	}