feat(fetch/nvd): use NVD API feed (#156)

* feat(fetch/nvd): use NVD API feed * feat(fetch/nvd): prefer lang:en title
vulsio · Dec 6, 2023 · 69719c5 · 69719c5
1 parent 7ca74d9
commit 69719c5
Showing 1 changed file with 78 additions and 72 deletions.
diff --git a/fetcher/nvd.go b/fetcher/nvd.go
@@ -1,13 +1,13 @@
 package fetcher
 
 import (
+	"archive/tar"
 	"bytes"
 	"compress/gzip"
 	"encoding/json"
-	"encoding/xml"
-	"errors"
 	"fmt"
 	"io"
+	"path/filepath"
 
 	"github.com/inconshreveable/log15"
 	"github.com/knqyf263/go-cpe/common"
@@ -19,22 +19,19 @@ import (
 	"github.com/vulsio/go-cpe-dictionary/models"
 )
 
-// cpeDictionary has cpe-item list
-// https://nvd.nist.gov/cpe.cfm
+// cpeDictionary : https://csrc.nist.gov/schema/nvd/api/2.0/cpe_api_json_2.0.schema
 type cpeDictionaryItem struct {
-	Name       string `xml:"name,attr"`
-	Deprecated string `xml:"deprecated,attr"`
-	Title      []struct {
-		Text string `xml:",chardata"`
-	} `xml:"title"`
-	Cpe23Item struct {
-		Name string `xml:"name,attr"`
-	} `xml:"cpe23-item"`
+	Deprecated bool   `json:"deprecated"`
+	Name       string `json:"cpeName"`
+	Titles     []struct {
+		Title string `json:"title"`
+		Lang  string `json:"lang"`
+	} `json:"titles,omitempty"`
 }
 
-// cpeMatch : https://csrc.nist.gov/schema/cpematch/feed/1.0/nvd_cpematch_feed_json_1.0.schema
+// cpeMatch : https://csrc.nist.gov/schema/nvd/api/2.0/cpematch_api_json_2.0.schema
 type cpeMatchElement struct {
-	Cpe23URI string `json:"cpe23Uri"`
+	Criteria string `json:"criteria"`
 }
 
 // FetchNVD NVD feeds
@@ -64,91 +61,106 @@ func FetchNVD() (models.FetchedCPEs, error) {
 }
 
 func fetchCpeDictionary(cpes, deprecated map[string]string) error {
-	url := "http://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz"
+	url := "https://github.com/vulsio/vuls-data-raw-nvd-api-cpe/archive/refs/heads/main.tar.gz"
 	log15.Info("Fetching...", "URL", url)
 	resp, bs, errs := gorequest.New().Proxy(viper.GetString("http-proxy")).Get(url).EndBytes()
 	if len(errs) > 0 || resp == nil || resp.StatusCode != 200 {
 		return xerrors.Errorf("HTTP error. errs: %v, url: %s", errs, url)
 	}
 
-	r, err := gzip.NewReader(bytes.NewReader(bs))
+	gr, err := gzip.NewReader(bytes.NewReader(bs))
 	if err != nil {
-		return xerrors.Errorf("Failed to decompress CPE Dictionary. url: %s, err: %w", url, err)
+		return xerrors.Errorf("Failed to create gzip reader. url: %s, err: %w", url, err)
 	}
-	defer r.Close()
+	defer gr.Close()
 
-	d := xml.NewDecoder(r)
+	tr := tar.NewReader(gr)
 	for {
-		t, err := d.Token()
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
 		if err != nil {
-			if errors.Is(err, io.EOF) {
-				break
-			}
-			return xerrors.Errorf("Failed to return next XML token. err: %w", err)
+			return xerrors.Errorf("Failed to next tar reader. err: %w", err)
+		}
+
+		if hdr.FileInfo().IsDir() {
+			continue
+		}
+
+		if filepath.Ext(hdr.Name) != ".json" {
+			continue
 		}
-		switch se := t.(type) {
-		case xml.StartElement:
-			if se.Name.Local != "cpe-item" {
+
+		var item cpeDictionaryItem
+		if err := json.NewDecoder(tr).Decode(&item); err != nil {
+			return xerrors.Errorf("Failed to decode %s. err: %w", hdr.Name, err)
+		}
+
+		if _, err := naming.UnbindFS(item.Name); err != nil {
+			log15.Warn("Failed to unbind", item.Name, err)
+			continue
+		}
+
+		var title string
+		for _, t := range item.Titles {
+			title = t.Title
+			if t.Lang == "en" {
 				break
 			}
-			var item cpeDictionaryItem
-			if err := d.DecodeElement(&item, &se); err != nil {
-				return xerrors.Errorf("Failed to decode. url: %s, err: %w", url, err)
-			}
-			if _, err := naming.UnbindFS(item.Cpe23Item.Name); err != nil {
-				// Logging only
-				log15.Warn("Failed to unbind", item.Cpe23Item.Name, err)
-				continue
-			}
-			for _, t := range item.Title {
-				if item.Deprecated == "true" {
-					deprecated[item.Cpe23Item.Name] = t.Text
-				} else {
-					cpes[item.Cpe23Item.Name] = t.Text
-				}
-			}
-		default:
+		}
+		if item.Deprecated {
+			deprecated[item.Name] = title
+		} else {
+			cpes[item.Name] = title
 		}
 	}
+
 	return nil
 }
 
 func fetchCpeMatch(cpes map[string]string) error {
-	url := "https://nvd.nist.gov/feeds/json/cpematch/1.0/nvdcpematch-1.0.json.gz"
+	url := "https://github.com/vulsio/vuls-data-raw-nvd-api-cpematch/archive/refs/heads/main.tar.gz"
 	log15.Info("Fetching...", "URL", url)
 	resp, bs, errs := gorequest.New().Proxy(viper.GetString("http-proxy")).Get(url).EndBytes()
 	if len(errs) > 0 || resp == nil || resp.StatusCode != 200 {
 		return xerrors.Errorf("HTTP error. errs: %v, url: %s", errs, url)
 	}
 
-	r, err := gzip.NewReader(bytes.NewReader(bs))
+	gr, err := gzip.NewReader(bytes.NewReader(bs))
 	if err != nil {
-		return xerrors.Errorf("Failed to decompress CPE Match. url: %s, err: %w", url, err)
+		return xerrors.Errorf("Failed to create gzip reader. url: %s, err: %w", url, err)
 	}
-	defer r.Close()
+	defer gr.Close()
+
+	tr := tar.NewReader(gr)
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return xerrors.Errorf("Failed to next tar reader. err: %w", err)
+		}
+
+		if hdr.FileInfo().IsDir() {
+			continue
+		}
+
+		if filepath.Ext(hdr.Name) != ".json" {
+			continue
+		}
 
-	d := json.NewDecoder(r)
-	if _, err := d.Token(); err != nil { // json.Delim: {
-		return xerrors.Errorf("Failed to return next JSON token. err: %w", err)
-	}
-	if _, err := d.Token(); err != nil { // string: matches
-		return xerrors.Errorf("Failed to return next JSON token. err: %w", err)
-	}
-	if _, err := d.Token(); err != nil { // json.Delim: [
-		return xerrors.Errorf("Failed to return next JSON token. err: %w", err)
-	}
-	for d.More() {
 		var cpeMatch cpeMatchElement
-		if err := d.Decode(&cpeMatch); err != nil {
-			return xerrors.Errorf("Failed to decode. url: %s, err: %w", url, err)
+		if err := json.NewDecoder(tr).Decode(&cpeMatch); err != nil {
+			return xerrors.Errorf("Failed to decode %s. err: %w", hdr.Name, err)
 		}
-		wfn, err := naming.UnbindFS(cpeMatch.Cpe23URI)
+		wfn, err := naming.UnbindFS(cpeMatch.Criteria)
 		if err != nil {
-			// Logging only
-			log15.Warn("Failed to unbind", cpeMatch.Cpe23URI, err)
+			log15.Warn("Failed to unbind", cpeMatch.Criteria, err)
 			continue
 		}
-		if _, ok := cpes[cpeMatch.Cpe23URI]; !ok {
+		if _, ok := cpes[cpeMatch.Criteria]; !ok {
 			title := fmt.Sprintf("%s %s", wfn.GetString(common.AttributeVendor), wfn.GetString(common.AttributeProduct))
 			if wfn.GetString(common.AttributeVersion) != "ANY" {
 				title = fmt.Sprintf("%s %s", title, wfn.GetString(common.AttributeVersion))
@@ -174,15 +186,9 @@ func fetchCpeMatch(cpes map[string]string) error {
 			if wfn.GetString(common.AttributeOther) != "ANY" {
 				title = fmt.Sprintf("%s %s", title, wfn.GetString(common.AttributeOther))
 			}
-			cpes[cpeMatch.Cpe23URI] = title
+			cpes[cpeMatch.Criteria] = title
 		}
 	}
-	if _, err := d.Token(); err != nil { // json.Delim: ]
-		return xerrors.Errorf("Failed to return next JSON token. err: %w", err)
-	}
-	if _, err := d.Token(); err != nil { // json.Delim: }
-		return xerrors.Errorf("Failed to return next JSON token. err: %w", err)
-	}
 
 	return nil
 }