From 2ce6d6de33c828dcf18f6c48f4523ce98bb59930 Mon Sep 17 00:00:00 2001 From: Caleb Brown Date: Mon, 8 Nov 2021 16:04:28 +1100 Subject: [PATCH] Use a concrete struct while parsing JSON to reduce memory consumption. (#153) * Use a concrete struct while parsing json to reduce memory consuption. The npm package json can be very large. Using the `map[string]interface{}` results in the entire json data structure being deserialized. This costs a lot of memory and cpu time. The feed parser only needs the "time" portion of the json data, so by restricting the struct to that data we avoid a lot of overhead. Testing shows memory peaks close to 70Mb with this fix, rather than the 512Mb+ previously. * Values in the time map can be either string, or a struct, so use the empty interface. --- pkg/feeds/npm/npm.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pkg/feeds/npm/npm.go b/pkg/feeds/npm/npm.go index 49cc6752..aacddd57 100644 --- a/pkg/feeds/npm/npm.go +++ b/pkg/feeds/npm/npm.go @@ -90,18 +90,19 @@ func fetchPackage(baseURL, pkgTitle string) ([]*Package, error) { if err != nil { return nil, err } - var jsonMap map[string]interface{} - err = json.Unmarshal(body, &jsonMap) + + // We only care about the `time` field as it contains all the versions in + // date order, from oldest to newest. + // Using a struct for parsing also avoids the cost of deserializing data + // that is ultimately unused. + var packageDetails struct { + Time map[string]interface{} `json:"time"` + } + err = json.Unmarshal(body, &packageDetails) if err != nil { return nil, fmt.Errorf("%w : %v for package %s", errJSON, err, pkgTitle) } - - // The json string `time` contains versions in date order, oldest to newest. - versions, ok := jsonMap["time"].(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("%w : 'time' not found for package %s ", - errJSON, pkgTitle) - } + versions := packageDetails.Time // If `unpublished` exists in the version map then at a given point in time // the package was 'entirely' removed, the packageEvent(s) received are for package