fix: excessive memory usage

instead of reading entire files before calculating their digests
stream them by using their Reader method.

Signed-off-by: Petu Eusebiu <[email protected]>

pkg/api/controller_test.go

@@ -61,6 +61,7 @@ import (
 	"zotregistry.io/zot/pkg/storage"
 	storageConstants "zotregistry.io/zot/pkg/storage/constants"
 	"zotregistry.io/zot/pkg/storage/gc"
+	storageTypes "zotregistry.io/zot/pkg/storage/types"
 	authutils "zotregistry.io/zot/pkg/test/auth"
 	test "zotregistry.io/zot/pkg/test/common"
 	"zotregistry.io/zot/pkg/test/deprecated"
@@ -959,6 +960,39 @@ func TestBlobReferenced(t *testing.T) {
 	})
 }
 
+func TestPrintTracebackOnPanic(t *testing.T) {
+	Convey("Run server on unavailable port", t, func() {
+		port := test.GetFreePort()
+		baseURL := test.GetBaseURL(port)
+
+		conf := config.New()
+		conf.HTTP.Port = port
+
+		logFile, err := os.CreateTemp("", "zot-log*.txt")
+		So(err, ShouldBeNil)
+		conf.Log.Output = logFile.Name()
+		defer os.Remove(logFile.Name()) // clean up
+
+		ctlr := makeController(conf, t.TempDir())
+		cm := test.NewControllerManager(ctlr)
+
+		cm.StartAndWait(port)
+		defer cm.StopServer()
+
+		ctlr.StoreController.SubStore = make(map[string]storageTypes.ImageStore)
+		ctlr.StoreController.SubStore["/a"] = nil
+
+		resp, err := resty.R().Get(baseURL + "/v2/_catalog")
+		So(err, ShouldBeNil)
+		So(resp, ShouldNotBeNil)
+		So(resp.StatusCode(), ShouldEqual, http.StatusInternalServerError)
+
+		data, err := os.ReadFile(logFile.Name())
+		So(err, ShouldBeNil)
+		So(string(data), ShouldContainSubstring, "runtime error: invalid memory address or nil pointer dereference")
+	})
+}
+
 func TestInterruptedBlobUpload(t *testing.T) {
 	Convey("Successfully cleaning interrupted blob uploads", t, func() {
 		port := test.GetFreePort()

pkg/storage/imagestore/imagestore.go

@@ -1,7 +1,6 @@
 package imagestore
 
 import (
-	"bytes"
 	"context"
 	"crypto/sha256"
 	"encoding/json"
@@ -951,25 +950,25 @@ func (is *ImageStore) FullBlobUpload(repo string, body io.Reader, dstDigest godi
 	uuid := u.String()
 	src := is.BlobUploadPath(repo, uuid)
 	digester := sha256.New()
-	buf := new(bytes.Buffer)
 
-	_, err = buf.ReadFrom(body)
+	blobFile, err := is.storeDriver.Writer(src, false)
 	if err != nil {
-		is.log.Error().Err(err).Msg("failed to read blob")
+		is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob")
 
-		return "", -1, err
+		return "", -1, zerr.ErrUploadNotFound
 	}
 
-	nbytes, err := is.storeDriver.WriteFile(src, buf.Bytes())
-	if err != nil {
-		is.log.Error().Err(err).Msg("failed to write blob")
+	defer blobFile.Close()
 
+	mw := io.MultiWriter(blobFile, digester)
+
+	nbytes, err := io.Copy(mw, body)
+	if err != nil {
 		return "", -1, err
 	}
 
-	_, err = digester.Write(buf.Bytes())
-	if err != nil {
-		is.log.Error().Err(err).Str("component", "digester").Msg("failed to write")
+	if err := blobFile.Commit(); err != nil {
+		is.log.Error().Err(err).Str("blob", src).Msg("failed to commit blob")
 
 		return "", -1, err
 	}
@@ -1008,7 +1007,7 @@ func (is *ImageStore) FullBlobUpload(repo string, body io.Reader, dstDigest godi
 		}
 	}
 
-	return uuid, int64(nbytes), nil
+	return uuid, nbytes, nil
 }
 
 func (is *ImageStore) DedupeBlob(src string, dstDigest godigest.Digest, dstRepo string, dst string) error {

pkg/storage/scrub.go

@@ -279,21 +279,28 @@ func CheckLayers(
 	}
 
 	for _, layer := range man.Layers {
-		layerContent, err := imgStore.GetBlobContent(imageName, layer.Digest)
+		layerReader, _, err := imgStore.GetBlob(imageName, layer.Digest, "")
 		if err != nil {
 			imageRes = getResult(imageName, tagName, layer.Digest, err)
 
 			break
 		}
 
-		computedDigest := godigest.FromBytes(layerContent)
+		computedDigest, err := godigest.FromReader(layerReader)
+		if err != nil {
+			imageRes = getResult(imageName, tagName, layer.Digest, err)
+
+			break
+		}
 
 		if computedDigest != layer.Digest {
 			imageRes = getResult(imageName, tagName, layer.Digest, errors.ErrBadBlobDigest)
 
 			break
 		}
 
+		_ = layerReader.Close()
+
 		imageRes = getResult(imageName, tagName, "", nil)
 	}