Fix 5000 charts limit with Azure Blob Storage (#38)

* Use the same env variable as in the test suite guard in TestAzureStorageTestSuite(...).

* Remove unnecessary GetProperties call per object, leading a *significant* performance improvement.

* Support paging with Azure Blob Storage to support more than 5000 objects.

Co-authored-by: Josh Dolitsky <[email protected]>

microsoft.go

@@ -76,25 +76,35 @@ func (b MicrosoftBlobBackend) ListObjects(prefix string) ([]Object, error) {
 	var params microsoft_storage.ListBlobsParameters
 	prefix = pathutil.Join(b.Prefix, prefix)
 	params.Prefix = prefix
-	response, err := b.Container.ListBlobs(params)
-	if err != nil {
-		return objects, err
-	}
 
-	for _, blob := range response.Blobs {
-		path := removePrefixFromObjectPath(prefix, blob.Name)
-		if objectPathIsInvalid(path) {
-			continue
+	for {
+		response, err := b.Container.ListBlobs(params)
+		if err != nil {
+			return objects, err
 		}
 
-		object := Object{
-			Path:         path,
-			Content:      []byte{},
-			LastModified: time.Time(blob.Properties.LastModified),
+		for _, blob := range response.Blobs {
+			path := removePrefixFromObjectPath(prefix, blob.Name)
+			if objectPathIsInvalid(path) {
+				continue
+			}
+
+			object := Object{
+				Path:         path,
+				Content:      []byte{},
+				LastModified: time.Time(blob.Properties.LastModified),
+			}
+
+			objects = append(objects, object)
 		}
 
-		objects = append(objects, object)
+		if response.NextMarker == "" {
+			break
+		}
+
+		params.Marker = response.NextMarker
 	}
+
 	return objects, nil
 }
 

microsoft_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package storage
 
 import (
+	"fmt"
 	"os"
 	"testing"
 
@@ -56,6 +57,28 @@ func (suite *MicrosoftTestSuite) TestListObjects() {
 	suite.Nil(err, "can list objects with good bucket, no prefix")
 }
 
+func (suite *MicrosoftTestSuite) TestListObjectsWithPaging() {
+	// create 5001 objects to trigger the need of paging, since
+	// the default page size for Azure Blobs are 5000.
+	data := []byte("some object")
+	for i := 0; i < 5001; i++ {
+		path := fmt.Sprintf("deleteme-%d.txt", i)
+		suite.NoPrefixAzureBlobBackend.PutObject(path, data)
+	}
+
+	// check if 5002 (5001 plus deleteme.txt from SetupSuite()) objects
+	// are returned. Without paging, we would get only 5000.
+	res, err := suite.NoPrefixAzureBlobBackend.ListObjects("")
+	suite.Nil(err, "can list objects with good bucket, no prefix")
+	suite.Equal(5001, len(res))
+
+	// clean up
+	for i := 0; i < 5001; i++ {
+		path := fmt.Sprintf("deleteme-%d.txt", i)
+		suite.NoPrefixAzureBlobBackend.DeleteObject(path)
+	}
+}
+
 func (suite *MicrosoftTestSuite) TestGetObject() {
 	_, err := suite.BrokenAzureBlobBackend.GetObject("this-file-cannot-possibly-exist.tgz")
 	suite.NotNil(err, "cannot get objects with bad bucket")