From 915776161f7cea3de02ec75502c4d202093f85b7 Mon Sep 17 00:00:00 2001
From: "John K. Luebs" <john@luebs.org>
Date: Sat, 23 Sep 2023 22:12:08 -0500
Subject: [PATCH] Use S3 ListObjectsV2 for listing files

ListObjects has been deprecated since 2016 and ListObjectsV2 with use of
explicit pagination tokens is more performant for large listings as well.

This also mitigates an issue with iDrive E2 where the StartAfter/Marker
is included in the output, leading to duplicate entries. Right now this
causes an exhaustive prune to delete chunks erroneously flagged as
duplicate, destroying the storage.
---
 src/duplicacy_s3storage.go | 44 ++++++++++++++++----------------------
 1 file changed, 18 insertions(+), 26 deletions(-)

diff --git a/src/duplicacy_s3storage.go b/src/duplicacy_s3storage.go
index f205d8f9..6344f07c 100644
--- a/src/duplicacy_s3storage.go
+++ b/src/duplicacy_s3storage.go
@@ -90,48 +90,40 @@ func (storage *S3Storage) ListFiles(threadIndex int, dir string) (files []string
 
 	if dir == "snapshots/" {
 		dir = storage.storageDir + dir
-		input := s3.ListObjectsInput{
+		input := s3.ListObjectsV2Input{
 			Bucket:    aws.String(storage.bucket),
 			Prefix:    aws.String(dir),
 			Delimiter: aws.String("/"),
-			MaxKeys:   aws.Int64(1000),
 		}
 
-		output, err := storage.client.ListObjects(&input)
+		err := storage.client.ListObjectsV2Pages(&input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+			for _, subDir := range page.CommonPrefixes {
+				files = append(files, (*subDir.Prefix)[len(dir):])
+			}
+			return true
+		})
 		if err != nil {
 			return nil, nil, err
 		}
 
-		for _, subDir := range output.CommonPrefixes {
-			files = append(files, (*subDir.Prefix)[len(dir):])
-		}
 		return files, nil, nil
 	} else {
 		dir = storage.storageDir + dir
-		marker := ""
-		for {
-			input := s3.ListObjectsInput{
-				Bucket:  aws.String(storage.bucket),
-				Prefix:  aws.String(dir),
-				MaxKeys: aws.Int64(1000),
-				Marker:  aws.String(marker),
-			}
-
-			output, err := storage.client.ListObjects(&input)
-			if err != nil {
-				return nil, nil, err
-			}
+		input := s3.ListObjectsV2Input{
+			Bucket:  aws.String(storage.bucket),
+			Prefix:  aws.String(dir),
+			MaxKeys: aws.Int64(1000),
+		}
 
-			for _, object := range output.Contents {
+		err := storage.client.ListObjectsV2Pages(&input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+			for _, object := range page.Contents {
 				files = append(files, (*object.Key)[len(dir):])
 				sizes = append(sizes, *object.Size)
 			}
-
-			if !*output.IsTruncated {
-				break
-			}
-
-			marker = *output.Contents[len(output.Contents)-1].Key
+			return true
+		})
+		if err != nil {
+			return nil, nil, err
 		}
 		return files, sizes, nil
 	}