acceptance test for downloading versioned files

internal/acctests/sos/download_test.go

@@ -1,22 +1,70 @@
 package sos_test
 
 func (s *SOSSuite) TestDownloadFiles() {
-	test := SOSTest{
+	s.Execute(SOSTest{
 		Steps: []Step{
 			{
 				PreparedFiles: LocalFiles{
 					"file1.txt": "expected content",
 				},
 				Commands: []string{
-					"storage upload {prepDir}file1.txt {bucket}",
-					"storage download -r {bucket} {downloadDir}",
+					"exo storage upload {prepDir}file1.txt {bucket}",
+					"exo storage download -r {bucket} {downloadDir}",
 				},
 				ExpectedDownloadFiles: LocalFiles{
 					"file1.txt": "expected content",
 				},
 			},
 		},
-	}
+	})
+}
 
-	s.Execute(test)
+func (s *SOSSuite) TestDownloadOverwrittenVersionedFile() {
+	s.Execute(SOSTest{
+		Steps: []Step{
+			{
+				PreparedFiles: LocalFiles{
+					"file1.txt": "original content",
+				},
+				Commands: []string{
+					"exo storage bucket versioning enable {bucket}",
+					"exo storage upload {prepDir}file1.txt {bucket}",
+				},
+				ExpectedDownloadFiles: LocalFiles{},
+			},
+			{
+				Description: "check if latest object is downloaded",
+				PreparedFiles: LocalFiles{
+					"file1.txt": "new expected content",
+				},
+				Commands: []string{
+					"exo storage upload {prepDir}file1.txt {bucket}",
+					"exo storage download -f -r {bucket} {downloadDir}",
+				},
+				ExpectedDownloadFiles: LocalFiles{
+					"file1.txt": "new expected content",
+				},
+			},
+			{
+				Description:   "check if v0 can be downloaded",
+				PreparedFiles: LocalFiles{},
+				Commands: []string{
+					"exo storage download -f --only-versions v0 {bucket}/file1.txt {downloadDir}",
+				},
+				ExpectedDownloadFiles: LocalFiles{
+					"file1.txt": "original content",
+				},
+			},
+			{
+				Description:   "check if v1 can be explicitly downloaded",
+				PreparedFiles: LocalFiles{},
+				Commands: []string{
+					"exo storage download -f --only-versions v1 {bucket}/file1.txt {downloadDir}",
+				},
+				ExpectedDownloadFiles: LocalFiles{
+					"file1.txt": "new expected content",
+				},
+			},
+		},
+	})
 }

internal/acctests/sos/sos_test.go

@@ -5,6 +5,7 @@ import "os"
 type LocalFiles map[string]string
 
 type Step struct {
+	Description           string
 	PreparedFiles         LocalFiles
 	Commands              []string
 	ExpectedDownloadFiles LocalFiles
@@ -15,7 +16,8 @@ type SOSTest struct {
 }
 
 func (s *SOSSuite) Execute(test SOSTest) {
-	for _, step := range test.Steps {
+	for stepNr, step := range test.Steps {
+		s.T().Logf("step number: %d %q", stepNr, step.Description)
 		for filename, content := range step.PreparedFiles {
 			s.writeFile(filename, content)
 		}
@@ -29,12 +31,14 @@ func (s *SOSSuite) Execute(test SOSTest) {
 			return
 		}
 
-		if !s.Equal(len(step.ExpectedDownloadFiles), len(files), "number of actual files doesn't match number of expected files") {
-			return
-		}
+		actualFileNumberMismatches := !s.Equal(len(step.ExpectedDownloadFiles), len(files), "number of actual files doesn't match number of expected files")
 
 		downloadDir := LocalFiles{}
 		for _, file := range files {
+			if actualFileNumberMismatches {
+				s.T().Logf("actual file: %s", file)
+			}
+
 			content, err := os.ReadFile(s.DownloadDir + file.Name())
 			if !s.NoError(err) {
 				return
@@ -43,6 +47,10 @@ func (s *SOSSuite) Execute(test SOSTest) {
 			downloadDir[file.Name()] = string(content)
 		}
 
+		if actualFileNumberMismatches {
+			return
+		}
+
 		for expectedFilename, expectedContent := range step.ExpectedDownloadFiles {
 			actualContent, ok := downloadDir[expectedFilename]
 			if !s.True(ok) {

internal/acctests/sos/suite_test.go

@@ -35,8 +35,6 @@ type SOSSuite struct {
 	PrepDir     string
 	DownloadDir string
 
-	ObjectList []string
-
 	S3Client *s3.Client
 }
 
@@ -74,15 +72,16 @@ func (s *SOSSuite) SetupTest() {
 		)...)
 	s.Assert().NoError(err)
 
+	testBucketName := fmt.Sprintf("exo-cli-acc-tests-%d", rand.Int())
+	s.BucketName = testBucketName
 	input := &s3.CreateBucketInput{
 		Bucket: &s.BucketName,
 	}
 
 	s.S3Client = s3.NewFromConfig(cfg)
 
-	s.T().Logf("creating test bucket %q", s.BucketName)
 	_, err = s.S3Client.CreateBucket(ctx, input)
-	s.Assert().NoError(err)
+	s.Assert().NoError(err, fmt.Sprintf("error creating test bucket %q", s.BucketName))
 }
 
 func (s *SOSSuite) TearDownTest() {
@@ -98,46 +97,34 @@ func (s *SOSSuite) TearDownTest() {
 	s.Assert().NoError(err)
 }
 
-func (s *SOSSuite) deleteBucket(bucketName string) {
-	ctx := context.Background()
-
-	s.T().Logf("deleting test bucket %q", bucketName)
-
-	// Delete all objects
-	listObjectsInput := &s3.ListObjectsV2Input{
-		Bucket: &bucketName,
+func getStr(a *string) string {
+	if a == nil {
+		return ""
 	}
 
-	listObjectsResp, err := s.S3Client.ListObjectsV2(ctx, listObjectsInput)
-	s.Assert().NoError(err)
+	return *a
+}
 
-	for _, obj := range listObjectsResp.Contents {
-		s.T().Log("deleting object: ", *obj.Key)
-		deleteObjectInput := &s3.DeleteObjectInput{
-			Bucket: &bucketName,
-			Key:    obj.Key,
-		}
-		_, err := s.S3Client.DeleteObject(ctx, deleteObjectInput)
-		s.Assert().NoError(err)
-	}
+// TODO(sauterp) once deletion of versioned buckets, tests should handle deletions themselves.
+func (s *SOSSuite) deleteBucket(bucketName string) {
+	ctx := context.Background()
 
 	// Delete all object versions
 	listObjectsVersionsInput := &s3.ListObjectVersionsInput{
 		Bucket: &bucketName,
 	}
 
 	listObjectsVersionsResp, err := s.S3Client.ListObjectVersions(ctx, listObjectsVersionsInput)
-	s.Assert().NoError(err)
+	s.Assert().NoError(err, fmt.Sprintf("error deleting test bucket %q", bucketName))
 
 	for _, obj := range listObjectsVersionsResp.Versions {
-		s.T().Log("deleting object version: ", *obj.Key, *obj.VersionId)
 		deleteObjectInput := &s3.DeleteObjectInput{
 			Bucket:    &bucketName,
 			Key:       obj.Key,
 			VersionId: obj.VersionId,
 		}
 		_, err := s.S3Client.DeleteObject(ctx, deleteObjectInput)
-		s.Assert().NoError(err)
+		s.Assert().NoError(err, fmt.Sprintf("deleting object %s version %s", getStr(obj.Key), getStr(obj.VersionId)))
 	}
 
 	// Delete bucket
@@ -150,12 +137,10 @@ func (s *SOSSuite) deleteBucket(bucketName string) {
 }
 
 func TestSOSSuite(t *testing.T) {
-	testBucketName := fmt.Sprintf("exo-cli-acc-tests-%d", rand.Int())
-
 	s := &SOSSuite{
-		BucketName:       testBucketName,
 		ExoCLIExecutable: "../../../bin/exo",
 	}
+
 	suite.Run(t, s)
 }
 
@@ -165,15 +150,23 @@ func (s *SOSSuite) writeFile(filename, content string) {
 }
 
 func (s *SOSSuite) exo(cmdStr string) string {
+	// remove the "exo " prefix
+	cmdStr = cmdStr[4:]
+
 	cmdWithBucket := strings.Replace(cmdStr, "{bucket}", s.BucketName, 1)
 	cmdWithPrepDir := strings.Replace(cmdWithBucket, "{prepDir}", s.PrepDir, 1)
 	cmdComplete := strings.Replace(cmdWithPrepDir, "{downloadDir}", s.DownloadDir, 1)
 	cmds := strings.Split(cmdComplete, " ")
 
+	cmds = append([]string{"--quiet"}, cmds...)
+
 	s.T().Logf("executing command: exo %s", strings.Join(cmds, " "))
+
 	command := exec.Command(s.ExoCLIExecutable, cmds...)
 	output, err := command.CombinedOutput()
-	s.T().Log(string(output))
+	if len(output) > 0 {
+		s.T().Log(string(output))
+	}
 	s.Assert().NoError(err)
 
 	return string(output)