Empty buckets during Delete() (#420)

* Empty buckets during Delete() * Restyled by gofmt * Restyled by gofmt * Add bucket deletion waiting mechanism * Revert Google Cloud Co-authored-by: Restyled.io <[email protected]>
iterative · Mar 7, 2022 · b47f801 · b47f801
1 parent 9fc440f
commit b47f801
Show file tree

Hide file tree

Showing 9 changed files with 61 additions and 49 deletions.
diff --git a/.gitignore b/.gitignore
@@ -41,4 +41,4 @@ terraform
 # Ignore the Go vendor directory, superseded by proxy.golang.org and go.{mod,sum}
 vendor
 
-./terraform-provider-iterative
+terraform-provider-iterative
diff --git a/task/aws/resources/resource_auto_scaling_group.go b/task/aws/resources/resource_auto_scaling_group.go
@@ -3,6 +3,7 @@ package resources
 import (
 	"context"
 	"errors"
+	"log"
 	"net"
 	"strconv"
 	"time"
@@ -133,6 +134,7 @@ func (a *AutoScalingGroup) Read(ctx context.Context) error {
 					if instance.StateReason != nil {
 						status += " " + aws.ToString(instance.StateReason.Message)
 					}
+					log.Println("[DEBUG] AutoScaling Group State:", status)
 					if status == "running" {
 						a.Attributes.Status[common.StatusCodeActive]++
 					}

diff --git a/task/aws/resources/resource_bucket.go b/task/aws/resources/resource_bucket.go
@@ -76,54 +76,17 @@ func (b *Bucket) Update(ctx context.Context) error {
 }
 
 func (b *Bucket) Delete(ctx context.Context) error {
-	listInput := s3.ListObjectsV2Input{
+	input := s3.DeleteBucketInput{
 		Bucket: aws.String(b.Identifier),
 	}
 
-	for paginator := s3.NewListObjectsV2Paginator(b.Client.Services.S3, &listInput); paginator.HasMorePages(); {
-		page, err := paginator.NextPage(ctx)
-
-		if err != nil {
-			var e smithy.APIError
-			if errors.As(err, &e) && e.ErrorCode() == "NoSuchBucket" {
-				b.Resource = nil
-				return nil
-			}
-			return err
-		}
-
-		if len(page.Contents) == 0 {
-			break
-		}
-
-		var objects []types.ObjectIdentifier
-		for _, object := range page.Contents {
-			objects = append(objects, types.ObjectIdentifier{
-				Key: object.Key,
-			})
-		}
-
-		input := s3.DeleteObjectsInput{
-			Bucket: aws.String(b.Identifier),
-			Delete: &types.Delete{
-				Objects: objects,
-			},
-		}
-
-		if _, err = b.Client.Services.S3.DeleteObjects(ctx, &input); err != nil {
+	if _, err := b.Client.Services.S3.DeleteBucket(ctx, &input); err != nil {
+		var e smithy.APIError
+		if errors.As(err, &e) && e.ErrorCode() != "NoSuchBucket" {
 			return err
 		}
 	}
 
-	deleteInput := s3.DeleteBucketInput{
-		Bucket: aws.String(b.Identifier),
-	}
-
-	_, err := b.Client.Services.S3.DeleteBucket(ctx, &deleteInput)
-	if err != nil {
-		return err
-	}
-
 	b.Resource = nil
 	return nil
 }
diff --git a/task/aws/task.go b/task/aws/task.go
@@ -196,8 +196,14 @@ func (t *Task) Read(ctx context.Context) error {
 
 func (t *Task) Delete(ctx context.Context) error {
 	logrus.Debug("Downloading Directory...")
-	if t.Attributes.Environment.DirectoryOut != "" && t.Read(ctx) == nil {
-		if err := t.Pull(ctx, t.Attributes.Environment.DirectoryOut); err != nil && err != common.NotFoundError {
+	if t.Read(ctx) == nil {
+		if t.Attributes.Environment.DirectoryOut != "" {
+			if err := t.Pull(ctx, t.Attributes.Environment.DirectoryOut); err != nil && err != common.NotFoundError {
+				return err
+			}
+		}
+		logrus.Debug("Emptying Bucket...")
+		if err := machine.Delete(ctx, (*t.DataSources.Credentials.Resource)["RCLONE_REMOTE"]); err != nil && err != common.NotFoundError {
 			return err
 		}
 	}

diff --git a/task/az/resources/resource_virtual_machine_scale_set.go b/task/az/resources/resource_virtual_machine_scale_set.go
@@ -5,6 +5,7 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
+	"log"
 	"net"
 	"regexp"
 	"time"
@@ -243,6 +244,7 @@ func (v *VirtualMachineScaleSet) Read(ctx context.Context) error {
 	if scaleSetView.VirtualMachine.StatusesSummary != nil {
 		for _, status := range *scaleSetView.VirtualMachine.StatusesSummary {
 			code := to.String(status.Code)
+			log.Println("[DEBUG] ScaleSet Status Summary:", code, int(to.Int32(status.Count)))
 			if code == "ProvisioningState/succeeded" {
 				v.Attributes.Status[common.StatusCodeActive] = int(to.Int32(status.Count))
 			}

diff --git a/task/az/task.go b/task/az/task.go
@@ -185,8 +185,14 @@ func (t *Task) Read(ctx context.Context) error {
 
 func (t *Task) Delete(ctx context.Context) error {
 	logrus.Debug("Downloading Directory...")
-	if t.Attributes.Environment.DirectoryOut != "" && t.Read(ctx) == nil {
-		if err := t.Pull(ctx, t.Attributes.Environment.DirectoryOut); err != nil && err != common.NotFoundError {
+	if t.Read(ctx) == nil {
+		if t.Attributes.Environment.DirectoryOut != "" {
+			if err := t.Pull(ctx, t.Attributes.Environment.DirectoryOut); err != nil && err != common.NotFoundError {
+				return err
+			}
+		}
+		logrus.Debug("Emptying Bucket...")
+		if err := machine.Delete(ctx, (*t.DataSources.Credentials.Resource)["RCLONE_REMOTE"]); err != nil && err != common.NotFoundError {
 			return err
 		}
 	}

diff --git a/task/common/machine/storage.go b/task/common/machine/storage.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"io"
 	"path/filepath"
 	"strings"
@@ -14,6 +15,7 @@ import (
 	_ "github.com/rclone/rclone/backend/s3"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/sync"
 
 	"terraform-provider-iterative/task/common"
@@ -99,9 +101,32 @@ func Transfer(ctx context.Context, source, destination string) error {
 		return err
 	}
 
-	if err := sync.CopyDir(ctx, destinationFileSystem, sourceFileSystem, true); err != nil {
+	return sync.CopyDir(ctx, destinationFileSystem, sourceFileSystem, true)
+}
+
+func Delete(ctx context.Context, destination string) error {
+	destinationFileSystem, err := fs.NewFs(ctx, destination)
+	if err != nil {
 		return err
 	}
 
+	actions := []func(context.Context) error{
+		func(ctx context.Context) error {
+			return operations.Delete(ctx, destinationFileSystem)
+		},
+		func(ctx context.Context) error {
+			return operations.Rmdirs(ctx, destinationFileSystem, "", true)
+		},
+	}
+
+	for _, action := range actions {
+		if err := action(ctx); err != nil {
+			if !errors.Is(err, fs.ErrorDirNotFound) && !strings.Contains(err.Error(), "no such host") {
+				return common.NotFoundError
+			}
+			return err
+		}
+	}
+
 	return nil
 }
diff --git a/task/gcp/resources/resource_instance_group_manager.go b/task/gcp/resources/resource_instance_group_manager.go
@@ -3,6 +3,7 @@ package resources
 import (
 	"context"
 	"errors"
+	"log"
 	"net"
 	"path/filepath"
 	"strings"
@@ -77,6 +78,7 @@ func (i *InstanceGroupManager) Read(ctx context.Context) error {
 	i.Attributes.Addresses = []net.IP{}
 	i.Attributes.Status = common.Status{common.StatusCodeActive: 0}
 	for _, groupInstance := range groupInstances.Items {
+		log.Println("[DEBUG] Instance Group Manager Status:", groupInstance.Status)
 		if groupInstance.Status == "RUNNING" {
 			instance, err := i.Client.Services.Compute.Instances.Get(i.Client.Credentials.ProjectID, i.Client.Region, filepath.Base(groupInstance.Instance)).Do()
 			if err != nil {

diff --git a/task/gcp/task.go b/task/gcp/task.go
@@ -264,8 +264,14 @@ func (t *Task) Read(ctx context.Context) error {
 
 func (t *Task) Delete(ctx context.Context) error {
 	logrus.Debug("Downloading Directory...")
-	if t.Attributes.Environment.DirectoryOut != "" && t.Read(ctx) == nil {
-		if err := t.Pull(ctx, t.Attributes.Environment.DirectoryOut); err != nil && err != common.NotFoundError {
+	if t.Read(ctx) == nil {
+		if t.Attributes.Environment.DirectoryOut != "" {
+			if err := t.Pull(ctx, t.Attributes.Environment.DirectoryOut); err != nil && err != common.NotFoundError {
+				return err
+			}
+		}
+		logrus.Debug("Emptying Bucket...")
+		if err := machine.Delete(ctx, (*t.DataSources.Credentials.Resource)["RCLONE_REMOTE"]); err != nil && err != common.NotFoundError {
 			return err
 		}
 	}