From a1404614f83dfca7e0ab43d3c0c3a6795b93e22f Mon Sep 17 00:00:00 2001 From: Oluwatoni Solarin-Sodara Date: Tue, 20 Feb 2018 14:07:58 +0100 Subject: [PATCH 01/10] ignore ebs delete errors when it's connected to a protected instance --- aws/ebs.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/aws/ebs.go b/aws/ebs.go index 8f701933..49d26853 100644 --- a/aws/ebs.go +++ b/aws/ebs.go @@ -1,9 +1,9 @@ package aws import ( - "strings" "time" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/gruntwork-io/aws-nuke/logging" @@ -47,16 +47,19 @@ func nukeAllEbsVolumes(session *session.Session, volumeIds []*string) error { _, err := svc.DeleteVolume(params) if err != nil { - // Ignore not found errors, some volumes are deleted along with EC2 Instances - if !strings.Contains(err.Error(), "InvalidVolume.NotFound") { - logging.Logger.Errorf("[Failed] %s", err) - return errors.WithStackTrace(err) + if awsErr, isAwsErr := err.(awserr.Error); isAwsErr && awsErr.Code() == "VolumeInUse" { + logging.Logger.Infof("EBS volume %s is attached to a protected EC2 instance", *volumeID) + return nil + } else if awsErr, isAwsErr := err.(awserr.Error); isAwsErr && awsErr.Code() == "InvalidVolume.NotFound" { + logging.Logger.Infof("EBS volume %s has already been deleted", *volumeID) + return nil } - logging.Logger.Infof("EBS volume %s has already been deleted", *volumeID) - } else { - logging.Logger.Infof("Deleted EBS Volume: %s", *volumeID) + logging.Logger.Errorf("[Failed] %s", err) + return errors.WithStackTrace(err) } + + logging.Logger.Infof("Deleted EBS Volume: %s", *volumeID) } err := svc.WaitUntilVolumeDeleted(&ec2.DescribeVolumesInput{ From d2d928ad11739045488bdd7ecc9772fd6c57d683 Mon Sep 17 00:00:00 2001 From: Oluwatoni Solarin-Sodara Date: Tue, 20 Feb 2018 14:32:16 +0100 Subject: [PATCH 02/10] update ebs tests --- aws/ebs_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/ebs_test.go b/aws/ebs_test.go index 88faaa16..b4030748 100644 --- a/aws/ebs_test.go +++ b/aws/ebs_test.go @@ -108,7 +108,7 @@ func TestNukeEBSVolumes(t *testing.T) { } uniqueTestID := "aws-nuke-test-" + util.UniqueID() - createTestEC2Instance(t, session, uniqueTestID, false) + createTestEBSVolume(t, session, uniqueTestID) output, err := ec2.New(session).DescribeVolumes(&ec2.DescribeVolumesInput{}) if err != nil { @@ -123,7 +123,7 @@ func TestNukeEBSVolumes(t *testing.T) { volumes, err := getAllEbsVolumes(session, region, time.Now().Add(1*time.Hour)) if err != nil { - assert.Fail(t, "Unable to fetch list of EC2 Instances") + assert.Fail(t, "Unable to fetch list of EBS Volumes") } for _, volumeID := range volumeIds { From 4cdc95babf865a00b88a0d0eaa2d6f8e7f314f60 Mon Sep 17 00:00:00 2001 From: Oluwatoni Solarin-Sodara Date: Wed, 21 Feb 2018 13:12:26 +0100 Subject: [PATCH 03/10] make VolumeInUse error log message more generic --- aws/ebs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/ebs.go b/aws/ebs.go index 49d26853..d11a93b5 100644 --- a/aws/ebs.go +++ b/aws/ebs.go @@ -48,7 +48,7 @@ func nukeAllEbsVolumes(session *session.Session, volumeIds []*string) error { _, err := svc.DeleteVolume(params) if err != nil { if awsErr, isAwsErr := err.(awserr.Error); isAwsErr && awsErr.Code() == "VolumeInUse" { - logging.Logger.Infof("EBS volume %s is attached to a protected EC2 instance", *volumeID) + logging.Logger.Infof("EBS volume %s is attached to an active resource", *volumeID) return nil } else if awsErr, isAwsErr := err.(awserr.Error); isAwsErr && awsErr.Code() == "InvalidVolume.NotFound" { logging.Logger.Infof("EBS volume %s has already been deleted", *volumeID) From 491991a71d40d98a47d1ba08b45367c46370863a Mon Sep 17 00:00:00 2001 From: Oluwatoni Solarin-Sodara Date: Wed, 21 Feb 2018 14:15:46 +0100 Subject: [PATCH 04/10] update ebs tests, check for still attached volume --- aws/ebs_test.go | 60 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/aws/ebs_test.go b/aws/ebs_test.go index b4030748..94cce2aa 100644 --- a/aws/ebs_test.go +++ b/aws/ebs_test.go @@ -108,7 +108,7 @@ func TestNukeEBSVolumes(t *testing.T) { } uniqueTestID := "aws-nuke-test-" + util.UniqueID() - createTestEBSVolume(t, session, uniqueTestID) + volume := createTestEBSVolume(t, session, uniqueTestID) output, err := ec2.New(session).DescribeVolumes(&ec2.DescribeVolumesInput{}) if err != nil { @@ -117,16 +117,68 @@ func TestNukeEBSVolumes(t *testing.T) { volumeIds := findEBSVolumesByNameTag(output, uniqueTestID) + assert.Len(t, volumeIds, 1) + assert.Equal(t, awsgo.StringValue(volume.VolumeId), awsgo.StringValue(volumeIds[0])) + if err := nukeAllEbsVolumes(session, volumeIds); err != nil { assert.Fail(t, errors.WithStackTrace(err).Error()) } - volumes, err := getAllEbsVolumes(session, region, time.Now().Add(1*time.Hour)) + volumeIds, err = getAllEbsVolumes(session, region, time.Now().Add(1*time.Hour)) if err != nil { assert.Fail(t, "Unable to fetch list of EBS Volumes") } - for _, volumeID := range volumeIds { - assert.NotContains(t, volumes, *volumeID) + assert.NotContains(t, awsgo.StringValueSlice(volumeIds), awsgo.StringValue(volume.VolumeId)) +} + +func TestNukeEBSVolumesInUse(t *testing.T) { + t.Parallel() + + region := getRandomRegion() + session, err := session.NewSession(&awsgo.Config{ + Region: awsgo.String(region)}, + ) + + if err != nil { + assert.Fail(t, errors.WithStackTrace(err).Error()) + } + + svc := ec2.New(session) + + uniqueTestID := "aws-nuke-test-" + util.UniqueID() + volume := createTestEBSVolume(t, session, uniqueTestID) + instance := createTestEC2Instance(t, session, uniqueTestID, false) + + defer nukeAllEbsVolumes(session, []*string{volume.VolumeId}) + defer nukeAllEc2Instances(session, []*string{instance.InstanceId}) + + // attach volume to protected instance + svc.AttachVolume(&ec2.AttachVolumeInput{ + Device: awsgo.String("/dev/sdf"), + InstanceId: instance.InstanceId, + VolumeId: volume.VolumeId, + }) + + output, err := svc.DescribeVolumes(&ec2.DescribeVolumesInput{}) + if err != nil { + assert.Fail(t, errors.WithStackTrace(err).Error()) + } + + volumeIds := findEBSVolumesByNameTag(output, uniqueTestID) + + assert.Len(t, volumeIds, 1) + assert.Equal(t, awsgo.StringValue(volume.VolumeId), awsgo.StringValue(volumeIds[0])) + + if err := nukeAllEbsVolumes(session, volumeIds); err != nil { + assert.Fail(t, errors.WithStackTrace(err).Error()) + } + + volumeIds, err = getAllEbsVolumes(session, region, time.Now().Add(1*time.Hour)) + if err != nil { + assert.Fail(t, "Unable to fetch list of EBS Volumes") } + + // Volumes should still be in returned slice + assert.Contains(t, awsgo.StringValueSlice(volumeIds), awsgo.StringValue(volume.VolumeId)) } From 31478a3019f1a03ad1390c38cb14499322499a23 Mon Sep 17 00:00:00 2001 From: Oluwatoni Solarin-Sodara Date: Wed, 21 Feb 2018 14:46:24 +0100 Subject: [PATCH 05/10] update defer calls, let nuke tests assist in cleanup --- aws/ebs_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/aws/ebs_test.go b/aws/ebs_test.go index 94cce2aa..ee9d68fe 100644 --- a/aws/ebs_test.go +++ b/aws/ebs_test.go @@ -151,7 +151,6 @@ func TestNukeEBSVolumesInUse(t *testing.T) { instance := createTestEC2Instance(t, session, uniqueTestID, false) defer nukeAllEbsVolumes(session, []*string{volume.VolumeId}) - defer nukeAllEc2Instances(session, []*string{instance.InstanceId}) // attach volume to protected instance svc.AttachVolume(&ec2.AttachVolumeInput{ From 94cd8ae8328f810dd314109ceb72419a754a3f30 Mon Sep 17 00:00:00 2001 From: Oluwatoni Solarin-Sodara Date: Wed, 21 Feb 2018 15:00:39 +0100 Subject: [PATCH 06/10] log ebs still in use message as warn --- aws/ebs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/ebs.go b/aws/ebs.go index d11a93b5..cdc665f8 100644 --- a/aws/ebs.go +++ b/aws/ebs.go @@ -48,7 +48,7 @@ func nukeAllEbsVolumes(session *session.Session, volumeIds []*string) error { _, err := svc.DeleteVolume(params) if err != nil { if awsErr, isAwsErr := err.(awserr.Error); isAwsErr && awsErr.Code() == "VolumeInUse" { - logging.Logger.Infof("EBS volume %s is attached to an active resource", *volumeID) + logging.Logger.Warnf("EBS volume %s can't be deleted, it is still attached to an active resource", *volumeID) return nil } else if awsErr, isAwsErr := err.(awserr.Error); isAwsErr && awsErr.Code() == "InvalidVolume.NotFound" { logging.Logger.Infof("EBS volume %s has already been deleted", *volumeID) From f50cf9488345e9ad0a838e3d7a8de1124054d5bf Mon Sep 17 00:00:00 2001 From: Oluwatoni Solarin-Sodara Date: Wed, 21 Feb 2018 15:22:52 +0100 Subject: [PATCH 07/10] fix ebs tests --- aws/ebs_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/aws/ebs_test.go b/aws/ebs_test.go index ee9d68fe..9db5ba37 100644 --- a/aws/ebs_test.go +++ b/aws/ebs_test.go @@ -150,8 +150,6 @@ func TestNukeEBSVolumesInUse(t *testing.T) { volume := createTestEBSVolume(t, session, uniqueTestID) instance := createTestEC2Instance(t, session, uniqueTestID, false) - defer nukeAllEbsVolumes(session, []*string{volume.VolumeId}) - // attach volume to protected instance svc.AttachVolume(&ec2.AttachVolumeInput{ Device: awsgo.String("/dev/sdf"), From 410f93da2170f8daada9e3eeabb4ae928286bbf8 Mon Sep 17 00:00:00 2001 From: Oluwatoni Solarin-Sodara Date: Wed, 21 Feb 2018 19:53:36 +0100 Subject: [PATCH 08/10] remove protection from protected ec2 test instances for clean up --- aws/ebs_test.go | 7 ++++++- aws/ec2_test.go | 12 ++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/aws/ebs_test.go b/aws/ebs_test.go index 9db5ba37..3fac7252 100644 --- a/aws/ebs_test.go +++ b/aws/ebs_test.go @@ -148,7 +148,10 @@ func TestNukeEBSVolumesInUse(t *testing.T) { uniqueTestID := "aws-nuke-test-" + util.UniqueID() volume := createTestEBSVolume(t, session, uniqueTestID) - instance := createTestEC2Instance(t, session, uniqueTestID, false) + instance := createTestEC2Instance(t, session, uniqueTestID, true) + + defer nukeAllEbsVolumes(session, []*string{volume.VolumeId}) + defer nukeAllEc2Instances(session, []*string{instance.InstanceId}) // attach volume to protected instance svc.AttachVolume(&ec2.AttachVolumeInput{ @@ -178,4 +181,6 @@ func TestNukeEBSVolumesInUse(t *testing.T) { // Volumes should still be in returned slice assert.Contains(t, awsgo.StringValueSlice(volumeIds), awsgo.StringValue(volume.VolumeId)) + // remove protection so instance can be cleaned up + removeEC2InstanceProtection(svc, &instance) } diff --git a/aws/ec2_test.go b/aws/ec2_test.go index 5c33346d..e8c2db59 100644 --- a/aws/ec2_test.go +++ b/aws/ec2_test.go @@ -93,6 +93,16 @@ func createTestEC2Instance(t *testing.T, session *session.Session, name string, return *runResult.Instances[0] } +func removeEC2InstanceProtection(svc *ec2.EC2, instance *ec2.Instance) { + // make instance unprotected so it can be cleaned up + svc.ModifyInstanceAttribute(&ec2.ModifyInstanceAttributeInput{ + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: awsgo.Bool(false), + }, + InstanceId: instance.InstanceId, + }) +} + func findEC2InstancesByNameTag(output *ec2.DescribeInstancesOutput, name string) []*string { var instanceIds []*string for _, reservation := range output.Reservations { @@ -147,6 +157,8 @@ func TestListInstances(t *testing.T) { assert.Contains(t, instanceIds, instance.InstanceId) assert.NotContains(t, instanceIds, protectedInstance.InstanceId) + + removeEC2InstanceProtection(ec2.New(session), &protectedInstance) } func TestNukeInstances(t *testing.T) { From 510f7f9f6029f9e4e3a7ca6a17cc0f94e347b069 Mon Sep 17 00:00:00 2001 From: Oluwatoni Solarin-Sodara Date: Wed, 21 Feb 2018 20:20:00 +0100 Subject: [PATCH 09/10] ensure volume is in use before proceeding to try a delete --- aws/ebs_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/aws/ebs_test.go b/aws/ebs_test.go index 3fac7252..ba903ced 100644 --- a/aws/ebs_test.go +++ b/aws/ebs_test.go @@ -160,6 +160,10 @@ func TestNukeEBSVolumesInUse(t *testing.T) { VolumeId: volume.VolumeId, }) + svc.WaitUntilVolumeInUse(&ec2.DescribeVolumesInput{ + VolumeIds: []*string{volume.VolumeId}, + }) + output, err := svc.DescribeVolumes(&ec2.DescribeVolumesInput{}) if err != nil { assert.Fail(t, errors.WithStackTrace(err).Error()) From f0864b09e378b80f943c18cb372afa65cba64ce9 Mon Sep 17 00:00:00 2001 From: Oluwatoni Solarin-Sodara Date: Thu, 22 Feb 2018 14:23:01 +0100 Subject: [PATCH 10/10] catch any errors that may be thrown by removeEC2InstanceProtection function --- aws/ebs_test.go | 4 +++- aws/ec2_test.go | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/aws/ebs_test.go b/aws/ebs_test.go index ba903ced..b8cd871f 100644 --- a/aws/ebs_test.go +++ b/aws/ebs_test.go @@ -186,5 +186,7 @@ func TestNukeEBSVolumesInUse(t *testing.T) { // Volumes should still be in returned slice assert.Contains(t, awsgo.StringValueSlice(volumeIds), awsgo.StringValue(volume.VolumeId)) // remove protection so instance can be cleaned up - removeEC2InstanceProtection(svc, &instance) + if err = removeEC2InstanceProtection(svc, &instance); err != nil { + assert.Fail(t, errors.WithStackTrace(err).Error()) + } } diff --git a/aws/ec2_test.go b/aws/ec2_test.go index e8c2db59..cfae3dcc 100644 --- a/aws/ec2_test.go +++ b/aws/ec2_test.go @@ -93,14 +93,16 @@ func createTestEC2Instance(t *testing.T, session *session.Session, name string, return *runResult.Instances[0] } -func removeEC2InstanceProtection(svc *ec2.EC2, instance *ec2.Instance) { +func removeEC2InstanceProtection(svc *ec2.EC2, instance *ec2.Instance) error { // make instance unprotected so it can be cleaned up - svc.ModifyInstanceAttribute(&ec2.ModifyInstanceAttributeInput{ + _, err := svc.ModifyInstanceAttribute(&ec2.ModifyInstanceAttributeInput{ DisableApiTermination: &ec2.AttributeBooleanValue{ Value: awsgo.Bool(false), }, InstanceId: instance.InstanceId, }) + + return err } func findEC2InstancesByNameTag(output *ec2.DescribeInstancesOutput, name string) []*string { @@ -158,7 +160,9 @@ func TestListInstances(t *testing.T) { assert.Contains(t, instanceIds, instance.InstanceId) assert.NotContains(t, instanceIds, protectedInstance.InstanceId) - removeEC2InstanceProtection(ec2.New(session), &protectedInstance) + if err = removeEC2InstanceProtection(ec2.New(session), &protectedInstance); err != nil { + assert.Fail(t, errors.WithStackTrace(err).Error()) + } } func TestNukeInstances(t *testing.T) {