Merge pull request #797 from degica/auto-recreate

Recreate stack instead of erroring

app/models/backend/ecs/v2/adapter.rb

@@ -49,16 +49,19 @@ def cf_executor
       @cf_executor ||= CloudFormation::Executor.new(service_stack, service.district)
     end
 
+    def service_resource_id
+      cf_executor.resource_ids["ECSService"]
+    end
+
     def ecs_service
       @ecs_service ||= begin
                          executor = cf_executor
-                         if executor.stack_status.nil?
+                         if executor.stack_status.nil? || service_resource_id.nil?
                            nil
                          else
-                           rid = executor.resource_ids["ECSService"]
                            aws.ecs.describe_services(
                              cluster: service.district.name,
-                             services: [rid]
+                             services: [service_resource_id]
                            ).services.first
                          end
                        end

lib/cloud_formation/executor.rb

@@ -22,6 +22,16 @@ def initialize(stack, district)
       @bucket = district.s3_bucket_name
     end
 
+    def logger
+      @logger ||= ActiveSupport::TaggedLogging.new(Logger.new(STDOUT))
+    end
+
+    def putlog(severity, logstring)
+      logger.tagged("stack:#{stack&.name}") do
+        logger.public_send(severity) { logstring }
+      end
+    end
+
     def describe
       client.describe_stacks(stack_name: stack.name).stacks[0]
     rescue Aws::CloudFormation::Errors::ValidationError
@@ -48,7 +58,7 @@ def update(change_set: false)
       end
     rescue Aws::CloudFormation::Errors::ValidationError => e
       if e.message == "No updates are to be performed."
-        Rails.logger.warn "No updates are to be performed."
+        putlog(:warn, e.message)
       else
         raise e
       end
@@ -107,14 +117,26 @@ def create_or_update
       when "CREATE_COMPLETE", "UPDATE_COMPLETE", "UPDATE_ROLLBACK_COMPLETE"
         update
       when "ROLLBACK_COMPLETE"
-        # ROLLBACK_COMPLETE only happens when creating stack failed
-        # The only way to solve is to delete and re-create the stack
-        raise CannotUpdateRolledbackStackException
+        try_recreate
       else
         raise UpdateInProgressException
       end
     end
 
+    def try_recreate
+      putlog :warn, 'Attempting to re-create stack from ROLLBACK_COMPLETE'
+      delete
+
+      loop do
+        break if stack_status.nil? || stack_status == 'DELETE_COMPLETE'
+        putlog :warn, "Waiting for stack '#{stack.name}' to delete..."
+        sleep 10
+      end
+
+      putlog :warn, 'Stack deleted. Re-creating'
+      create
+    end
+
     def in_progress?
       status = stack_status
       return false if status.nil?

spec/lib/cloud_formation/executor_spec.rb

@@ -38,11 +38,22 @@
   end
 
   describe '#create_or_update' do
-    it 'raises an error if the stack was rolled back' do
-      allow(executor).to receive(:stack_status) { 'ROLLBACK_COMPLETE' }
-      expect { executor.create_or_update }.to raise_error CloudFormation::CannotUpdateRolledbackStackException
+    it 'deletes the stack if it was rolled back' do
+      status = 'ROLLBACK_COMPLETE'
+      allow(executor).to receive(:stack_status) { status }
+
+      expect(client).to receive(:delete_stack).with({stack_name: "test"}) {
+        status = 'DELETE_COMPLETE'
+      }.once
+
+      expect(executor).to receive(:create)
+
+
+      executor.create_or_update
     end
 
+
+
     it 'raises an error if the stack is still being updated' do
       allow(executor).to receive(:stack_status) { 'UPDATE_IN_PROGRESS' }
       expect { executor.create_or_update }.to raise_error CloudFormation::UpdateInProgressException