From 230cb03ee0e9d0c87f7a59884e50b8955ba032d2 Mon Sep 17 00:00:00 2001 From: David Siaw Date: Thu, 31 Aug 2023 15:53:56 +0900 Subject: [PATCH 1/5] recreate instead of erroring --- lib/cloud_formation/executor.rb | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/cloud_formation/executor.rb b/lib/cloud_formation/executor.rb index 3ec4d33e..2a7c8837 100644 --- a/lib/cloud_formation/executor.rb +++ b/lib/cloud_formation/executor.rb @@ -107,14 +107,26 @@ def create_or_update when "CREATE_COMPLETE", "UPDATE_COMPLETE", "UPDATE_ROLLBACK_COMPLETE" update when "ROLLBACK_COMPLETE" - # ROLLBACK_COMPLETE only happens when creating stack failed - # The only way to solve is to delete and re-create the stack - raise CannotUpdateRolledbackStackException + try_recreate else raise UpdateInProgressException end end + def try_recreate + puts 'Attempting to re-create stack from ROLLBACK_COMPLETE' + delete + + loop do + break if stack_status.nil? || stack_status == 'DELETE_COMPLETE' + puts "Waiting for stack '#{stack.name}' to delete..." + sleep 10 + end + + puts 'Stack deleted. Re-creating' + create + end + def in_progress? status = stack_status return false if status.nil? From 2a57d5aa76620ff725254df1bd60386759ab0517 Mon Sep 17 00:00:00 2001 From: David Siaw Date: Thu, 7 Sep 2023 17:42:03 +0900 Subject: [PATCH 2/5] fix test --- spec/lib/cloud_formation/executor_spec.rb | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/spec/lib/cloud_formation/executor_spec.rb b/spec/lib/cloud_formation/executor_spec.rb index 20d94dee..d59b59a9 100644 --- a/spec/lib/cloud_formation/executor_spec.rb +++ b/spec/lib/cloud_formation/executor_spec.rb @@ -38,11 +38,22 @@ end describe '#create_or_update' do - it 'raises an error if the stack was rolled back' do - allow(executor).to receive(:stack_status) { 'ROLLBACK_COMPLETE' } - expect { executor.create_or_update }.to raise_error CloudFormation::CannotUpdateRolledbackStackException + it 'deletes the stack if it was rolled back' do + status = 'ROLLBACK_COMPLETE' + allow(executor).to receive(:stack_status) { status } + + expect(client).to receive(:delete_stack).with({stack_name: "test"}) { + status = 'DELETE_COMPLETE' + }.once + + expect(executor).to receive(:create) + + + executor.create_or_update end + + it 'raises an error if the stack is still being updated' do allow(executor).to receive(:stack_status) { 'UPDATE_IN_PROGRESS' } expect { executor.create_or_update }.to raise_error CloudFormation::UpdateInProgressException From e5b78ea2ac1bf921cbceebda96a39b281507433c Mon Sep 17 00:00:00 2001 From: David Siaw Date: Fri, 8 Sep 2023 12:09:34 +0900 Subject: [PATCH 3/5] attempt to fix nilness --- app/models/backend/ecs/v2/adapter.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/models/backend/ecs/v2/adapter.rb b/app/models/backend/ecs/v2/adapter.rb index 5ea8b15a..a17e1a35 100644 --- a/app/models/backend/ecs/v2/adapter.rb +++ b/app/models/backend/ecs/v2/adapter.rb @@ -49,16 +49,19 @@ def cf_executor @cf_executor ||= CloudFormation::Executor.new(service_stack, service.district) end + def service_resource_id + cf_executor.resource_ids["ECSService"] + end + def ecs_service @ecs_service ||= begin executor = cf_executor - if executor.stack_status.nil? + if executor.stack_status.nil? || service_resource_id.nil? nil else - rid = executor.resource_ids["ECSService"] aws.ecs.describe_services( cluster: service.district.name, - services: [rid] + services: [service_resource_id] ).services.first end end From 9295828eb3c528f50be55e80620d5f6a918cb6f7 Mon Sep 17 00:00:00 2001 From: David Siaw Date: Fri, 8 Sep 2023 14:36:35 +0900 Subject: [PATCH 4/5] use rails logger with tag --- lib/cloud_formation/executor.rb | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/cloud_formation/executor.rb b/lib/cloud_formation/executor.rb index 2a7c8837..32c63052 100644 --- a/lib/cloud_formation/executor.rb +++ b/lib/cloud_formation/executor.rb @@ -22,6 +22,16 @@ def initialize(stack, district) @bucket = district.s3_bucket_name end + def logger + @logger ||= ActiveSupport::TaggedLogging.new(Logger.new(STDOUT)) + end + + def putlog(logstring, severity: :info) + logger.tagged("stack:#{stack&.name}") do + logger.public_send(severity) { logstring } + end + end + def describe client.describe_stacks(stack_name: stack.name).stacks[0] rescue Aws::CloudFormation::Errors::ValidationError @@ -48,7 +58,7 @@ def update(change_set: false) end rescue Aws::CloudFormation::Errors::ValidationError => e if e.message == "No updates are to be performed." - Rails.logger.warn "No updates are to be performed." + putlog(:warn, e.message) else raise e end @@ -114,16 +124,16 @@ def create_or_update end def try_recreate - puts 'Attempting to re-create stack from ROLLBACK_COMPLETE' + putlog :warn, 'Attempting to re-create stack from ROLLBACK_COMPLETE' delete loop do break if stack_status.nil? || stack_status == 'DELETE_COMPLETE' - puts "Waiting for stack '#{stack.name}' to delete..." + putlog :warn, "Waiting for stack '#{stack.name}' to delete..." sleep 10 end - puts 'Stack deleted. Re-creating' + putlog :warn, 'Stack deleted. Re-creating' create end From 14e33b01b216394d422673dcf16ed426b01141b9 Mon Sep 17 00:00:00 2001 From: David Siaw Date: Fri, 8 Sep 2023 16:18:33 +0900 Subject: [PATCH 5/5] fix syntax --- lib/cloud_formation/executor.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cloud_formation/executor.rb b/lib/cloud_formation/executor.rb index 32c63052..3f470db0 100644 --- a/lib/cloud_formation/executor.rb +++ b/lib/cloud_formation/executor.rb @@ -26,7 +26,7 @@ def logger @logger ||= ActiveSupport::TaggedLogging.new(Logger.new(STDOUT)) end - def putlog(logstring, severity: :info) + def putlog(severity, logstring) logger.tagged("stack:#{stack&.name}") do logger.public_send(severity) { logstring } end