Enable deploy tracking

app/jobs/deploy_runner_job.rb

@@ -42,6 +42,10 @@ def perform(heritage, without_before_deploy:, description: "")
         sleep 3
       end
     end
+
+  rescue => e
+    Rails.logger.error e
+    Rails.logger.error e.backtrace
   end
 
   def other_deploy_in_progress?(heritage)
@@ -51,6 +55,7 @@ def other_deploy_in_progress?(heritage)
   end
 
   def notify(level: :good, message:)
+    Rails.logger.info message
     Event.new(@heritage.district).notify(level: level, message: "[#{@heritage.name}] #{message}")
   end
 end

app/jobs/monitor_deployment_job.rb

@@ -2,6 +2,14 @@ class MonitorDeploymentJob < ActiveJob::Base
   queue_as :default
 
   def perform(service, count: 0, deployment_id: nil)
+    if service.heritage.version == 2
+      ServiceDeployment.create!(service: service)
+      return
+    end
+
+    # old version does not rely on cloudformation and thus has to be
+    # polled one by one. We will need to clean this up later.
+
     if service.deployment_finished?(deployment_id)
       notify(service, message: "#{service.name} service deployed")
     elsif count > 20
@@ -15,6 +23,7 @@ def perform(service, count: 0, deployment_id: nil)
   end
 
   def notify(service, level: :good, message:)
+    Rails.logger.info message
     Event.new(service.district).notify(level: level, message: "[#{service.heritage.name}] #{message}")
   end
 end

app/models/service.rb

@@ -82,8 +82,11 @@ def https_port_mapping
     port_mappings.find_by(protocol: 'https')
   end
 
-  def deployment_finished?(deployment_id)
-    backend.deployment_finished?(deployment_id)
+  def deployment_finished?(deployment_id=nil)
+    backend.deployment_finished?(deployment_id) if heritage.version == 1
+
+    return true if service_deployment_object.nil?
+    service_deployment_object.finished?
   end
 
   def save_and_update_container_count!(desired_container_count)
@@ -123,6 +126,10 @@ def service_arns
     s.flat_map(&:service_arns)
   end
 
+  def stack_name
+    "#{district.name}-#{heritage.name}-#{name}"
+  end
+
   def arn_prefix
     [
       'arn:aws:ecs',
@@ -141,8 +148,20 @@ def arn_prefix_legacy
     ].join(':')
   end
 
+  def deployment
+    if service_deployment_object.nil?
+      ServiceDeployment.create!(service: self)
+    end
+
+    service_deployment_object
+  end
+
   private
 
+  def service_deployment_object
+    service_deployments.unfinished.last || service_deployments.last
+  end
+
   def ecs
     @ecs ||= district.aws.ecs
   end

app/services/deploy_service.rb

@@ -0,0 +1,153 @@
+class DeployService
+
+  STATUS_TO_ACTION_MAP = {
+                              "CREATE_IN_PROGRESS" => :incomplete,
+                                   "CREATE_FAILED" => :failed,
+                                 "CREATE_COMPLETE" => :completed,
+                            "ROLLBACK_IN_PROGRESS" => :incomplete,
+                                 "ROLLBACK_FAILED" => :failed,
+                               "ROLLBACK_COMPLETE" => :failed,
+                              "DELETE_IN_PROGRESS" => :incomplete,
+                                   "DELETE_FAILED" => :failed,
+                                 "DELETE_COMPLETE" => :completed,
+                              "UPDATE_IN_PROGRESS" => :incomplete,
+             "UPDATE_COMPLETE_CLEANUP_IN_PROGRESS" => :incomplete,
+                                 "UPDATE_COMPLETE" => :completed,
+                                   "UPDATE_FAILED" => :failed,
+                     "UPDATE_ROLLBACK_IN_PROGRESS" => :failed,
+                          "UPDATE_ROLLBACK_FAILED" => :failed,
+    "UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS" => :failed,
+                        "UPDATE_ROLLBACK_COMPLETE" => :failed,
+                              "REVIEW_IN_PROGRESS" => :incomplete,
+                              "IMPORT_IN_PROGRESS" => :incomplete,
+                                 "IMPORT_COMPLETE" => :completed,
+                     "IMPORT_ROLLBACK_IN_PROGRESS" => :failed,
+                          "IMPORT_ROLLBACK_FAILED" => :failed,
+                        "IMPORT_ROLLBACK_COMPLETE" => :failed
+  }.freeze
+
+  class << self
+    def deploy_service(service)
+      ServiceDeployment.create!(service: service)
+    end
+
+    DEPLOY_SERVICE_LOCK=1000001
+
+    def synchronize
+      got_lock = ActiveRecord::Base.connection.get_advisory_lock(DEPLOY_SERVICE_LOCK)
+      if !got_lock
+        Rails.logger.info("[DeployService] Lock already held. Skipping.")
+      return
+
+      yield
+    ensure
+      if got_lock && !ActiveRecord::Base.connection.release_advisory_lock(DEPLOY_SERVICE_LOCK)
+        Rails.logger.info("[DeployService] Failed to release lock")
+      end
+    end
+
+    def check_all
+      Rails.logger.info("[DeployService] Starting checks.")
+
+      synchronize do
+        District.all.each do |district|
+          Rails.logger.info("Checking district #{district.name}")
+          DeployService.new(district).check
+        end
+      end
+    end
+  end
+
+  def initialize(district)
+    @district = district
+  end
+
+  def check
+    @district.heritages.each do |heritage|
+      heritage.services.each do |service|
+        next if service.deployment.finished?
+
+        status = stack_statuses[service.stack_name]
+        action = STATUS_TO_ACTION_MAP[status]
+        notify(service, action)
+      end
+    end
+  end
+
+  def notify(service, action)
+    if action.nil?
+      Rails.logger.error("[deploy_service] stack #{service.stack_name} not found!")
+      return
+    end
+
+    send("notify_#{action}", service)
+  end
+
+  def notify_completed(service)
+    Rails.logger.info("Heritage: #{service.heritage.name} Service: #{service.name} Deployment Completed")
+    service.service_deployments.unfinished.each do |record|
+      record.complete!
+    end
+    event(service, message: "#{service.name} service deployed")
+  end
+
+  def notify_incomplete(service)
+    Rails.logger.info("Heritage: #{service.heritage.name} Service: #{service.name} Deployment Incomplete")
+    runtime = Time.now - service.deployment.created_at
+    if runtime > 20.minutes
+      event(service, level: :error, message: "Deploying #{service.name} service has not finished for a while.")
+    end
+  end
+
+  def notify_failed(service)
+    Rails.logger.info("Heritage: #{service.heritage.name} Service: #{service.name} Deployment Failed")
+    service.service_deployments.unfinished.each do |record|
+      record.fail!
+    end
+    event(service, level: :error, message: "Deployment of #{service.name} service has failed.")
+  end
+
+  def stack_names
+    @stack_names ||= begin
+      results = {}
+      @district.heritages.map do |heritage|
+        heritage.services.map do |service|
+          results[service.stack_name] = true
+        end
+      end
+      results
+    end
+  end
+
+  def stack_statuses
+    @stack_statuses ||= begin
+      results = {}
+
+      cloudformation.list_stacks.each do |response|
+        response.stack_summaries.each do |summary|
+          if stack_names.key?(summary.stack_name)
+            results[summary.stack_name] = summary.stack_status
+          end
+        end
+      end
+
+      Rails.logger.info(results.to_yaml)
+
+      results
+    rescue StandardError => e
+      Rails.logger.error("Failed to retrieve stack statuses!")
+      raise e
+    end
+  end
+
+  private
+
+  def event(service, level: :good, message:)
+    Event.new(@district).notify(level: level, message: "[#{service.heritage.name}] #{message}")
+  end
+
+  def cloudformation
+    @cloudformation ||= @district.aws.cloudformation
+  end
+
+end

barcelona.yml

@@ -1,12 +1,18 @@
+scheduled_tasks: &scheduled_tasks
+  scheduled_tasks:
+      # 10AM JST every week day
+      - schedule: cron(0 1 ? * MON-FRI *)
+        command: bin/chaos
+      # every 5 minutes
+      - schedule: cron(0/5 * ? * * *)
+        command: rake bcn:deployment_check
+
 environments:
   production:
+    <<: *scheduled_tasks
     name: barcelona2
     image_name: public.ecr.aws/degica/barcelona
     before_deploy: rake db:migrate
-    scheduled_tasks:
-      # 10AM JST every week day
-      - schedule: cron(0 1 ? * MON-FRI *)
-        command: bin/chaos
     services:
       - name: web
         service_type: web
@@ -23,13 +29,10 @@ environments:
         cpu: 128
         memory: 256
   test:
+    <<: *scheduled_tasks
     name: barcelona
     image_name: public.ecr.aws/degica/barcelona
     before_deploy: rake db:migrate
-    scheduled_tasks:
-      # 10AM JST every week day
-      - schedule: cron(0 1 ? * MON-FRI *)
-        command: bin/chaos
     services:
       - name: web
         service_type: web

lib/tasks/deployment_check.rake

@@ -0,0 +1,10 @@
+namespace :bcn do
+  desc "Check deployments"
+  task :deployment_check => :environment do
+    Rails.logger = Logger.new(STDOUT)
+    Rails.logger.level = :info
+
+    Rails.logger.info("Starting deployment check...")
+    DeployService.check_all
+  end
+end

spec/models/service_spec.rb

@@ -210,4 +210,36 @@
       expect(s2.service_deployments).to eq [s2d1]
     end
   end
+
+  describe '#deployment_finished?' do
+    it 'returns true if there are no deployments (backwards compat)' do
+      s = create :service
+
+      expect(s).to be_deployment_finished
+    end
+
+    it 'returns true if the last one is finished' do
+      s = create :service
+      create :service_deployment, service: s, completed_at: Time.now
+
+      expect(s).to be_deployment_finished
+    end
+
+    it 'returns false if the last one is not yet finished' do
+      s = create :service
+      create :service_deployment, service: s
+
+      expect(s).to_not be_deployment_finished
+    end
+  end
+
+  describe '#stack_name' do
+    it 'returns the corresponding stack name' do
+      s = create :service, name: 'serv'
+      allow(s.district).to receive(:name) { 'dist' }
+      allow(s.heritage).to receive(:name) { 'heri' }
+
+      expect(s.stack_name).to eq "dist-heri-serv"
+    end
+  end
 end