add configurable wait time to recover from ECONNRESET error

config/defaults/options.yml

@@ -4,6 +4,7 @@ defaults:
   log: ~
   client_timeout: 172800
   require_auth_token: false
+  connection_error_wait: 30
   alerts:
     email_to: ~
     email_from: ~

lib/backbeat/workers/async_worker.rb

@@ -76,6 +76,11 @@ def business_perform(event_class, node_data, options)
       rescue DeserializeError => e
         error(status: :deserialize_node_error, node: node_data["node_id"], error: e, backtrace: e.backtrace)
         raise e
+      rescue Errno::ECONNRESET => e
+        Kernel.sleep(Config.options[:connection_error_wait])
+        if (node.reload.current_client_status != :complete)
+          handle_processing_error(e, event_class, node, options)
+        end
       rescue => e
         handle_processing_error(e, event_class, node, options)
       end

spec/unit/workers/async_worker_spec.rb

@@ -79,6 +79,21 @@
       expect(Backbeat::Workers::AsyncWorker.jobs.count).to eq(1)
     end
 
+    it "waits for a client complete response if the server receives a connection reset" do
+      allow(Backbeat::Server).to receive(:fire_event) { raise Errno::ECONNRESET }
+      allow(Backbeat::Config).to receive(:options).and_return({connection_error_wait: 0.01})
+
+      expect(Kernel).to receive(:sleep).with(0.01)
+
+      Backbeat::Workers::AsyncWorker.new.perform(
+        Backbeat::Events::MarkChildrenReady.name,
+        { "node_class" => node.class.name, "node_id" => node.id },
+        { "retries" => 1 }
+      )
+
+      expect(Backbeat::Workers::AsyncWorker.jobs.count).to eq(1)
+    end
+
     it "retries the job if there is an error deserializing the node" do
       expect(Backbeat::Node).to receive(:find) { raise "Could not connect to the database" }