Merge pull request #728 from bugsnag/tms/appium-reporter

Reporting of Appium session success statuses to Bugsnag

CHANGELOG.md

@@ -1,4 +1,8 @@
-# 9.25.0 - 2025/03/05
+# 9.25.0 - 2025/03/06
+
+## Enhancements
+
+- Reporting of Appium session success statuses to Bugsnag [728](https://github.com/bugsnag/maze-runner/pull/728)
 
 ## Fixes
 

bin/maze-runner

@@ -29,6 +29,7 @@ require_relative '../lib/maze/client/appium/bs_client'
 require_relative '../lib/maze/client/appium/bs_legacy_client'
 require_relative '../lib/maze/client/appium/bs_devices'
 require_relative '../lib/maze/client/appium/local_client'
+require_relative '../lib/maze/client/appium/session_metadata'
 require_relative '../lib/maze/client/selenium'
 require_relative '../lib/maze/client/selenium/base_client'
 require_relative '../lib/maze/client/selenium/bb_client'

docker-compose.yml

@@ -16,6 +16,7 @@ x-common-environment: &common-environment
   BUILDKITE_REPO:
   BUILDKITE_RETRY_COUNT:
   BUILDKITE_STEP_KEY:
+  MAZE_APPIUM_BUGSNAG_API_KEY:
   MAZE_BUGSNAG_API_KEY:
   MAZE_SCENARIO_BUGSNAG_API_KEY:
 

lib/maze/api/appium/app_manager.rb

@@ -19,7 +19,7 @@ def activate
           true
         rescue Selenium::WebDriver::Error::ServerError => e
           # Assume the remote appium session has stopped, so crash out of the session
-          fail_driver
+          fail_driver(e.message)
           raise e
         end
 
@@ -35,7 +35,7 @@ def terminate
           true
         rescue Selenium::WebDriver::Error::ServerError => e
           # Assume the remote appium session has stopped, so crash out of the session
-          fail_driver
+          fail_driver(e.message)
           raise e
         end
 
@@ -51,7 +51,7 @@ def launch
           true
         rescue Selenium::WebDriver::Error::ServerError => e
           # Assume the remote appium session has stopped, so crash out of the session
-          fail_driver
+          fail_driver(e.message)
           raise e
         end
 
@@ -67,7 +67,7 @@ def close
           true
         rescue Selenium::WebDriver::Error::ServerError => e
           # Assume the remote appium session has stopped, so crash out of the session
-          fail_driver
+          fail_driver(e.message)
           raise e
         end
 
@@ -82,7 +82,7 @@ def state
           @driver.app_state(@driver.app_id)
         rescue Selenium::WebDriver::Error::ServerError => e
           # Assume the remote appium session has stopped, so crash out of the session
-          fail_driver
+          fail_driver(e.message)
           raise e
         end
       end

lib/maze/api/appium/device_manager.rb

@@ -20,7 +20,7 @@ def unlock
           true
         rescue Selenium::WebDriver::Error::ServerError => e
           # Assume the remote appium session has stopped, so crash out of the session
-          fail_driver
+          fail_driver(e.message)
           raise e
         end
 
@@ -37,7 +37,7 @@ def set_rotation(orientation)
           true
         rescue Selenium::WebDriver::Error::ServerError => e
           # Assume the remote appium session has stopped, so crash out of the session
-          fail_driver
+          fail_driver(e.message)
           raise e
         end
 
@@ -52,7 +52,7 @@ def info
           JSON.generate(@driver.device_info)
         rescue Selenium::WebDriver::Error::ServerError => e
           # Assume the remote appium session has stopped, so crash out of the session
-          fail_driver
+          fail_driver(e.message)
           raise e
         end
       end

lib/maze/api/appium/file_manager.rb

@@ -36,7 +36,7 @@ def write_app_file(contents, filename)
           false
         rescue Selenium::WebDriver::Error::ServerError => e
           # Assume the remote appium session has stopped, so crash out of the session
-          fail_driver
+          fail_driver(e.message)
           raise e
         end
 
@@ -73,7 +73,7 @@ def read_app_file(filename, directory = nil)
           nil
         rescue Selenium::WebDriver::Error::ServerError => e
           # Assume the remote appium session has stopped, so crash out of the session
-          fail_driver
+          fail_driver(e.message)
           raise e
         end
       end

lib/maze/api/appium/manager.rb

@@ -11,8 +11,8 @@ def failed_driver?
           @driver.failed?
         end
 
-        def fail_driver
-          @driver.fail_driver
+        def fail_driver(message)
+          @driver.fail_driver(message)
         end
       end
     end

lib/maze/client/appium/base_client.rb

@@ -8,7 +8,6 @@ class BaseClient
         FIXTURE_CONFIG = 'fixture_config.json'
 
         def initialize
-          @session_ids = []
           @start_attempts = 0
         end
 
@@ -79,9 +78,9 @@ def attempt_start_driver(config)
           if result
             # Log details of this session
             $logger.info "Created Appium session: #{driver.session_id}"
-            @session_ids << driver.session_id
-            udid = driver.session_capabilities['udid']
-            $logger.info "Running on device: #{udid}" unless udid.nil?
+            @session_metadata = Maze::Client::Appium::SessionMetadata.new
+            @session_metadata.id = driver.session_id
+            @session_metadata.farm = Maze.config.farm.to_s
           end
           driver
         end
@@ -147,9 +146,47 @@ def log_run_outro
         end
 
         def stop_session
-          Maze.driver.driver_quit unless Maze.driver.failed?
+          if Maze.driver.failed?
+            @session_metadata.success = false
+            @session_metadata.failure_message = Maze.driver.failure_reason
+          else
+            # TODO: The call to quit could also fail
+            Maze.driver.driver_quit
+            @session_metadata.success = true
+          end
+
+          # Report session outcome to Bugsnag
+          report_session if ENV['MAZE_APPIUM_BUGSNAG_API_KEY']
+
           Maze::AppiumServer.stop if Maze::AppiumServer.running
         end
+
+        def report_session
+          message = @session_metadata.success ? 'Success' : @session_metadata.failure_message
+          error = Exception.new(message)
+
+          Bugsnag.notify(error) do |event|
+            event.api_key = ENV['MAZE_APPIUM_BUGSNAG_API_KEY']
+            event.grouping_hash = event.exceptions.first[:message]
+
+            metadata = {
+              'session id': @session_metadata.id,
+              'success': @session_metadata.success,
+              'device farm': @session_metadata.farm.to_s,
+            }
+            metadata['device'] = @session_metadata.device if @session_metadata.device
+
+            if @session_metadata.success
+              event.severity = 'info'
+            else
+              event.severity = 'error'
+              event.unhandled = true
+              metadata['failure message'] = @session_metadata.failure_message
+            end
+
+            event.add_metadata(:'Appium Session', metadata)
+          end
+        end
       end
     end
   end

lib/maze/client/appium/bb_client.rb

@@ -91,14 +91,12 @@ def log_run_intro
         def log_run_outro
           api_client = BitBarApiClient.new(Maze.config.access_key)
 
-          $logger.info 'Appium session(s) created:'
-          @session_ids.each do |id|
-            info = api_client.get_device_session_info(id)
-            if info
-              link = Maze::Loggers::LogUtil.linkify(info[:dashboard_link], "BitBar session: #{id}")
-              $logger.info link
-              $logger.info "Device used: #{info[:device_name]}"
-            end
+          info = api_client.get_device_session_info(@session_metadata.id)
+          if info
+            link = Maze::Loggers::LogUtil.linkify(info[:dashboard_link], "BitBar session: #{@session_metadata.id}")
+            $logger.info link
+            @session_metadata.device = info[:device_name]
+            $logger.info "Device used: #{@session_metadata.device}"
           end
         end
 

lib/maze/client/appium/bs_client.rb

@@ -72,8 +72,7 @@ def log_run_intro
         end
 
         def log_run_outro
-          $logger.info 'Appium session(s) created:'
-          @session_ids.each { |id| $logger.info "  #{id}" }
+          $logger.info "Appium session created: #{@session_metadata.id}"
           log_run_intro
         end
 

lib/maze/client/appium/session_metadata.rb

@@ -0,0 +1,18 @@
+module Maze
+  module Client
+    module Appium
+      class SessionMetadata
+        def initialize
+          @success = false
+          @failure_message = 'Default failure message'
+        end
+
+        attr_accessor :id
+        attr_accessor :farm
+        attr_accessor :device
+        attr_accessor :success
+        attr_accessor :failure_message
+      end
+    end
+  end
+end

lib/maze/client/bb_api_client.rb

@@ -88,6 +88,7 @@ def get_device_session_info(session_id)
           }
         else
           $logger.warn "Could not retrieve details for device session #{session_id}, BitBar may not have had time to create it yet."
+          nil
         end
       end
 

lib/maze/driver/appium.rb

@@ -33,6 +33,7 @@ def initialize(server_url, capabilities, locator = :id)
         capabilities ||= {}
 
         @failed = false
+        @failure_reason = ''
         @element_locator = locator
         @capabilities = capabilities
 
@@ -70,10 +71,15 @@ def failed?
         @failed
       end
 
+      def failure_reason
+        @failure_reason
+      end
+
       # Marks the driver as failed
-      def fail_driver
-        $logger.error 'Appium driver failed, remaining scenarios will be skipped'
+      def fail_driver(reason)
+        $logger.error "Appium driver failed: #{reason}"
         @failed = true
+        @failure_reason = reason
       end
 
       # Checks for an element, waiting until it is present or the method times out
@@ -95,7 +101,7 @@ def wait_for_element(element_id, timeout = 15, retry_if_stale = true)
         end
       rescue Selenium::WebDriver::Error::ServerError => e
         # Assume the remote appium session has stopped, so crash out of the session
-        fail_driver
+        fail_driver(e.message)
         raise e
       else
         true
@@ -106,7 +112,7 @@ def launch_app
         super
       rescue Selenium::WebDriver::Error::ServerError => e
         # Assume the remote appium session has stopped, so crash out of the session
-        fail_driver
+        fail_driver(e.message)
         raise e
       end
 
@@ -115,7 +121,7 @@ def close_app
         super
       rescue Selenium::WebDriver::Error::ServerError => e
         # Assume the remote appium session has stopped, so crash out of the session
-        fail_driver
+        fail_driver(e.message)
         raise e
       end
 
@@ -126,7 +132,7 @@ def find_element_timed(element_id)
         end
       rescue Selenium::WebDriver::Error::ServerError => e
         # Assume the remote appium session has stopped, so crash out of the session
-        fail_driver
+        fail_driver(e.message)
         raise e
       end
 
@@ -140,7 +146,7 @@ def click_element(element_id)
         end
       rescue Selenium::WebDriver::Error::ServerError => e
         # Assume the remote appium session has stopped, so crash out of the session
-        fail_driver
+        fail_driver(e.message)
         raise e
       end
 
@@ -158,7 +164,7 @@ def click_element_if_present(element_id)
         false
       rescue Selenium::WebDriver::Error::ServerError => e
         # Assume the remote appium session has stopped, so crash out of the session
-        fail_driver
+        fail_driver(e.message)
         raise e
       end
 
@@ -172,7 +178,7 @@ def clear_element(element_id)
         end
       rescue Selenium::WebDriver::Error::ServerError => e
         # Assume the remote appium session has stopped, so crash out of the session
-        fail_driver
+        fail_driver(e.message)
         raise e
       end
 
@@ -197,7 +203,7 @@ def send_keys_to_element(element_id, text)
         end
       rescue Selenium::WebDriver::Error::ServerError => e
         # Assume the remote appium session has stopped, so crash out of the session
-        fail_driver
+        fail_driver(e.message)
         raise e
       end
 
@@ -234,7 +240,7 @@ def clear_and_send_keys_to_element(element_id, text)
         end
       rescue Selenium::WebDriver::Error::ServerError => e
         # Assume the remote appium session has stopped, so crash out of the session
-        fail_driver
+        fail_driver(e.message)
         raise e
       end
 

test/unit/maze/client/appium/bb_client_test.rb

@@ -9,6 +9,7 @@
 require_relative '../../../../../lib/maze/client/appium/base_client'
 require_relative '../../../../../lib/maze/client/appium/bb_client'
 require_relative '../../../../../lib/maze/client/appium/bb_devices'
+require_relative '../../../../../lib/maze/client/appium/session_metadata'
 require_relative '../../../../../lib/utils/deep_merge'
 
 module Maze
@@ -78,7 +79,6 @@ def test_start_driver_success
 
           # Successful starting of the driver
           @mock_driver.expects(:session_id).twice.returns 'session_id'
-          @mock_driver.expects(:session_capabilities).returns({ 'uuid' => 'uuid' })
           Bugsnag.expects(:notify).never
 
           client = BitBarClient.new
@@ -104,7 +104,6 @@ def test_start_driver_recovers
           #
           $logger.expects(:info).with('Created Appium session: session_id')
           @mock_driver.expects(:session_id).twice.returns 'session_id'
-          @mock_driver.expects(:session_capabilities).returns({ 'uuid' => 'uuid' })
           Bugsnag.expects(:notify).never
 
           client = BitBarClient.new