Test run should pass when Page crashed execution error occures (see #13)

src/tester/Runner.js

@@ -67,7 +67,7 @@ export default class Runner {
                 sendFailingScenario(results);
             }
 
-            if (results && results.executionError) {
+            if (results && results.executionError && instance.unknownExecutionErrorOccured) {
                 this._isSuccess = false;
             }
 

src/tester/__tests__/RunnerSpec.js

@@ -124,8 +124,8 @@ describe('Runner', () => {
         expect(messanger.report).toHaveBeenCalledWith('runner:end');
     });
 
-    it('can start a test instance and handle run with execution errors', async () => {
-        let instance = { clear: jest.fn() };
+    it('can start a test instance and handle run with unknown execution errors', async () => {
+        let instance = { clear: jest.fn(), unknownExecutionErrorOccured: true };
         messanger.requestScenario = jest
             .fn()
             .mockReturnValueOnce(Promise.resolve({ type: 'random', scenario: 'scenario' }))

src/tester/browser/Browser.js

@@ -2,6 +2,13 @@ import puppeteer from 'puppeteer';
 import EventEmitter from 'events';
 import { report } from '../messanger';
 
+/**
+ * These errors will not cause test run to exit as failrue
+ */
+const KNOWN_FATAL_ERROR_MESSAGES = [
+    'Page crashed!', // See https://github.com/seznam/QApe/issues/13
+];
+
 /**
  * Browser instance
  */
@@ -17,6 +24,8 @@ export default class Browser {
         this._page = null;
 
         this._pageErrorHandler = null;
+
+        this._unknownExecutionErrorOccured = false;
     }
 
     /**
@@ -45,6 +54,13 @@ export default class Browser {
         return this._pageErrorHandler;
     }
 
+    /**
+     * Test run with execution error will fail only when unknown execution error occured
+     */
+    get unknownExecutionErrorOccured() {
+        return this._unknownExecutionErrorOccured;
+    }
+
     /**
      * Initializes browser instance
      * @returns {Promise<Browser>}
@@ -110,10 +126,25 @@ export default class Browser {
                 'tester should recover by itself.';
 
             report('browser:error', { error: msg });
+
+            this._handleUnknownExecutionErrors(error);
+
             this.clear();
         });
     }
 
+    /**
+     * Sets unknown execution error indicator
+     * @param {Object} error
+     */
+    _handleUnknownExecutionErrors(error) {
+        if (KNOWN_FATAL_ERROR_MESSAGES.some((message) => error.message.includes(message))) {
+            return;
+        }
+
+        this._unknownExecutionErrorOccured = true;
+    }
+
     /**
      * Initializes pageErrorHandler, exposes it to the
      * page instance and loads config.pageErrorHandler

src/tester/browser/__tests__/BrowserSpec.js

@@ -62,6 +62,7 @@ describe('Browser', () => {
                 fn({ stack: 'error' });
             }),
         };
+        browser._handleUnknownExecutionErrors = jest.fn();
         browser.clear = jest.fn();
 
         browser._initFatalErrorHandler();
@@ -70,9 +71,20 @@ describe('Browser', () => {
         expect(messanger.report).toHaveBeenCalledWith('browser:error', {
             error: expect.any(String),
         });
+        expect(browser._handleUnknownExecutionErrors).toHaveBeenCalled();
         expect(browser.clear).toHaveBeenCalled();
     });
 
+    it('can handle unknown execution error', () => {
+        browser._handleUnknownExecutionErrors({ message: 'Error: Page crashed!' });
+
+        expect(browser.unknownExecutionErrorOccured).toEqual(false);
+
+        browser._handleUnknownExecutionErrors({ message: 'Unknown error' });
+
+        expect(browser.unknownExecutionErrorOccured).toEqual(true);
+    });
+
     it('can initialize page error handler', async () => {
         let eventEmitter = {};
         browser._getEventEmitter = jest.fn().mockReturnValue(eventEmitter);