Mark tests as passed/failed in Browserstack with reason

README.md

@@ -1,7 +1,17 @@
-# protractor-browserstack
+# protractor-browserstack with marking tests as passed/failed with reason 
+
+#### The incentive behind this fork is to demonstrate how to mark tests as passed/ failed in browserstack. <br/>Browserstack's current documentation does not allow you to do so. So even if your test fails it will be marked as passed (green) on browserstack.<br/><br/>In addition, there is an example of how to update the reason for the test's failure using browserstack rest api. see 'single_failed.js'
 
 [Protractor](https://github.com/angular/protractor/) Integration with BrowserStack.
 
+Mark tests as passed/failed in Browserstack
+
+Report reason for failed test to Browserstack
+
+Report failed expectations to Browserstack
+
+Upgrade to support Protractor 5   
+
 ![BrowserStack Logo](https://d98b8t1nnulk5.cloudfront.net/production/images/layout/logo-header.png?1469004780)
 
 <img src ="http://nchaulet.github.io/slide-protractor-lyonjs/images/protractor-logo.png" height = "100">
@@ -14,6 +24,7 @@
 
 ## Running your tests
 * To run a single test, run `npm run single`
+* To run a single test that will report the reason for failed tests to browserstack, run `npm run single_failed`
 * To run local tests, run `npm run local`
 * To run parallel tests, run `npm run parallel`
 

conf/local.conf.js

@@ -1,12 +1,11 @@
 var browserstack = require('browserstack-local');
-
 exports.config = {
   'specs': [ '../specs/local.js' ],
-  'seleniumAddress': 'http://hub-cloud.browserstack.com/wd/hub',
+
+  browserstackUser: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME',
+  browserstackKey: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY',
 
   'capabilities': {
-    'browserstack.user': process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME',
-    'browserstack.key': process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY',
     'build': 'protractor-browserstack',
     'name': 'local_test',
     'browserName': 'chrome',
@@ -19,7 +18,7 @@ exports.config = {
     console.log("Connecting local");
     return new Promise(function(resolve, reject){
       exports.bs_local = new browserstack.Local();
-      exports.bs_local.start({'key': exports.config.capabilities['browserstack.key'] }, function(error) {
+      exports.bs_local.start({'key': exports.config.browserstackKey, force: true}, function (error) {
         if (error) return reject(error);
         console.log('Connected. Now testing...');
 

conf/local_failed.conf.js

@@ -0,0 +1,36 @@
+var browserstack = require('browserstack-local');
+exports.config = {
+  'specs': [ '../specs/local_fail.js' ],
+
+  browserstackUser: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME',
+  browserstackKey: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY',
+
+  'capabilities': {
+    'build': 'protractor-browserstack',
+    'name': 'local test reporting failed',
+    'browserName': 'chrome',
+    'browserstack.local': true,
+    'browserstack.debug': 'true'
+  },
+
+  // Code to start browserstack local before start of test
+  beforeLaunch: function(){
+    console.log("Connecting local");
+    return new Promise(function(resolve, reject){
+      exports.bs_local = new browserstack.Local();
+      exports.bs_local.start({'key': exports.config.browserstackKey, force: true}, function (error) {
+        if (error) return reject(error);
+        console.log('Connected. Now testing...');
+
+        resolve();
+      });
+    });
+  },
+
+  // Code to stop browserstack local after end of test
+  afterLaunch: function(){
+    return new Promise(function(resolve, reject){
+      exports.bs_local.stop(resolve);
+    });
+  }
+};

conf/parallel.conf.js

@@ -1,10 +1,10 @@
 exports.config = {
   'specs': [ '../specs/single.js' ],
-  'seleniumAddress': 'http://hub-cloud.browserstack.com/wd/hub',
+
+  browserstackUser: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME',
+  browserstackKey: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY',
 
   'commonCapabilities': {
-    'browserstack.user': process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME',
-    'browserstack.key': process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY',
     'build': 'protractor-browserstack',
     'name': 'parallel_test',
     'browserstack.debug': 'true',

conf/parallel_local.conf.js

@@ -2,11 +2,11 @@ var browserstack = require('browserstack-local');
 
 exports.config = {
   'specs': [ '../specs/local.js' ],
-  'seleniumAddress': 'http://hub-cloud.browserstack.com/wd/hub',
+
+  browserstackUser: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME',
+  browserstackKey: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY',
 
   'commonCapabilities': {
-    'browserstack.user': process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME',
-    'browserstack.key': process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY',
     'build': 'protractor-browserstack',
     'name': 'parallel_local_test',
     'browserstack.local': true,
@@ -26,7 +26,7 @@ exports.config = {
     console.log("Connecting local");
     return new Promise(function(resolve, reject){
       exports.bs_local = new browserstack.Local();
-      exports.bs_local.start({'key': exports.config.commonCapabilities['browserstack.key'] }, function(error) {
+      exports.bs_local.start({'key': exports.config.browserstackKey }, function(error) {
         if (error) return reject(error);
         console.log('Connected. Now testing...');
 

conf/single.conf.js

@@ -1,12 +1,12 @@
 exports.config = {
   'specs': [ '../specs/single.js' ],
-  'seleniumAddress': 'http://hub-cloud.browserstack.com/wd/hub',
+
+  browserstackUser: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME',
+  browserstackKey: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY',
 
   'capabilities': {
-    'browserstack.user': process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME',
-    'browserstack.key': process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY',
     'build': 'protractor-browserstack',
-    'name': 'single_test',
+    'name': 'single test',
     'browserName': 'chrome',
     'resolution': '1024x768',
     'browserstack.debug': 'true'

conf/single_fail.conf.js

@@ -0,0 +1,14 @@
+exports.config = {
+  'specs': [ '../specs/single_fail.js' ],
+
+  browserstackUser: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME',
+  browserstackKey: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY',
+
+  'capabilities': {
+    'build': 'protractor-browserstack',
+    'name': 'single test reporting failed',
+    'browserName': 'chrome',
+    'resolution': '1024x768',
+    'browserstack.debug': 'true'
+  }
+};

lib/BrowserstackErrorReporter.js

@@ -0,0 +1,34 @@
+const request = require('request');
+class BrowserstackErrorReporter {
+  constructor() {
+    this.BROWSERSTACK_USERNAME = process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME';
+    this.BROWSERSTACK_ACCESS_KEY = process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY';
+    this.errors = [];
+    this.sessionId = "";
+  }
+
+  buildErrorsReport() {
+    return JSON.stringify(this.errors);
+  }
+
+  setSessionId(sessionId) {
+    this.sessionId = sessionId;
+  }
+
+  addError(title, error) {
+    // this.errors.push({title, error});
+    this.errors.push(title);
+  }
+
+  async reportErrors() {
+    if (this.errors.length) {
+      await request({
+        uri: `https://${this.BROWSERSTACK_USERNAME}:${this.BROWSERSTACK_ACCESS_KEY}@api.browserstack.com/automate/sessions/${this.sessionId}.json`,
+        method: 'PUT',
+        form: {status: 'failed', reason: this.buildErrorsReport()}
+      });
+    }
+  }
+}
+
+module.exports = BrowserstackErrorReporter;

package.json

@@ -6,7 +6,9 @@
   "scripts": {
     "test": "npm run single && npm run local && npm run parallel",
     "single": "./node_modules/.bin/protractor conf/single.conf.js",
+    "single_fail": "./node_modules/.bin/protractor conf/single_fail.conf.js",
     "local": "./node_modules/.bin/protractor conf/local.conf.js",
+    "local_fail": "./node_modules/.bin/protractor conf/local_fail.conf.js",
     "parallel": "./node_modules/.bin/protractor conf/parallel.conf.js",
     "parallel_local": "./node_modules/.bin/protractor conf/parallel_local.conf.js"
   },
@@ -16,7 +18,7 @@
   },
   "dependencies": {
     "browserstack-local": "^1.0.0",
-    "protractor": "^2.5.1"
+    "protractor": "^5.3.1"
   },
   "license": "MIT"
 }

specs/local_fail.js

@@ -0,0 +1,6 @@
+describe('BrowserStack Local Testing', function() {
+  it('should fail and report to browserstack', function() {
+    expect(true).toBe(false);
+  });
+});
+

specs/single.js

@@ -2,11 +2,11 @@ describe('Google\'s Search Functionality', function() {
   it('can find search results', function() {
     browser.driver.get('https://google.com/ncr').then(function() {
       browser.driver.findElement(by.name('q')).sendKeys('BrowserStack').then(function() {
-        browser.driver.findElement(by.name('btnG')).click().then(function() {
+        browser.actions().sendKeys(protractor.Key.ENTER).perform().then(function() {
           browser.driver.wait(function() {
             return browser.driver.findElements(by.id('resultStats')).then(function(elems) {
               return elems.length > 0;
-            });;
+            });
           });
           expect(browser.driver.getTitle()).toEqual('BrowserStack - Google Search');
         });

specs/single_fail.js

@@ -0,0 +1,39 @@
+var BrowserstackErrorReporter = require('../lib/BrowserstackErrorReporter');
+
+jasmine.getEnv().addReporter({
+  specStarted(result) {
+    jasmine.getEnv().currentSpec = result;
+  },
+  specDone() {
+    jasmine.getEnv().currentSpec = null;
+  }
+});
+
+describe('Reporting to BrowserStack that test failed with reason', function() {
+
+  const reporter = new BrowserstackErrorReporter();
+  beforeAll(async () => {
+    await protractor.browser.driver.getSession().then((session) => {
+      reporter.setSessionId(session['id_']);
+    });
+  });
+
+  afterEach(function () {
+    let failedExpectations = jasmine.getEnv().currentSpec.failedExpectations;
+    if (failedExpectations.length) {
+      failedExpectations.forEach(f => reporter.addError(f.message, f.stack));
+    }
+  });
+
+  afterAll(async () => {
+    return reporter.reportErrors();
+  });
+
+  it('should fail and report reason to browserstack', function() {
+    expect(true).toBe(false);
+  });
+
+  it('should not fail and will not be reported to browserstack', function() {
+    expect(true).toBe(true);
+  });
+});