feat: add junit reporter (#235)

* feat: add junit reporter

* chore: add proper indentation

* chore: enable junit reporting for the e2e

* ci: enable junit reporting for the e2e

* ci: use the stage timeout

* update timeout

* ci: use parent folder to store the junit and debug traces

* fix: show stacktrace for failed test

* chore: fix stack trace test

* feat: add support for env flag

* chore: support SYNTHETICS_JUNIT_FILE

* chore: not required

Co-authored-by: Victor Martinez <[email protected]>
Co-authored-by: Shahzad <[email protected]>

.ci/Jenkinsfile

@@ -121,6 +121,9 @@ pipeline {
       }
     }
     stage('E2e Test') {
+      environment {
+        E2E_FOLDER = "__tests__/e2e"
+      }
       options {
         timeout(15)
       }
@@ -132,15 +135,19 @@ pipeline {
               sh(label: 'set permissions', script: '''
                 chmod -R ugo+rw examples
               ''')
-              dir("__tests__/e2e"){
-                timeout(time: 10, unit: 'MINUTES') {
-                  sh(label: 'run e2e tests',script: 'npm run test')
-                }
+              dir("${E2E_FOLDER}"){
+                sh(label: 'run e2e tests',script: 'npm run test')
               }
             }
           }
         }
       }
+      post {
+        always {
+          archiveArtifacts(allowEmptyArchive: true, artifacts: "${BASE_DIR}/${E2E_FOLDER}/junit.xml")
+          junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/${E2E_FOLDER}/junit.xml")
+        }
+      }
     }
     /**
      Publish Docker images.

.gitignore

@@ -109,6 +109,7 @@ tmp
 *.swp
 
 __tests__/e2e/tmp/
+__tests__/e2e/junit.xml
 
 .idea/
-seccomp/build
+seccomp/build

__tests__/e2e/docker-compose.yml

@@ -30,7 +30,7 @@ services:
       - elasticsearch
       - synthetic
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:5601/api/status"]
+      test: ["CMD", "curl", "-s", "-f", "http://localhost:5601/api/status"]
       interval: 30s
       retries: 10
       start_period: 30s

__tests__/e2e/scripts/setup.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -xe
+set -e
 
 # variables
 

__tests__/e2e/scripts/test.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -xe
+set -eo pipefail
 
 if [ -z "${JENKINS_URL}" ]; then
   # formatting
@@ -14,7 +14,7 @@ fi
 ##################################################
 echo "" # newline
 echo "${bold}Waiting for synthetics docker to start...${normal}"
-until [ "$(docker inspect -f {{.State.Running}} synthetics)" == "true" ]; do
+until [ "$(docker inspect -f '{{.State.Running}}' synthetics)" == "true" ]; do
   sleep ${SLEEP_TIME};
 done;
 
@@ -23,5 +23,4 @@ echo "✅ Setup completed successfully. Running e2e tests..."
 #
 # run e2e tests journey
 ##################################################
-
-npx @elastic/synthetics uptime.journey.ts
+SYNTHETICS_JUNIT_FILE='junit.xml' npx @elastic/synthetics uptime.journey.ts --reporter junit

__tests__/e2e/uptime.journey.ts

@@ -43,10 +43,13 @@ journey('E2e test synthetics', async ({ page }) => {
 
   step('Go to kibana uptime app', async () => {
     await page.goto('http://localhost:5620/app/uptime');
+    await page.waitForTimeout(30 * 1000);
   });
 
   step('Check if there is table data', async () => {
-    await page.click('[data-test-subj=uptimeOverviewPage]');
+    await page.click('[data-test-subj=uptimeOverviewPage]', {
+      timeout: 60 * 1000,
+    });
     await refreshUptimeApp();
     await page.click('div.euiBasicTable', { timeout: 60 * 1000 });
   });

__tests__/reporters/__snapshots__/junit.test.ts.snap

@@ -0,0 +1,29 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`junit reporter writes the output to a file 1`] = `
+"<testsuites name=\\"\\" tests=\\"1\\" failures=\\"0\\" skipped=\\"1\\" errors=\\"0\\" time=\\"0\\">
+   <testsuite name=\\"j1\\" tests=\\"1\\" failures=\\"0\\" skipped=\\"1\\" errors=\\"0\\">
+      <testcase name=\\"s1\\" classname=\\"j1 s1\\" time=\\"1\\">
+         <skipped message=\\"previous step failed\\">
+         </skipped>
+      </testcase>
+   </testsuite>
+</testsuites>"
+`;
+
+exports[`junit reporter writes the output to fd 1`] = `
+"<testsuites name=\\"\\" tests=\\"2\\" failures=\\"1\\" skipped=\\"1\\" errors=\\"0\\" time=\\"0\\">
+   <testsuite name=\\"j1\\" tests=\\"2\\" failures=\\"1\\" skipped=\\"1\\" errors=\\"0\\">
+      <testcase name=\\"s1\\" classname=\\"j1 s1\\" time=\\"1\\">
+         <failure message=\\"Boom\\" type=\\"Error\\">
+         at /__tests/reporters/junit.test.ts
+         </failure>
+      </testcase>
+      <testcase name=\\"s2\\" classname=\\"j1 s2\\" time=\\"1\\">
+         <skipped message=\\"previous step failed\\">
+         </skipped>
+      </testcase>
+   </testsuite>
+</testsuites>
+"
+`;

__tests__/reporters/junit.test.ts

@@ -0,0 +1,124 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import fs from 'fs';
+import { join } from 'path';
+import { step, journey } from '../../src/core';
+import Runner from '../../src/core/runner';
+import JUnitReporter from '../../src/reporters/junit';
+import * as helpers from '../../src/helpers';
+
+describe('junit reporter', () => {
+  beforeEach(() => {});
+  let dest: string;
+  let runner: Runner;
+  let stream;
+  const timestamp = 1600300800000000;
+  const j1 = journey('j1', async () => {});
+  const s1 = step('s1', async () => {});
+
+  beforeAll(() => {
+    runner = new Runner();
+  });
+
+  beforeEach(() => {
+    runner = new Runner();
+    dest = helpers.generateTempPath();
+    stream = new JUnitReporter(runner, { fd: fs.openSync(dest, 'w') }).stream;
+    jest.spyOn(helpers, 'now').mockImplementation(() => 0);
+  });
+
+  afterAll(() => fs.unlinkSync(dest));
+
+  it('writes the output to fd', async () => {
+    runner.emit('journey:start', {
+      journey: j1,
+      params: {},
+      timestamp,
+    });
+    const error = new Error('Boom');
+    /**
+     * Snapshots would be different for everyone
+     * so keep it simple
+     */
+    error.stack = 'at /__tests/reporters/junit.test.ts';
+    runner.emit('step:end', {
+      journey: j1,
+      status: 'failed',
+      error,
+      step: s1,
+      start: 0,
+      end: 1,
+    });
+    runner.emit('step:end', {
+      journey: j1,
+      status: 'skipped',
+      step: step('s2', async () => {}),
+      start: 0,
+      end: 1,
+    });
+    runner.emit('journey:end', {
+      journey: j1,
+      start: 0,
+      status: 'failed',
+    });
+    runner.emit('end', 'done');
+    /**
+     * Close the underyling stream writing to FD to read all its contents
+     */
+    stream.end();
+    await new Promise(resolve => stream.once('finish', resolve));
+    const fd = fs.openSync(dest, 'r');
+    const buffer = fs.readFileSync(fd);
+    expect(buffer.toString()).toMatchSnapshot();
+  });
+
+  it('writes the output to a file', async () => {
+    const filepath = join(__dirname, '../../tmp', 'junit.xml');
+    process.env.SYNTHETICS_JUNIT_FILE = filepath;
+    runner.emit('journey:start', {
+      journey: j1,
+      params: {},
+      timestamp,
+    });
+    runner.emit('step:end', {
+      journey: j1,
+      status: 'skipped',
+      step: s1,
+      start: 0,
+      end: 1,
+    });
+    runner.emit('journey:end', {
+      journey: j1,
+      start: 0,
+      status: 'failed',
+    });
+    runner.emit('end', 'done');
+    stream.end();
+    expect(fs.readFileSync(filepath, 'utf-8')).toMatchSnapshot();
+    fs.unlinkSync(filepath);
+    process.env.SYNTHETICS_JUNIT_FILE = '';
+  });
+});

src/cli.ts

@@ -160,7 +160,7 @@ async function prepareSuites(inputs: string[]) {
   /**
    * use JSON reporter if json flag is enabled
    */
-  const reporter = options.json ? 'json' : 'default';
+  const reporter = options.json ? 'json' : options.reporter;
 
   const results = await run({
     params: JSON.parse(options.suiteParams),

src/common_types.ts

@@ -25,10 +25,11 @@
 
 import { Protocol } from 'playwright-chromium/types/protocol';
 import { Step } from './dsl';
+import { reporters } from './reporters';
 
 export type VoidCallback = () => void;
-
 export type StatusValue = 'succeeded' | 'failed' | 'skipped';
+export type Reporters = keyof typeof reporters;
 
 export type FilmStrip = {
   snapshot: string;
@@ -83,7 +84,7 @@ export type CliArgs = {
   journeyName?: string;
   network?: boolean;
   pauseOnError?: boolean;
-  reporter?: string;
+  reporter?: Reporters;
   wsEndpoint?: string;
   sandbox?: boolean;
   json?: boolean;

src/core/runner.ts

@@ -33,31 +33,22 @@ import {
   FilmStrip,
   NetworkInfo,
   VoidCallback,
+  CliArgs,
 } from '../common_types';
 import { BrowserMessage, PluginManager } from '../plugins';
 import { PerformanceManager, Metrics } from '../plugins';
 import { Driver, Gatherer } from './gatherer';
 import { log } from './logger';
 
-type RunParamaters = Record<string, any>;
-
-export type RunOptions = {
+export type RunOptions = Omit<
+  CliArgs,
+  'debug' | 'json' | 'pattern' | 'inline' | 'require' | 'suiteParams'
+> & {
   params?: RunParamaters;
-  environment?: string;
-  reporter?: 'default' | 'json';
-  headless?: boolean;
-  screenshots?: boolean;
-  filmstrips?: boolean;
-  dryRun?: boolean;
-  journeyName?: string;
-  pauseOnError?: boolean;
-  network?: boolean;
-  outfd?: number;
-  metrics?: boolean;
-  wsEndpoint?: string;
-  sandbox?: boolean;
 };
 
+type RunParamaters = Record<string, unknown>;
+
 type BaseContext = {
   params?: RunParamaters;
   start: number;
@@ -306,11 +297,11 @@ export default class Runner {
     }
     this.active = true;
     log(`Runner: run ${this.journeys.length} journeys`);
-    const { reporter = 'default', journeyName, outfd } = options;
+    const { reporter, journeyName, outfd } = options;
     /**
-     * Set up the corresponding reporter
+     * Set up the corresponding reporter and fallback
      */
-    const Reporter = reporters[reporter];
+    const Reporter = reporters[reporter] || reporters['default'];
     new Reporter(this, { fd: outfd });
     this.emit('start', { numJourneys: this.journeys.length });
     await this.runBeforeAllHook();

src/parse_args.ts

@@ -24,6 +24,11 @@
  */
 
 import { program } from 'commander';
+import { reporters } from './reporters';
+
+const availableReporters = Object.keys(reporters)
+  .map(r => String(r))
+  .join();
 
 /* eslint-disable-next-line @typescript-eslint/no-var-requires */
 const { name, version } = require('../package.json');
@@ -33,6 +38,10 @@ program
   .option('-s, --suite-params <jsonstring>', 'Variables', '{}')
   .option('-e, --environment <envname>', 'e.g. production', 'development')
   .option('-j, --json', 'output newline delimited JSON')
+  .option(
+    '--reporter <reporter>',
+    `output repoter format, can be one of ${availableReporters}`
+  )
   .option('-d, --debug', 'print debug logs info')
   .option(
     '--pattern <pattern>',

src/reporters/base.ts

@@ -65,8 +65,8 @@ function renderDuration(durationMs) {
 export default class BaseReporter {
   stream: SonicBoom;
   fd: number;
-  constructor(public runner: Runner, public options: ReporterOptions = {}) {
-    this.runner = runner;
+
+  constructor(public runner: Runner, options: ReporterOptions = {}) {
     this.fd = options.fd || process.stdout.fd;
     this.stream = new SonicBoom({ fd: this.fd, sync: true });
     this._registerListeners();

src/reporters/index.ts

@@ -25,8 +25,10 @@
 
 import BaseReporter from './base';
 import JSONReporter from './json';
+import JUnitReporter from './junit';
 
 export const reporters = {
   default: BaseReporter,
   json: JSONReporter,
+  junit: JUnitReporter,
 };