diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 9b5337e9..761e846e 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -121,6 +121,9 @@ pipeline { } } stage('E2e Test') { + environment { + E2E_FOLDER = "__tests__/e2e" + } options { timeout(15) } @@ -132,15 +135,19 @@ pipeline { sh(label: 'set permissions', script: ''' chmod -R ugo+rw examples ''') - dir("__tests__/e2e"){ - timeout(time: 10, unit: 'MINUTES') { - sh(label: 'run e2e tests',script: 'npm run test') - } + dir("${E2E_FOLDER}"){ + sh(label: 'run e2e tests',script: 'npm run test') } } } } } + post { + always { + archiveArtifacts(allowEmptyArchive: true, artifacts: "${BASE_DIR}/${E2E_FOLDER}/junit.xml") + junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/${E2E_FOLDER}/junit.xml") + } + } } /** Publish Docker images. diff --git a/.gitignore b/.gitignore index 0e6b6f27..a650b75a 100644 --- a/.gitignore +++ b/.gitignore @@ -109,6 +109,7 @@ tmp *.swp __tests__/e2e/tmp/ +__tests__/e2e/junit.xml .idea/ -seccomp/build \ No newline at end of file +seccomp/build diff --git a/__tests__/e2e/docker-compose.yml b/__tests__/e2e/docker-compose.yml index f7c0d915..4f738e69 100644 --- a/__tests__/e2e/docker-compose.yml +++ b/__tests__/e2e/docker-compose.yml @@ -30,7 +30,7 @@ services: - elasticsearch - synthetic healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:5601/api/status"] + test: ["CMD", "curl", "-s", "-f", "http://localhost:5601/api/status"] interval: 30s retries: 10 start_period: 30s diff --git a/__tests__/e2e/scripts/setup.sh b/__tests__/e2e/scripts/setup.sh index 38fc2077..b5ca1743 100755 --- a/__tests__/e2e/scripts/setup.sh +++ b/__tests__/e2e/scripts/setup.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -xe +set -e # variables diff --git a/__tests__/e2e/scripts/test.sh b/__tests__/e2e/scripts/test.sh index c8d005db..5018eea1 100755 --- a/__tests__/e2e/scripts/test.sh +++ b/__tests__/e2e/scripts/test.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -xe +set -eo pipefail if [ -z "${JENKINS_URL}" ]; then # formatting @@ -14,7 +14,7 @@ fi ################################################## echo "" # newline echo "${bold}Waiting for synthetics docker to start...${normal}" -until [ "$(docker inspect -f {{.State.Running}} synthetics)" == "true" ]; do +until [ "$(docker inspect -f '{{.State.Running}}' synthetics)" == "true" ]; do sleep ${SLEEP_TIME}; done; @@ -23,5 +23,4 @@ echo "✅ Setup completed successfully. Running e2e tests..." # # run e2e tests journey ################################################## - -npx @elastic/synthetics uptime.journey.ts +SYNTHETICS_JUNIT_FILE='junit.xml' npx @elastic/synthetics uptime.journey.ts --reporter junit diff --git a/__tests__/e2e/uptime.journey.ts b/__tests__/e2e/uptime.journey.ts index 9ec079e0..27acda44 100644 --- a/__tests__/e2e/uptime.journey.ts +++ b/__tests__/e2e/uptime.journey.ts @@ -43,10 +43,13 @@ journey('E2e test synthetics', async ({ page }) => { step('Go to kibana uptime app', async () => { await page.goto('http://localhost:5620/app/uptime'); + await page.waitForTimeout(30 * 1000); }); step('Check if there is table data', async () => { - await page.click('[data-test-subj=uptimeOverviewPage]'); + await page.click('[data-test-subj=uptimeOverviewPage]', { + timeout: 60 * 1000, + }); await refreshUptimeApp(); await page.click('div.euiBasicTable', { timeout: 60 * 1000 }); }); diff --git a/__tests__/reporters/__snapshots__/junit.test.ts.snap b/__tests__/reporters/__snapshots__/junit.test.ts.snap new file mode 100644 index 00000000..2d278c55 --- /dev/null +++ b/__tests__/reporters/__snapshots__/junit.test.ts.snap @@ -0,0 +1,29 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`junit reporter writes the output to a file 1`] = ` +" + + + + + + +" +`; + +exports[`junit reporter writes the output to fd 1`] = ` +" + + + + at /__tests/reporters/junit.test.ts + + + + + + + + +" +`; diff --git a/__tests__/reporters/junit.test.ts b/__tests__/reporters/junit.test.ts new file mode 100644 index 00000000..9cb71f24 --- /dev/null +++ b/__tests__/reporters/junit.test.ts @@ -0,0 +1,124 @@ +/** + * MIT License + * + * Copyright (c) 2020-present, Elastic NV + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +import fs from 'fs'; +import { join } from 'path'; +import { step, journey } from '../../src/core'; +import Runner from '../../src/core/runner'; +import JUnitReporter from '../../src/reporters/junit'; +import * as helpers from '../../src/helpers'; + +describe('junit reporter', () => { + beforeEach(() => {}); + let dest: string; + let runner: Runner; + let stream; + const timestamp = 1600300800000000; + const j1 = journey('j1', async () => {}); + const s1 = step('s1', async () => {}); + + beforeAll(() => { + runner = new Runner(); + }); + + beforeEach(() => { + runner = new Runner(); + dest = helpers.generateTempPath(); + stream = new JUnitReporter(runner, { fd: fs.openSync(dest, 'w') }).stream; + jest.spyOn(helpers, 'now').mockImplementation(() => 0); + }); + + afterAll(() => fs.unlinkSync(dest)); + + it('writes the output to fd', async () => { + runner.emit('journey:start', { + journey: j1, + params: {}, + timestamp, + }); + const error = new Error('Boom'); + /** + * Snapshots would be different for everyone + * so keep it simple + */ + error.stack = 'at /__tests/reporters/junit.test.ts'; + runner.emit('step:end', { + journey: j1, + status: 'failed', + error, + step: s1, + start: 0, + end: 1, + }); + runner.emit('step:end', { + journey: j1, + status: 'skipped', + step: step('s2', async () => {}), + start: 0, + end: 1, + }); + runner.emit('journey:end', { + journey: j1, + start: 0, + status: 'failed', + }); + runner.emit('end', 'done'); + /** + * Close the underyling stream writing to FD to read all its contents + */ + stream.end(); + await new Promise(resolve => stream.once('finish', resolve)); + const fd = fs.openSync(dest, 'r'); + const buffer = fs.readFileSync(fd); + expect(buffer.toString()).toMatchSnapshot(); + }); + + it('writes the output to a file', async () => { + const filepath = join(__dirname, '../../tmp', 'junit.xml'); + process.env.SYNTHETICS_JUNIT_FILE = filepath; + runner.emit('journey:start', { + journey: j1, + params: {}, + timestamp, + }); + runner.emit('step:end', { + journey: j1, + status: 'skipped', + step: s1, + start: 0, + end: 1, + }); + runner.emit('journey:end', { + journey: j1, + start: 0, + status: 'failed', + }); + runner.emit('end', 'done'); + stream.end(); + expect(fs.readFileSync(filepath, 'utf-8')).toMatchSnapshot(); + fs.unlinkSync(filepath); + process.env.SYNTHETICS_JUNIT_FILE = ''; + }); +}); diff --git a/src/cli.ts b/src/cli.ts index bbb15916..39d79aca 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -160,7 +160,7 @@ async function prepareSuites(inputs: string[]) { /** * use JSON reporter if json flag is enabled */ - const reporter = options.json ? 'json' : 'default'; + const reporter = options.json ? 'json' : options.reporter; const results = await run({ params: JSON.parse(options.suiteParams), diff --git a/src/common_types.ts b/src/common_types.ts index 6f960c78..c9943d32 100644 --- a/src/common_types.ts +++ b/src/common_types.ts @@ -25,10 +25,11 @@ import { Protocol } from 'playwright-chromium/types/protocol'; import { Step } from './dsl'; +import { reporters } from './reporters'; export type VoidCallback = () => void; - export type StatusValue = 'succeeded' | 'failed' | 'skipped'; +export type Reporters = keyof typeof reporters; export type FilmStrip = { snapshot: string; @@ -83,7 +84,7 @@ export type CliArgs = { journeyName?: string; network?: boolean; pauseOnError?: boolean; - reporter?: string; + reporter?: Reporters; wsEndpoint?: string; sandbox?: boolean; json?: boolean; diff --git a/src/core/runner.ts b/src/core/runner.ts index ac7416c8..2ff76a7f 100644 --- a/src/core/runner.ts +++ b/src/core/runner.ts @@ -33,31 +33,22 @@ import { FilmStrip, NetworkInfo, VoidCallback, + CliArgs, } from '../common_types'; import { BrowserMessage, PluginManager } from '../plugins'; import { PerformanceManager, Metrics } from '../plugins'; import { Driver, Gatherer } from './gatherer'; import { log } from './logger'; -type RunParamaters = Record; - -export type RunOptions = { +export type RunOptions = Omit< + CliArgs, + 'debug' | 'json' | 'pattern' | 'inline' | 'require' | 'suiteParams' +> & { params?: RunParamaters; - environment?: string; - reporter?: 'default' | 'json'; - headless?: boolean; - screenshots?: boolean; - filmstrips?: boolean; - dryRun?: boolean; - journeyName?: string; - pauseOnError?: boolean; - network?: boolean; - outfd?: number; - metrics?: boolean; - wsEndpoint?: string; - sandbox?: boolean; }; +type RunParamaters = Record; + type BaseContext = { params?: RunParamaters; start: number; @@ -306,11 +297,11 @@ export default class Runner { } this.active = true; log(`Runner: run ${this.journeys.length} journeys`); - const { reporter = 'default', journeyName, outfd } = options; + const { reporter, journeyName, outfd } = options; /** - * Set up the corresponding reporter + * Set up the corresponding reporter and fallback */ - const Reporter = reporters[reporter]; + const Reporter = reporters[reporter] || reporters['default']; new Reporter(this, { fd: outfd }); this.emit('start', { numJourneys: this.journeys.length }); await this.runBeforeAllHook(); diff --git a/src/parse_args.ts b/src/parse_args.ts index 91932fdc..b03cfa0b 100644 --- a/src/parse_args.ts +++ b/src/parse_args.ts @@ -24,6 +24,11 @@ */ import { program } from 'commander'; +import { reporters } from './reporters'; + +const availableReporters = Object.keys(reporters) + .map(r => String(r)) + .join(); /* eslint-disable-next-line @typescript-eslint/no-var-requires */ const { name, version } = require('../package.json'); @@ -33,6 +38,10 @@ program .option('-s, --suite-params ', 'Variables', '{}') .option('-e, --environment ', 'e.g. production', 'development') .option('-j, --json', 'output newline delimited JSON') + .option( + '--reporter ', + `output repoter format, can be one of ${availableReporters}` + ) .option('-d, --debug', 'print debug logs info') .option( '--pattern ', diff --git a/src/reporters/base.ts b/src/reporters/base.ts index 0594c0ec..8a064c58 100644 --- a/src/reporters/base.ts +++ b/src/reporters/base.ts @@ -65,8 +65,8 @@ function renderDuration(durationMs) { export default class BaseReporter { stream: SonicBoom; fd: number; - constructor(public runner: Runner, public options: ReporterOptions = {}) { - this.runner = runner; + + constructor(public runner: Runner, options: ReporterOptions = {}) { this.fd = options.fd || process.stdout.fd; this.stream = new SonicBoom({ fd: this.fd, sync: true }); this._registerListeners(); diff --git a/src/reporters/index.ts b/src/reporters/index.ts index 6e887f67..0e7d9673 100644 --- a/src/reporters/index.ts +++ b/src/reporters/index.ts @@ -25,8 +25,10 @@ import BaseReporter from './base'; import JSONReporter from './json'; +import JUnitReporter from './junit'; export const reporters = { default: BaseReporter, json: JSONReporter, + junit: JUnitReporter, }; diff --git a/src/reporters/junit.ts b/src/reporters/junit.ts new file mode 100644 index 00000000..bc3041f7 --- /dev/null +++ b/src/reporters/junit.ts @@ -0,0 +1,178 @@ +/** + * MIT License + * + * Copyright (c) 2020-present, Elastic NV + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +import { mkdirSync, writeFileSync } from 'fs'; +import { dirname } from 'path'; +import { formatError, indent, now } from '../helpers'; +import BaseReporter from './base'; + +type XMLEntry = { + name: string; + attributes?: { + name: string; + timestamp?: number; + classname?: string; + time?: number; + tests?: number; + failures?: number; + skipped?: number; + errors?: number; + }; + children?: XMLEntry[]; + text?: string; +}; + +/** + * JUnit Reporting Format - https://llg.cubic.org/docs/junit/ + */ +export default class JUnitReporter extends BaseReporter { + private totalTests = 0; + private totalFailures = 0; + private totalSkipped = 0; + + _registerListeners() { + const journeyMap = new Map(); + + this.runner.on('journey:start', ({ journey }) => { + if (!journeyMap.has(journey.name)) { + const entry = { + name: 'testsuite', + attributes: { + name: journey.name, + tests: 0, + failures: 0, + skipped: 0, + errors: 0, + }, + children: [], + }; + journeyMap.set(journey.name, entry); + } + }); + + this.runner.on( + 'step:end', + ({ journey, step, status, error, start, end }) => { + if (!journeyMap.has(journey.name)) { + return; + } + const entry = journeyMap.get(journey.name); + const caseEntry = { + name: 'testcase', + attributes: { + name: step.name, + classname: journey.name + ' ' + step.name, + time: end - start, + }, + children: [], + }; + + entry.attributes.tests++; + if (status === 'failed') { + const { name, message, stack } = formatError(error); + caseEntry.children.push({ + name: 'failure', + attributes: { + message, + type: name, + }, + text: stack, + }); + entry.attributes.failures++; + } else if (status === 'skipped') { + caseEntry.children.push({ + name: 'skipped', + attributes: { + message: 'previous step failed', + }, + }); + entry.attributes.skipped++; + } + entry.children.push(caseEntry); + } + ); + + this.runner.on('journey:end', ({ journey }) => { + if (!journeyMap.has(journey.name)) { + return; + } + const { attributes } = journeyMap.get(journey.name); + this.totalTests += attributes.tests; + this.totalFailures += attributes.failures; + this.totalSkipped += attributes.skipped; + }); + + this.runner.on('end', () => { + const root: XMLEntry = { + name: 'testsuites', + attributes: { + name: '', + tests: this.totalTests, + failures: this.totalFailures, + skipped: this.totalSkipped, + errors: 0, + time: parseInt(String(now())) / 1000, + }, + children: [...journeyMap.values()], + }; + const output = serializeEntries(root).join('\n'); + /** + * write the xml output to a file if specified via env flag + */ + const fileName = process.env['SYNTHETICS_JUNIT_FILE']; + if (fileName) { + mkdirSync(dirname(fileName), { recursive: true }); + writeFileSync(fileName, output); + } else { + this.write(output); + } + }); + } +} + +function escape(text: string): string { + return text + .replace(/"/g, '"') + .replace(/&/g, '&') + .replace(//g, '>'); +} + +function serializeEntries(entry: XMLEntry, tokens: string[] = [], space = '') { + tokens.push( + indent( + `<${entry.name} ${Object.entries(entry.attributes || {}) + .map(([key, value]) => `${key}="${escape(String(value))}"`) + .join(' ')}>`, + space + ) + ); + for (const child of entry.children || []) { + serializeEntries(child, tokens, space + ' '); + } + if (entry.text) tokens.push(indent(escape(entry.text), space)); + tokens.push(indent(``, space)); + return tokens; +}