diff --git a/spec/helpers/jasmine2-custom-matchers.js b/spec/helpers/jasmine2-custom-matchers.js index 6378f68ac9..68f1b93453 100644 --- a/spec/helpers/jasmine2-custom-matchers.js +++ b/spec/helpers/jasmine2-custom-matchers.js @@ -2,160 +2,166 @@ const _ = require("underscore-plus"); const fs = require("fs-plus"); const path = require("path"); -beforeEach(function () { - jasmine.getEnv().addCustomEqualityTester(function(a, b) { - // Match jasmine.any's equality matching logic - if ((a != null ? a.jasmineMatches : undefined) != null) { return a.jasmineMatches(b); } - if ((b != null ? b.jasmineMatches : undefined) != null) { return b.jasmineMatches(a); } - - // Use underscore's definition of equality for toEqual assertions - return _.isEqual(a, b); - }); +exports.register = (jasmineEnv) => { + jasmineEnv.beforeEach(function () { + jasmineEnv.addCustomEqualityTester(function (a, b) { + // Match jasmine.any's equality matching logic + if ((a != null ? a.jasmineMatches : undefined) != null) { + return a.jasmineMatches(b); + } + if ((b != null ? b.jasmineMatches : undefined) != null) { + return b.jasmineMatches(a); + } + + // Use underscore's definition of equality for toEqual assertions + return _.isEqual(a, b); + }); + + jasmineEnv.addMatchers({ + toHaveLength: function (util, customEqualityTesters) { + return { + compare: function (actual, expected) { + if (actual == null) { + return { + pass: false, + message: `Expected object ${actual} has no length method`, + }; + } else { + return { + pass: actual.length === expected, + message: `Expected object with length ${actual.length} to have length ${expected}`, + }; + } + }, + } + }, - jasmine.getEnv().addMatchers({ - toHaveLength: function(util, customEqualityTesters) { - return { - compare: function (actual, expected) { - if (actual == null) { + toExistOnDisk: function (util, customEqualityTesters) { + return { + compare: function (actual) { return { - pass: false, - message: `Expected object ${actual} has no length method`, + pass: fs.existsSync(actual), + message: `Expected path '${actual}' to exist.`, }; - } else { + }, + } + }, + + toHaveFocus: function (util, customEqualityTesters) { + return { + compare: function (actual) { + if (!document.hasFocus()) { + console.error("Specs will fail because the Dev Tools have focus. To fix this close the Dev Tools or click the spec runner."); + } + + let element = actual; + if (element.jquery) { + element = element.get(0); + } + return { - pass: actual.length === expected, - message: `Expected object with length ${actual.length} to have length ${expected}`, + pass: (element === document.activeElement) || element.contains(document.activeElement), + message: `Expected element '${actual}' or its descendants to have focus.`, }; - } - }, - } - }, - - toExistOnDisk: function(util, customEqualityTesters) { - return { - compare: function (actual) { - return { - pass: fs.existsSync(actual), - message: `Expected path '${actual}' to exist.`, - }; - }, - } - }, + }, + } + }, - toHaveFocus: function(util, customEqualityTesters) { - return { - compare: function (actual) { - if (!document.hasFocus()) { - console.error("Specs will fail because the Dev Tools have focus. To fix this close the Dev Tools or click the spec runner."); - } + toShow: function (util, customEqualityTesters) { + return { + compare: function (actual) { + let element = actual; + if (element.jquery) { + element = element.get(0); + } + const computedStyle = getComputedStyle(element); - let element = actual; - if (element.jquery) { - element = element.get(0); - } + return { + pass: (computedStyle.display !== 'none') && (computedStyle.visibility === 'visible') && !element.hidden, + message: `Expected element '${element}' or its descendants to show.`, + }; + }, + } + }, - return { - pass: (element === document.activeElement) || element.contains(document.activeElement), - message: `Expected element '${actual}' or its descendants to have focus.`, - }; - }, - } - }, - - toShow: function(util, customEqualityTesters) { - return { - compare: function (actual) { - let element = actual; - if (element.jquery) { - element = element.get(0); - } - const computedStyle = getComputedStyle(element); + toEqualPath: function (util, customEqualityTesters) { + return { + compare: function (actual, expected) { + const actualPath = path.normalize(actual); + const expectedPath = path.normalize(expected); - return { - pass: (computedStyle.display !== 'none') && (computedStyle.visibility === 'visible') && !element.hidden, - message: `Expected element '${element}' or its descendants to show.`, - }; - }, - } - }, - - toEqualPath: function(util, customEqualityTesters) { - return { - compare: function (actual, expected) { - const actualPath = path.normalize(actual); - const expectedPath = path.normalize(expected); - - return { - pass: actualPath === expectedPath, - message: `Expected path '${actualPath}' to be equal to '${expectedPath}'.`, - }; - }, - } - }, - - toBeNear: function(util, customEqualityTesters) { - return { - compare: function (actual, expected) { - let acceptedError = 1; - - return { - pass: ((expected - acceptedError) <= actual) && (actual <= (expected + acceptedError)), - message: `Expected '${actual}' to be near to '${expected}'.`, - }; - }, - } - }, + return { + pass: actualPath === expectedPath, + message: `Expected path '${actualPath}' to be equal to '${expectedPath}'.`, + }; + }, + } + }, + + toBeNear: function (util, customEqualityTesters) { + return { + compare: function (actual, expected) { + let acceptedError = 1; - toHaveNearPixels: function(util, customEqualityTesters) { - return { - compare: function (actual, expected) { - let acceptedError = 1; + return { + pass: ((expected - acceptedError) <= actual) && (actual <= (expected + acceptedError)), + message: `Expected '${actual}' to be near to '${expected}'.`, + }; + }, + } + }, - const expectedNumber = parseFloat(expected); - const actualNumber = parseFloat(actual); + toHaveNearPixels: function (util, customEqualityTesters) { + return { + compare: function (actual, expected) { + let acceptedError = 1; - return { - pass: (expected.indexOf('px') >= 1) && (actual.indexOf('px') >= 1) && ((expectedNumber - acceptedError) <= actualNumber) && (actualNumber <= (expectedNumber + acceptedError)), - message: `Expected '${actual}' to have near pixels to '${expected}'.`, + const expectedNumber = parseFloat(expected); + const actualNumber = parseFloat(actual); + + return { + pass: (expected.indexOf('px') >= 1) && (actual.indexOf('px') >= 1) && ((expectedNumber - acceptedError) <= actualNumber) && (actualNumber <= (expectedNumber + acceptedError)), + message: `Expected '${actual}' to have near pixels to '${expected}'.`, + } } } - } - }, - - toHaveClass: function(util, customEqualityTesters) { - return { - compare: function (actual, expected) { - return { - pass: actual instanceof HTMLElement && actual.classList.contains(expected), - message: `Expected '${actual}' to have '${expected}' class` + }, + + toHaveClass: function (util, customEqualityTesters) { + return { + compare: function (actual, expected) { + return { + pass: actual instanceof HTMLElement && actual.classList.contains(expected), + message: `Expected '${actual}' to have '${expected}' class` + } } } - } - }, - - toHaveText: function(util, customEqualityTesters) { - return { - compare: function (actual, expected) { - return { - pass: actual instanceof HTMLElement && actual.textContent == expected, - message: `Expected '${actual}' to have text: '${expected}'` + }, + + toHaveText: function (util, customEqualityTesters) { + return { + compare: function (actual, expected) { + return { + pass: actual instanceof HTMLElement && actual.textContent == expected, + message: `Expected '${actual}' to have text: '${expected}'` + } } } - } - }, - - toExist: function(util, customEqualityTesters) { - return { - compare: function (actual) { - if (actual instanceof HTMLElement) { - return { pass: true } - } else if (actual) { - return { pass: actual.size() > 0 } - } else { - return { pass: false } + }, + + toExist: function (util, customEqualityTesters) { + return { + compare: function (actual) { + if (actual instanceof HTMLElement) { + return {pass: true} + } else if (actual) { + return {pass: actual.size() > 0} + } else { + return {pass: false} + } } } } - } + }); }); -}); +} diff --git a/spec/helpers/jasmine2-spies.js b/spec/helpers/jasmine2-spies.js index 00ba82357f..f2b5abf17a 100644 --- a/spec/helpers/jasmine2-spies.js +++ b/spec/helpers/jasmine2-spies.js @@ -26,67 +26,73 @@ if (specDirectory) { specProjectPath = require('os').tmpdir(); } -beforeEach(function() { - // Do not clobber recent project history - spyOn(Object.getPrototypeOf(atom.history), 'saveState').and.returnValue(Promise.resolve()); +exports.register = (jasmineEnv) => { + jasmineEnv.beforeEach(function () { + // Do not clobber recent project history + spyOn(Object.getPrototypeOf(atom.history), 'saveState').and.returnValue(Promise.resolve()); - atom.project.setPaths([specProjectPath]); + atom.project.setPaths([specProjectPath]); - atom.packages._originalResolvePackagePath = atom.packages.resolvePackagePath; - const spy = spyOn(atom.packages, 'resolvePackagePath') - spy.and.callFake(function(packageName) { - if (specPackageName && (packageName === specPackageName)) { - return atom.packages._originalResolvePackagePath(specPackagePath); - } else { - return atom.packages._originalResolvePackagePath(packageName); - } - }); + atom.packages._originalResolvePackagePath = atom.packages.resolvePackagePath; + const spy = spyOn(atom.packages, 'resolvePackagePath') + spy.and.callFake(function (packageName) { + if (specPackageName && (packageName === specPackageName)) { + return atom.packages._originalResolvePackagePath(specPackagePath); + } else { + return atom.packages._originalResolvePackagePath(packageName); + } + }); - // prevent specs from modifying Atom's menus - spyOn(atom.menu, 'sendToBrowserProcess'); + // prevent specs from modifying Atom's menus + spyOn(atom.menu, 'sendToBrowserProcess'); - // reset config before each spec - atom.config.set("core.destroyEmptyPanes", false); - atom.config.set("editor.fontFamily", "Courier"); - atom.config.set("editor.fontSize", 16); - atom.config.set("editor.autoIndent", false); - atom.config.set("core.disabledPackages", ["package-that-throws-an-exception", - "package-with-broken-package-json", "package-with-broken-keymap"]); + // reset config before each spec + atom.config.set("core.destroyEmptyPanes", false); + atom.config.set("editor.fontFamily", "Courier"); + atom.config.set("editor.fontSize", 16); + atom.config.set("editor.autoIndent", false); + atom.config.set("core.disabledPackages", ["package-that-throws-an-exception", + "package-with-broken-package-json", "package-with-broken-keymap"]); - // advanceClock(1000); - // window.setTimeout.calls.reset(); + // advanceClock(1000); + // window.setTimeout.calls.reset(); - // make editor display updates synchronous - TextEditorElement.prototype.setUpdatedSynchronously(true); + // make editor display updates synchronous + TextEditorElement.prototype.setUpdatedSynchronously(true); - spyOn(pathwatcher.File.prototype, "detectResurrectionAfterDelay").and.callFake(function() { return this.detectResurrection(); }); - spyOn(TextEditor.prototype, "shouldPromptToSave").and.returnValue(false); + spyOn(pathwatcher.File.prototype, "detectResurrectionAfterDelay").and.callFake(function () { + return this.detectResurrection(); + }); + spyOn(TextEditor.prototype, "shouldPromptToSave").and.returnValue(false); - // make tokenization synchronous - TextMateLanguageMode.prototype.chunkSize = Infinity; - TreeSitterLanguageMode.prototype.syncTimeoutMicros = Infinity; - spyOn(TextMateLanguageMode.prototype, "tokenizeInBackground").and.callFake(function() { return this.tokenizeNextChunk(); }); + // make tokenization synchronous + TextMateLanguageMode.prototype.chunkSize = Infinity; + TreeSitterLanguageMode.prototype.syncTimeoutMicros = Infinity; + spyOn(TextMateLanguageMode.prototype, "tokenizeInBackground").and.callFake(function () { + return this.tokenizeNextChunk(); + }); - // Without this spy, TextEditor.onDidTokenize callbacks would not be called - // after the buffer's language mode changed, because by the time the editor - // called its new language mode's onDidTokenize method, the language mode - // would already be fully tokenized. - spyOn(TextEditor.prototype, "onDidTokenize").and.callFake(function(callback) { - return new CompositeDisposable( - this.emitter.on("did-tokenize", callback), - this.onDidChangeGrammar(() => { - const languageMode = this.buffer.getLanguageMode(); - if (languageMode.tokenizeInBackground != null ? languageMode.tokenizeInBackground.originalValue : undefined) { - return callback(); - } - }) - ); - }); + // Without this spy, TextEditor.onDidTokenize callbacks would not be called + // after the buffer's language mode changed, because by the time the editor + // called its new language mode's onDidTokenize method, the language mode + // would already be fully tokenized. + spyOn(TextEditor.prototype, "onDidTokenize").and.callFake(function (callback) { + return new CompositeDisposable( + this.emitter.on("did-tokenize", callback), + this.onDidChangeGrammar(() => { + const languageMode = this.buffer.getLanguageMode(); + if (languageMode.tokenizeInBackground != null ? languageMode.tokenizeInBackground.originalValue : undefined) { + return callback(); + } + }) + ); + }); - let clipboardContent = 'initial clipboard content'; - spyOn(clipboard, 'writeText').and.callFake(text => clipboardContent = text); - spyOn(clipboard, 'readText').and.callFake(() => clipboardContent); -}); + let clipboardContent = 'initial clipboard content'; + spyOn(clipboard, 'writeText').and.callFake(text => clipboardContent = text); + spyOn(clipboard, 'readText').and.callFake(() => clipboardContent); + }); +} jasmine.unspy = function(object, methodName) { object[methodName].and.callThrough(); diff --git a/spec/helpers/jasmine2-time.js b/spec/helpers/jasmine2-time.js index 77061b045d..e61cf1c933 100644 --- a/spec/helpers/jasmine2-time.js +++ b/spec/helpers/jasmine2-time.js @@ -77,13 +77,15 @@ window.advanceClock = function(delta) { })(); }; -beforeEach(() => { - resetTimeouts(); - spyOn(_._, "now").and.callFake(() => now); - spyOn(Date, 'now').and.callFake(() => now); - spyOn(window, "setTimeout").and.callFake(fakeSetTimeout); - spyOn(window, "clearTimeout").and.callFake(fakeClearTimeout); - spyOn(window, 'setInterval').and.callFake(fakeSetInterval); - spyOn(window, 'clearInterval').and.callFake(fakeClearInterval); - spyOn(_, "debounce").and.callFake(mockDebounce); -}) +exports.register = (jasmineEnv) => { + jasmineEnv.beforeEach(() => { + resetTimeouts(); + spyOn(_._, "now").and.callFake(() => now); + spyOn(Date, 'now').and.callFake(() => now); + spyOn(window, "setTimeout").and.callFake(fakeSetTimeout); + spyOn(window, "clearTimeout").and.callFake(fakeClearTimeout); + spyOn(window, 'setInterval').and.callFake(fakeSetInterval); + spyOn(window, 'clearInterval').and.callFake(fakeClearInterval); + spyOn(_, "debounce").and.callFake(mockDebounce); + }) +} diff --git a/spec/helpers/jasmine2-warnings.js b/spec/helpers/jasmine2-warnings.js index a8d3b6addc..24e100896b 100644 --- a/spec/helpers/jasmine2-warnings.js +++ b/spec/helpers/jasmine2-warnings.js @@ -4,14 +4,18 @@ const { warnIfLeakingPathSubscriptions } = require('./warnings') -afterEach(async (done) => { - ensureNoDeprecatedFunctionCalls(); - ensureNoDeprecatedStylesheets(); +exports.register = (jasmineEnv) => { + jasmineEnv.afterEach(async (done) => { + ensureNoDeprecatedFunctionCalls(); + ensureNoDeprecatedStylesheets(); - await atom.reset(); + await atom.reset(); - if (!window.debugContent) { document.getElementById('jasmine-content').innerHTML = ''; } - warnIfLeakingPathSubscriptions(); + if (!window.debugContent) { + document.getElementById('jasmine-content').innerHTML = ''; + } + warnIfLeakingPathSubscriptions(); - done(); -}); + done(); + }); +} diff --git a/spec/runners/jasmine2-test-runner.js b/spec/runners/jasmine2-test-runner.js index b745e98f21..6e28749fde 100644 --- a/spec/runners/jasmine2-test-runner.js +++ b/spec/runners/jasmine2-test-runner.js @@ -18,11 +18,6 @@ temp.track(); module.exports = function({logFile, headless, testPaths, buildAtomEnvironment}) { // Load Jasmine 2.x require('../helpers/jasmine2-singleton'); - require('../helpers/jasmine2-custom-matchers'); - - // Load helpers - require('../helpers/normalize-comments'); - require('../helpers/document-title'); // Build Atom Environment const { atomHome, applicationDelegate } = require('../helpers/build-atom-environment'); @@ -32,56 +27,115 @@ module.exports = function({logFile, headless, testPaths, buildAtomEnvironment}) enablePersistence: false }); + // Load general helpers require('../../src/window'); + require('../helpers/normalize-comments'); + require('../helpers/document-title'); require('../helpers/load-jasmine-stylesheet'); require('../helpers/fixture-packages'); require('../helpers/set-prototype-extensions'); - require('../helpers/jasmine2-spies'); - require('../helpers/jasmine2-time'); - require('../helpers/jasmine2-warnings'); require('../helpers/default-timeout'); require('../helpers/attach-to-dom'); require('../helpers/deprecation-snapshots'); require('../helpers/platform-filter'); - for (let testPath of Array.from(testPaths)) { requireSpecs(testPath); } + const jasmineContent = document.createElement('div'); + jasmineContent.setAttribute('id', 'jasmine-content'); + document.body.appendChild(jasmineContent); - setSpecType('user'); + return loadSpecsAndRunThem(logFile, headless, testPaths) + .then((result) => { + // All specs passed, don't need to rerun any of them - pass the results to handle possible Grim deprecations + if (result.failedSpecs.length === 0) return result; - let resolveWithExitCode = null; - const promise = new Promise((resolve, reject) => resolveWithExitCode = resolve); - const jasmineEnv = jasmine.getEnv(); - jasmineEnv.addReporter(buildReporter({logFile, headless, resolveWithExitCode})); + console.log('\n', '\n', `Retrying ${result.failedSpecs.length} spec(s)`, '\n', '\n'); - const jasmineContent = document.createElement('div'); - jasmineContent.setAttribute('id', 'jasmine-content'); + // Gather the full names of the failed specs - this is the closest to be a unique identifier for all specs + const fullNamesOfFailedSpecs = result.failedSpecs.map((spec) => { + return spec.fullName; + }) - document.body.appendChild(jasmineContent); + // Force-delete the current env - this way Jasmine will reset and we'll be able to re-run failed specs only. The next time the code calls getEnv(), it'll generate a new environment + jasmine.currentEnv_ = null; + + // As all the jasmine helpers (it, describe, etc..) were registered to the previous environment, we need to re-set them on window + for (let key in jasmine.getEnv()) { + window[key] = jasmine.getEnv()[key]; + } + + // Set up a specFilter to disable all passing spec and re-run only the flaky ones + jasmine.getEnv().specFilter = (spec) => { + return fullNamesOfFailedSpecs.includes(spec.result.fullName); + }; + + // Run the specs again - due to the spec filter, only the failed specs will run this time + return loadSpecsAndRunThem(logFile, headless, testPaths); + }).then((result) => { + // Some of the specs failed, we should return with a non-zero exit code + if (result.failedSpecs.length !== 0) return 1; + + // Some of the tests had deprecation warnings, we should log them and return with a non-zero exit code + if (result.hasDeprecations) { + Grim.logDeprecations(); + return 1; + } - jasmine.getEnv().execute(); - return promise; + // Everything went good, time to return with a zero exit code + return 0; + }) }; -var requireSpecs = function(testPath, specType) { +const loadSpecsAndRunThem = (logFile, headless, testPaths) => { + return new Promise((resolve) => { + const jasmineEnv = jasmine.getEnv(); + + // Load before and after hooks, custom matchers + require('../helpers/jasmine2-custom-matchers').register(jasmineEnv); + require('../helpers/jasmine2-spies').register(jasmineEnv); + require('../helpers/jasmine2-time').register(jasmineEnv); + require('../helpers/jasmine2-warnings').register(jasmineEnv); + + // Load specs and set spec type + for (let testPath of Array.from(testPaths)) { requireSpecs(testPath); } + setSpecType('user'); + + // Add the reporter and register the promise resolve as a callback + jasmineEnv.addReporter(buildReporter({logFile, headless})); + jasmineEnv.addReporter(buildRetryReporter(resolve)); + + // And finally execute the tests + jasmineEnv.execute(); + }) +} + +// This is a helper function to remove a file from the require cache. +// We are using this to force a re-evaluation of the test files when we need to re-run some flaky tests +const unrequire = (requiredPath) => { + for (const path in require.cache) { + if (path === requiredPath) { + delete require.cache[path]; + } + } +} + +const requireSpecs = (testPath) => { if (fs.isDirectorySync(testPath)) { - return (() => { - const result = []; - for (let testFilePath of Array.from(fs.listTreeSync(testPath))) { - if (/-spec\.(coffee|js)$/.test(testFilePath)) { - require(testFilePath); - // Set spec directory on spec for setting up the project in spec-helper - result.push(setSpecDirectory(testPath)); - } + for (let testFilePath of fs.listTreeSync(testPath)) { + if (/-spec\.js$/.test(testFilePath)) { + unrequire(testFilePath); + require(testFilePath); + // Set spec directory on spec for setting up the project in spec-helper + setSpecDirectory(testPath); } - return result; - })(); + } } else { + unrequire(testPath); require(testPath); - return setSpecDirectory(path.dirname(testPath)); + setSpecDirectory(path.dirname(testPath)); } }; -const setSpecField = function(name, value) { +const setSpecField = (name, value) => { const specs = (new jasmine.JsApiReporter({})).specs(); if (specs.length === 0) { return; } @@ -91,20 +145,41 @@ const setSpecField = function(name, value) { } }; -var setSpecType = specType => setSpecField('specType', specType); +const setSpecType = specType => setSpecField('specType', specType); -var setSpecDirectory = specDirectory => setSpecField('specDirectory', specDirectory); +const setSpecDirectory = specDirectory => setSpecField('specDirectory', specDirectory); -var buildReporter = function({logFile, headless, resolveWithExitCode}) { +const buildReporter = ({logFile, headless}) => { if (headless) { - return buildTerminalReporter(logFile, resolveWithExitCode); + return buildConsoleReporter(logFile); } else { - const AtomReporter = require('./atom-reporter.js'); + const AtomReporter = require('../helpers/atom-reporter.js'); return new AtomReporter(); } }; -var buildTerminalReporter = function(logFile, resolveWithExitCode) { +const buildRetryReporter = (onCompleteCallback) => { + const failedSpecs = []; + + return { + jasmineStarted: () => {}, + suiteStarted: () => {}, + specStarted: () => {}, + suiteDone: () => {}, + + specDone: (spec) => { + if (spec.status === 'failed') { + failedSpecs.push(spec); + } + }, + + jasmineDone: () => { + onCompleteCallback({failedSpecs, hasDeprecations: Grim.getDeprecationsLength() > 0}); + } + }; +} + +const buildConsoleReporter = (logFile) => { let logStream; if (logFile != null) { logStream = fs.openSync(logFile, 'w'); } const log = function(str) { @@ -119,19 +194,8 @@ var buildTerminalReporter = function(logFile, resolveWithExitCode) { print(str) { log(str); }, - onComplete(failureCountIsZero) { + onComplete() { if (logStream != null) { fs.closeSync(logStream); } - if (Grim.getDeprecationsLength() > 0) { - Grim.logDeprecations(); - resolveWithExitCode(1); - return; - } - - if (!failureCountIsZero) { - return resolveWithExitCode(1); - } else { - return resolveWithExitCode(0); - } }, printDeprecation: (msg) => { console.log(msg) diff --git a/spec/text-editor-component-spec.js b/spec/text-editor-component-spec.js index cc230e4f06..e6b920593a 100644 --- a/spec/text-editor-component-spec.js +++ b/spec/text-editor-component-spec.js @@ -24,17 +24,19 @@ class DummyElement extends HTMLElement { } } -window.customElements.define( - 'text-editor-component-test-element', - DummyElement -); - -document.createElement('text-editor-component-test-element'); - const editors = []; let verticalScrollbarWidth, horizontalScrollbarHeight; describe('TextEditorComponent', () => { + beforeEach(() => { + if(!window.customElements.get('text-editor-component-test-element')) { + window.customElements.define( + 'text-editor-component-test-element', + DummyElement + ); + } + }) + beforeEach(() => { jasmine.useRealClock();