From b99b8800fe965308692d0d93d0b67bc18709eafd Mon Sep 17 00:00:00 2001 From: aminya Date: Thu, 16 Jul 2020 02:04:08 -0500 Subject: [PATCH 01/14] run package tests in parallel for windows tests --- script/test | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/script/test b/script/test index 974f29d832a6..ee9421e5765b 100755 --- a/script/test +++ b/script/test @@ -205,28 +205,24 @@ function requestedTestSuites () { function testSuitesForPlatform (platform) { let suites = [] - switch (platform) { - case 'darwin': - const PACKAGES_TO_TEST_IN_PARALLEL = 23 - - if (process.env.ATOM_RUN_CORE_TESTS === 'true') { - suites = [runCoreMainProcessTests, runCoreRenderProcessTests] - } else if (process.env.ATOM_RUN_PACKAGE_TESTS === '1') { - suites = packageTestSuites.slice(0, PACKAGES_TO_TEST_IN_PARALLEL) - } else if (process.env.ATOM_RUN_PACKAGE_TESTS === '2') { - suites = packageTestSuites.slice(PACKAGES_TO_TEST_IN_PARALLEL) - } else { - suites = [runCoreMainProcessTests, runCoreRenderProcessTests].concat(packageTestSuites) - } - break - case 'win32': - suites = (process.arch === 'x64') ? [runCoreMainProcessTests, runCoreRenderProcessTests] : [runCoreMainProcessTests] - break - case 'linux': - suites = [runCoreMainProcessTests] - break - default: - console.log(`Unrecognized platform: ${platform}`) + if ((platform === 'darwin') || (platform === 'win32' && process.arch === 'x64')) { + const PACKAGES_TO_TEST_IN_PARALLEL = 23 + + if (process.env.ATOM_RUN_CORE_TESTS === 'true') { + suites = [runCoreMainProcessTests, runCoreRenderProcessTests] + } else if (process.env.ATOM_RUN_PACKAGE_TESTS === '1') { + suites = packageTestSuites.slice(0, PACKAGES_TO_TEST_IN_PARALLEL) + } else if (process.env.ATOM_RUN_PACKAGE_TESTS === '2') { + suites = packageTestSuites.slice(PACKAGES_TO_TEST_IN_PARALLEL) + } else { + suites = [runCoreMainProcessTests, runCoreRenderProcessTests].concat(packageTestSuites) + } + } + else if ((platform === 'linux') || (platform === 'win32' && process.arch === 'x86')) { + suites = [runCoreMainProcessTests] + } + else { + console.log(`Unrecognized platform: ${platform}`) } if (argv.skipMainProcessTests) { From 038d6394a91367582ab7d7673c9c70f15ec95bc7 Mon Sep 17 00:00:00 2001 From: aminya Date: Sat, 11 Jul 2020 23:36:57 -0500 Subject: [PATCH 02/14] parallelize core tests --- script/test | 77 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 30 deletions(-) diff --git a/script/test b/script/test index ee9421e5765b..24431b53c2f5 100755 --- a/script/test +++ b/script/test @@ -97,18 +97,23 @@ function runCoreMainProcessTests (callback) { cp.on('close', exitCode => { callback(null, {exitCode, step: 'core-main-process'}) }) } -function runCoreRenderProcessTests (callback) { - const testPath = path.join(CONFIG.repositoryRootPath, 'spec') - const testArguments = [ - '--resource-path', resourcePath, - '--test', testPath - ] - const testEnv = prepareEnv('core-render-process') - - console.log('Executing core render process tests'.bold.green) - const cp = childProcess.spawn(executablePath, testArguments, {stdio: 'inherit', env: testEnv}) - cp.on('error', error => { callback(error) }) - cp.on('close', exitCode => { callback(null, {exitCode, step: 'core-render-process'}) }) +// Build an array of functions, each running tests for a different rendering test +const coreRenderProcessTestSuites = [] +const testPath = path.join(CONFIG.repositoryRootPath, 'spec') +let testFiles = glob.sync(path.join(testPath, '*-spec.+(js|coffee|ts|jsx|tsx|mjs)')) +for (let testFile of testFiles) { + coreRenderProcessTestSuites.push( function (callback) { + + const testEnv = prepareEnv('core-render-process') + console.log(`Executing core render process tests for ${testFile}`.bold.green) + const testArguments = [ + '--resource-path', resourcePath, + '--test', testFile + ] + const cp = childProcess.spawn(executablePath, testArguments, {stdio: 'inherit', env: testEnv}) + cp.on('error', error => { callback(error) }) + cp.on('close', exitCode => { callback(null, {exitCode, step: `core-render-process for ${testFile}`}) }) + }) } // Build an array of functions, each running tests for a different bundled package @@ -192,7 +197,7 @@ function requestedTestSuites () { suites.push(runCoreMainProcessTests) } if (argv.coreRenderer) { - suites.push(runCoreRenderProcessTests) + suites.push(...coreRenderProcessTestSuites) } if (argv.coreBenchmark) { suites.push(runBenchmarkTests) @@ -205,24 +210,36 @@ function requestedTestSuites () { function testSuitesForPlatform (platform) { let suites = [] - if ((platform === 'darwin') || (platform === 'win32' && process.arch === 'x64')) { - const PACKAGES_TO_TEST_IN_PARALLEL = 23 - - if (process.env.ATOM_RUN_CORE_TESTS === 'true') { - suites = [runCoreMainProcessTests, runCoreRenderProcessTests] - } else if (process.env.ATOM_RUN_PACKAGE_TESTS === '1') { - suites = packageTestSuites.slice(0, PACKAGES_TO_TEST_IN_PARALLEL) - } else if (process.env.ATOM_RUN_PACKAGE_TESTS === '2') { - suites = packageTestSuites.slice(PACKAGES_TO_TEST_IN_PARALLEL) - } else { - suites = [runCoreMainProcessTests, runCoreRenderProcessTests].concat(packageTestSuites) - } - } - else if ((platform === 'linux') || (platform === 'win32' && process.arch === 'x86')) { + // Env variable options + let coreMain = process.env.ATOM_RUN_CORE_MAIN_TESTS === 'true' + let coreRenderer1 = process.env.ATOM_RUN_CORE_RENDER_TESTS === '1' + let coreRenderer2 = process.env.ATOM_RUN_CORE_RENDER_TESTS === '2' + let coreAll = process.env.ATOM_RUN_CORE_TESTS === 'true' + let packages1 = process.env.ATOM_RUN_PACKAGE_TESTS === '1' + let packages2 = process.env.ATOM_RUN_PACKAGE_TESTS === '2' + + // Operating system overrides: + coreMain = coreMain || (platform === 'linux') || (platform === 'win32' && process.arch === 'x86') + + // split package tests (used for macos in CI) + const PACKAGES_TO_TEST_IN_PARALLEL = 23 + // split core render test (used for windows x64 in CI) + const CORE_RENDER_TO_TEST_IN_PARALLEL = 35 + + if (coreMain) { suites = [runCoreMainProcessTests] - } - else { - console.log(`Unrecognized platform: ${platform}`) + } else if (coreRenderer1) { + suites = coreRenderProcessTestSuites.slice(0, CORE_RENDER_TO_TEST_IN_PARALLEL) + } else if (coreRenderer2) { + suites = coreRenderProcessTestSuites.slice(CORE_RENDER_TO_TEST_IN_PARALLEL) + } else if (coreAll) { + suites = [runCoreMainProcessTests, ...coreRenderProcessTestSuites] + } else if (packages1) { + suites = packageTestSuites.slice(0, PACKAGES_TO_TEST_IN_PARALLEL) + } else if (packages2) { + suites = packageTestSuites.slice(PACKAGES_TO_TEST_IN_PARALLEL) + } else { + suites = [runCoreMainProcessTests, ...coreRenderProcessTestSuites, ...packageTestSuites] // all } if (argv.skipMainProcessTests) { From bbfd0ec29adf43b802eeab16986469dec0851e91 Mon Sep 17 00:00:00 2001 From: aminya Date: Sun, 12 Jul 2020 03:23:11 -0500 Subject: [PATCH 03/14] allow requesting parallel tests for all OS --- script/test | 96 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 58 insertions(+), 38 deletions(-) diff --git a/script/test b/script/test index 24431b53c2f5..ecdc6698f842 100755 --- a/script/test +++ b/script/test @@ -189,34 +189,19 @@ function runBenchmarkTests (callback) { cp.on('close', exitCode => { callback(null, {exitCode, step: 'core-benchmarks'}) }) } -let testSuitesToRun = requestedTestSuites() || testSuitesForPlatform(process.platform) +let testSuitesToRun = requestedTestSuites(process.platform) -function requestedTestSuites () { - const suites = [] - if (argv.coreMain) { - suites.push(runCoreMainProcessTests) - } - if (argv.coreRenderer) { - suites.push(...coreRenderProcessTestSuites) - } - if (argv.coreBenchmark) { - suites.push(runBenchmarkTests) - } - if (argv.package) { - suites.push(...packageTestSuites) - } - return suites.length > 0 ? suites : null -} - -function testSuitesForPlatform (platform) { - let suites = [] - // Env variable options - let coreMain = process.env.ATOM_RUN_CORE_MAIN_TESTS === 'true' +function requestedTestSuites (platform) { + // env variable or argv options + let coreAll = process.env.ATOM_RUN_CORE_TESTS === 'true' + let coreMain = process.env.ATOM_RUN_CORE_MAIN_TESTS === 'true' || argv.coreMain + let coreRenderer = argv.coreRenderer || process.env.ATOM_RUN_CORE_RENDER_TESTS == 'true' let coreRenderer1 = process.env.ATOM_RUN_CORE_RENDER_TESTS === '1' let coreRenderer2 = process.env.ATOM_RUN_CORE_RENDER_TESTS === '2' - let coreAll = process.env.ATOM_RUN_CORE_TESTS === 'true' + let packageAll = argv.package || process.env.ATOM_RUN_PACKAGE_TESTS == 'true' let packages1 = process.env.ATOM_RUN_PACKAGE_TESTS === '1' let packages2 = process.env.ATOM_RUN_PACKAGE_TESTS === '2' + let benchmark = argv.coreBenchmark // Operating system overrides: coreMain = coreMain || (platform === 'linux') || (platform === 'win32' && process.arch === 'x86') @@ -224,28 +209,63 @@ function testSuitesForPlatform (platform) { // split package tests (used for macos in CI) const PACKAGES_TO_TEST_IN_PARALLEL = 23 // split core render test (used for windows x64 in CI) - const CORE_RENDER_TO_TEST_IN_PARALLEL = 35 - - if (coreMain) { - suites = [runCoreMainProcessTests] - } else if (coreRenderer1) { - suites = coreRenderProcessTestSuites.slice(0, CORE_RENDER_TO_TEST_IN_PARALLEL) - } else if (coreRenderer2) { - suites = coreRenderProcessTestSuites.slice(CORE_RENDER_TO_TEST_IN_PARALLEL) - } else if (coreAll) { - suites = [runCoreMainProcessTests, ...coreRenderProcessTestSuites] - } else if (packages1) { - suites = packageTestSuites.slice(0, PACKAGES_TO_TEST_IN_PARALLEL) - } else if (packages2) { - suites = packageTestSuites.slice(PACKAGES_TO_TEST_IN_PARALLEL) + const CORE_RENDER_TO_TEST_IN_PARALLEL = 45 + + let suites = [] + // Core tess + if (coreAll) { + suites.push(...[runCoreMainProcessTests, ...coreRenderProcessTestSuites]) } else { - suites = [runCoreMainProcessTests, ...coreRenderProcessTestSuites, ...packageTestSuites] // all + + // Core main tests + if (coreMain) { + suites.push(runCoreMainProcessTests) + } + + // Core renderer tests + if (coreRenderer) { + suites.push(...coreRenderProcessTestSuites) + } else { + // split + if (coreRenderer1) { + suites.push(...coreRenderProcessTestSuites.slice(0, CORE_RENDER_TO_TEST_IN_PARALLEL)) + } + if (coreRenderer2) { + suites.push(...coreRenderProcessTestSuites.slice(CORE_RENDER_TO_TEST_IN_PARALLEL)) + } + } + + } + + // Package tests + if (packageAll) { + suites.push(...packageTestSuites) + } else { + // split + if (packages1) { + suites.push(...packageTestSuites.slice(0, PACKAGES_TO_TEST_IN_PARALLEL)) + } + if (packages2) { + suites.push(...packageTestSuites.slice(PACKAGES_TO_TEST_IN_PARALLEL)) + } + } + + // Benchmark tests + if (benchmark) { + suites.push(runBenchmarkTests) } if (argv.skipMainProcessTests) { suites = suites.filter(suite => suite !== runCoreMainProcessTests) } + // Remove duplicates + suites = Array.from(new Set(suites)) + + if (suites.length == 0) { + throw new Error("No tests was requested") + } + return suites } From b61475ffe420672b3360b50aba7b0f347ea319d4 Mon Sep 17 00:00:00 2001 From: aminya Date: Wed, 15 Jul 2020 00:51:39 -0500 Subject: [PATCH 04/14] run tests using async.parallel --- script/test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/test b/script/test index ecdc6698f842..ff8bdf0c76cd 100755 --- a/script/test +++ b/script/test @@ -269,7 +269,7 @@ function requestedTestSuites (platform) { return suites } -async.series(testSuitesToRun, function (err, results) { +async.parallel(testSuitesToRun, function (err, results) { if (err) { console.error(err) process.exit(1) From 3d400d02cd748486fbf27d273b1936500eff583b Mon Sep 17 00:00:00 2001 From: aminya Date: Wed, 15 Jul 2020 04:37:49 -0500 Subject: [PATCH 05/14] don't use temp.track() cleanup process fails when the file is locked (in parallel tests) --- src/main-process/start.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main-process/start.js b/src/main-process/start.js index 82c0591a5e27..9f0d5816185a 100644 --- a/src/main-process/start.js +++ b/src/main-process/start.js @@ -1,7 +1,7 @@ const { app } = require('electron'); const nslog = require('nslog'); const path = require('path'); -const temp = require('temp').track(); +const temp = require('temp'); const parseCommandLine = require('./parse-command-line'); const startCrashReporter = require('../crash-reporter-start'); const getReleaseChannel = require('../get-release-channel'); From 266910d0598872ac6f2d545ee7d3802627f35111 Mon Sep 17 00:00:00 2001 From: aminya Date: Wed, 15 Jul 2020 05:03:00 -0500 Subject: [PATCH 06/14] warn before error reporting Helps visually --- script/test | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/script/test b/script/test index ff8bdf0c76cd..80a4d6b54cbf 100755 --- a/script/test +++ b/script/test @@ -276,10 +276,14 @@ async.parallel(testSuitesToRun, function (err, results) { } else { const failedSteps = results.filter(({exitCode}) => exitCode !== 0) - for (const {step} of failedSteps) { - console.error(`Error! The '${step}' test step finished with a non-zero exit code`) + if (failedSteps.length > 0) { + console.warn("\n \n *** Reporting the errors that happened in all of the tests: *** \n \n") + for (const {step} of failedSteps) { + console.error(`Error! The '${step}' test step finished with a non-zero exit code`) + } + process.exit(1) } - process.exit(failedSteps.length === 0 ? 0 : 1) + process.exit(0) } }) From 2fa0f6ebb9dfb43239d4ed27b4fd0cdcfc0324ca Mon Sep 17 00:00:00 2001 From: aminya Date: Thu, 16 Jul 2020 00:51:26 -0500 Subject: [PATCH 07/14] test: use Azure format for printing --- script/test | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/script/test b/script/test index 80a4d6b54cbf..a0ddfa4ce28c 100755 --- a/script/test +++ b/script/test @@ -60,7 +60,7 @@ if (process.platform === 'darwin') { assert(executablePaths.length === 1, `More than one application to run tests against was found. ${executablePaths.join(',')}`) executablePath = executablePaths[0] } else { - throw new Error('Running tests on this platform is not supported.') + throw new Error('##[error] Running tests on this platform is not supported.') } function prepareEnv (suiteName) { @@ -91,7 +91,7 @@ function runCoreMainProcessTests (callback) { const testEnv = Object.assign({}, prepareEnv('core-main-process'), {ATOM_GITHUB_INLINE_GIT_EXEC: 'true'}) - console.log('Executing core main process tests'.bold.green) + console.log('##[command] Executing core main process tests'.bold.green) const cp = childProcess.spawn(executablePath, testArguments, {stdio: 'inherit', env: testEnv}) cp.on('error', error => { callback(error) }) cp.on('close', exitCode => { callback(null, {exitCode, step: 'core-main-process'}) }) @@ -105,7 +105,7 @@ for (let testFile of testFiles) { coreRenderProcessTestSuites.push( function (callback) { const testEnv = prepareEnv('core-render-process') - console.log(`Executing core render process tests for ${testFile}`.bold.green) + console.log(`##[command] Executing core render process tests for ${testFile}`.bold.green) const testArguments = [ '--resource-path', resourcePath, '--test', testFile @@ -145,7 +145,7 @@ for (let packageName in CONFIG.appMetadata.packageDependencies) { const nodeModulesPath = path.join(repositoryPackagePath, 'node_modules') let finalize = () => null if (require(pkgJsonPath).atomTestRunner) { - console.log(`Installing test runner dependencies for ${packageName}`.bold.green) + console.log(`##[command] Installing test runner dependencies for ${packageName}`.bold.green) if (fs.existsSync(nodeModulesPath)) { const backup = backupNodeModules(repositoryPackagePath) finalize = backup.restore @@ -153,9 +153,9 @@ for (let packageName in CONFIG.appMetadata.packageDependencies) { finalize = () => fs.removeSync(nodeModulesPath) } runApmInstall(repositoryPackagePath) - console.log(`Executing ${packageName} tests`.green) + console.log(`##[command] Executing ${packageName} tests`.green) } else { - console.log(`Executing ${packageName} tests`.bold.green) + console.log(`##[command] Executing ${packageName} tests`.bold.green) } const cp = childProcess.spawn(executablePath, testArguments, {env: testEnv}) @@ -169,7 +169,7 @@ for (let packageName in CONFIG.appMetadata.packageDependencies) { }) cp.on('close', exitCode => { if (exitCode !== 0) { - console.log(`Package tests failed for ${packageName}:`.red) + console.log(`##[error] Package tests failed for ${packageName}:`.red) console.log(stderrOutput) } finalize() @@ -183,7 +183,7 @@ function runBenchmarkTests (callback) { const testArguments = ['--benchmark-test', benchmarksPath] const testEnv = prepareEnv('benchmark') - console.log('Executing benchmark tests'.bold.green) + console.log('##[command] Executing benchmark tests'.bold.green) const cp = childProcess.spawn(executablePath, testArguments, {stdio: 'inherit', env: testEnv}) cp.on('error', error => { callback(error) }) cp.on('close', exitCode => { callback(null, {exitCode, step: 'core-benchmarks'}) }) @@ -277,9 +277,9 @@ async.parallel(testSuitesToRun, function (err, results) { const failedSteps = results.filter(({exitCode}) => exitCode !== 0) if (failedSteps.length > 0) { - console.warn("\n \n *** Reporting the errors that happened in all of the tests: *** \n \n") + console.warn("##[error] \n \n *** Reporting the errors that happened in all of the tests: *** \n \n") for (const {step} of failedSteps) { - console.error(`Error! The '${step}' test step finished with a non-zero exit code`) + console.error(`##[error] The '${step}' test step finished with a non-zero exit code`) } process.exit(1) } From 92f4f7891e5a94728a0c69931439fc82f1ffb4d5 Mon Sep 17 00:00:00 2001 From: aminya Date: Thu, 16 Jul 2020 02:16:04 -0500 Subject: [PATCH 08/14] Run windows renderer tests in parallel --- script/vsts/platforms/templates/test.yml | 1 + script/vsts/platforms/windows.yml | 51 +++++++++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/script/vsts/platforms/templates/test.yml b/script/vsts/platforms/templates/test.yml index 72389353e264..a116fa3dcf33 100644 --- a/script/vsts/platforms/templates/test.yml +++ b/script/vsts/platforms/templates/test.yml @@ -21,6 +21,7 @@ steps: ATOM_JASMINE_REPORTER: list TEST_JUNIT_XML_ROOT: $(Common.TestResultsDirectory)/junit ATOM_RUN_CORE_TESTS: $(RunCoreTests) + ATOM_RUN_CORE_RENDER_TESTS: $(RunCoreRendererTests) ATOM_RUN_PACKAGE_TESTS: $(RunPackageTests) displayName: Run tests condition: and(succeeded(), ne(variables['Atom.SkipTests'], 'true')) diff --git a/script/vsts/platforms/windows.yml b/script/vsts/platforms/windows.yml index b9bd457023c7..a8083a5878c8 100644 --- a/script/vsts/platforms/windows.yml +++ b/script/vsts/platforms/windows.yml @@ -1,5 +1,5 @@ jobs: - - job: Windows + - job: Windows_Build dependsOn: GetReleaseVersion timeoutInMinutes: 180 strategy: @@ -67,3 +67,52 @@ jobs: - filename: RELEASES$(FileID) dir: $(Build.SourcesDirectory)/out condition: and(succeeded(), eq(variables['IsReleaseBranch'], 'true')) + + - job: Windows_RendererTests + dependsOn: Windows_Build + timeoutInMinutes: 180 + strategy: + maxParallel: 2 + matrix: + x64_Renderer_Test1: + RunCoreMainTests: false + RunCoreRendererTests: 1 + buildArch: x64 + x64_Renderer_Test2: + RunCoreMainTests: false + RunCoreRendererTests: 2 + buildArch: x64 + + pool: + vmImage: vs2017-win2016 + + variables: + AppName: $[ dependencies.GetReleaseVersion.outputs['Version.AppName'] ] + ReleaseVersion: $[ dependencies.GetReleaseVersion.outputs['Version.ReleaseVersion'] ] + IsReleaseBranch: $[ dependencies.GetReleaseVersion.outputs['Version.IsReleaseBranch'] ] + IsSignedZipBranch: $[ dependencies.GetReleaseVersion.outputs['Version.IsSignedZipBranch'] ] + + steps: + - template: templates/preparation.yml + + - template: templates/cache.yml + parameters: + OS: windows + + # Downloading the build artifacts + - pwsh: | + if ($env:BUILD_ARCH -eq "x64") { + $env:FileID="-x64" + echo "##vso[task.setvariable variable=FileID]$env:FileID" # Azure syntax + } + env: + BUILD_ARCH: $(buildArch) + displayName: Set FileID based on the arch + + - template: templates/download-unzip.yml + parameters: + artifacts: + - atom$(FileID)-windows.zip + + # Core renderer tests + - template: templates/test.yml From 314a2b6a2b906c8ee49ab2aa6437a3083412cecc Mon Sep 17 00:00:00 2001 From: aminya Date: Thu, 16 Jul 2020 02:55:36 -0500 Subject: [PATCH 09/14] Run windows core main tests in the build step --- script/vsts/platforms/templates/test.yml | 1 + script/vsts/platforms/windows.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/script/vsts/platforms/templates/test.yml b/script/vsts/platforms/templates/test.yml index a116fa3dcf33..b991f36f83eb 100644 --- a/script/vsts/platforms/templates/test.yml +++ b/script/vsts/platforms/templates/test.yml @@ -21,6 +21,7 @@ steps: ATOM_JASMINE_REPORTER: list TEST_JUNIT_XML_ROOT: $(Common.TestResultsDirectory)/junit ATOM_RUN_CORE_TESTS: $(RunCoreTests) + ATOM_RUN_CORE_MAIN_TESTS: $(RunCoreMainTests) ATOM_RUN_CORE_RENDER_TESTS: $(RunCoreRendererTests) ATOM_RUN_PACKAGE_TESTS: $(RunPackageTests) displayName: Run tests diff --git a/script/vsts/platforms/windows.yml b/script/vsts/platforms/windows.yml index a8083a5878c8..a79b6509d7db 100644 --- a/script/vsts/platforms/windows.yml +++ b/script/vsts/platforms/windows.yml @@ -7,8 +7,10 @@ jobs: matrix: x64: buildArch: x64 + RunCoreMainTests: true x86: buildArch: x86 + RunCoreMainTests: true pool: vmImage: vs2017-win2016 From f55b7b8dcee0bad16d73cf22da3d7a402b307d7e Mon Sep 17 00:00:00 2001 From: aminya Date: Thu, 16 Jul 2020 08:42:47 -0500 Subject: [PATCH 10/14] always upload atom windows.zip for x64 --- script/vsts/platforms/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/vsts/platforms/windows.yml b/script/vsts/platforms/windows.yml index a79b6509d7db..c6334483a408 100644 --- a/script/vsts/platforms/windows.yml +++ b/script/vsts/platforms/windows.yml @@ -55,7 +55,7 @@ jobs: artifacts: - filename: atom$(FileID)-windows.zip dir: $(Build.SourcesDirectory)/out - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + condition: and( succeeded(), or( eq(variables['BUILD_ARCH'], 'x64'), ne(variables['Build.Reason'], 'PullRequest') ) ) - filename: AtomSetup$(FileID).exe dir: $(Build.SourcesDirectory)/out condition: and(succeeded(), eq(variables['IsReleaseBranch'], 'true')) From c5af5781ab274270dc451f4ecff055b4ffbafcf3 Mon Sep 17 00:00:00 2001 From: aminya Date: Thu, 16 Jul 2020 08:42:52 -0500 Subject: [PATCH 11/14] bootstrap in case cache misses --- script/vsts/platforms/windows.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/vsts/platforms/windows.yml b/script/vsts/platforms/windows.yml index c6334483a408..81a36df75fb3 100644 --- a/script/vsts/platforms/windows.yml +++ b/script/vsts/platforms/windows.yml @@ -101,6 +101,8 @@ jobs: parameters: OS: windows + - template: templates/bootstrap.yml + # Downloading the build artifacts - pwsh: | if ($env:BUILD_ARCH -eq "x64") { From d56fef6d9fbb50d7f70b776e7b6a29572e83e037 Mon Sep 17 00:00:00 2001 From: aminya Date: Thu, 16 Jul 2020 20:36:53 -0500 Subject: [PATCH 12/14] macos: run core main tests in the build phase --- script/vsts/platforms/macos.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/script/vsts/platforms/macos.yml b/script/vsts/platforms/macos.yml index cc13d2c77d66..cd9b08fe83d5 100644 --- a/script/vsts/platforms/macos.yml +++ b/script/vsts/platforms/macos.yml @@ -8,6 +8,7 @@ jobs: ReleaseVersion: $[ dependencies.GetReleaseVersion.outputs['Version.ReleaseVersion'] ] IsReleaseBranch: $[ dependencies.GetReleaseVersion.outputs['Version.IsReleaseBranch'] ] IsSignedZipBranch: $[ dependencies.GetReleaseVersion.outputs['Version.IsSignedZipBranch'] ] + RunCoreMainTests: true pool: vmImage: macos-10.14 @@ -25,6 +26,9 @@ jobs: - template: templates/build.yml + # core main tests + - template: templates/test.yml + - script: | cp $(Build.SourcesDirectory)/out/*.zip $(Build.ArtifactStagingDirectory) displayName: Stage Artifacts @@ -51,8 +55,8 @@ jobs: strategy: maxParallel: 3 matrix: - core: - RunCoreTests: true + renderer: + RunCoreRendererTests: true RunPackageTests: false packages-1: RunCoreTests: false From a1dab28e4eded68342c10cffefad0b2308f3daf0 Mon Sep 17 00:00:00 2001 From: aminya Date: Thu, 16 Jul 2020 23:09:00 -0500 Subject: [PATCH 13/14] print the used testCommand for failed tests --- script/test | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/script/test b/script/test index a0ddfa4ce28c..77e481e298b4 100755 --- a/script/test +++ b/script/test @@ -94,7 +94,7 @@ function runCoreMainProcessTests (callback) { console.log('##[command] Executing core main process tests'.bold.green) const cp = childProcess.spawn(executablePath, testArguments, {stdio: 'inherit', env: testEnv}) cp.on('error', error => { callback(error) }) - cp.on('close', exitCode => { callback(null, {exitCode, step: 'core-main-process'}) }) + cp.on('close', exitCode => { callback(null, {exitCode, step: 'core-main-process', testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`}) }) } // Build an array of functions, each running tests for a different rendering test @@ -112,7 +112,7 @@ for (let testFile of testFiles) { ] const cp = childProcess.spawn(executablePath, testArguments, {stdio: 'inherit', env: testEnv}) cp.on('error', error => { callback(error) }) - cp.on('close', exitCode => { callback(null, {exitCode, step: `core-render-process for ${testFile}`}) }) + cp.on('close', exitCode => { callback(null, {exitCode, step: `core-render-process for ${testFile}.`, testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`}) }) }) } @@ -173,7 +173,7 @@ for (let packageName in CONFIG.appMetadata.packageDependencies) { console.log(stderrOutput) } finalize() - callback(null, {exitCode, step: `package-${packageName}`}) + callback(null, {exitCode, step: `package-${packageName}.`, testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`}) }) }) } @@ -186,7 +186,7 @@ function runBenchmarkTests (callback) { console.log('##[command] Executing benchmark tests'.bold.green) const cp = childProcess.spawn(executablePath, testArguments, {stdio: 'inherit', env: testEnv}) cp.on('error', error => { callback(error) }) - cp.on('close', exitCode => { callback(null, {exitCode, step: 'core-benchmarks'}) }) + cp.on('close', exitCode => { callback(null, {exitCode, step: 'core-benchmarks', testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`}) }) } let testSuitesToRun = requestedTestSuites(process.platform) @@ -278,8 +278,8 @@ async.parallel(testSuitesToRun, function (err, results) { if (failedSteps.length > 0) { console.warn("##[error] \n \n *** Reporting the errors that happened in all of the tests: *** \n \n") - for (const {step} of failedSteps) { - console.error(`##[error] The '${step}' test step finished with a non-zero exit code`) + for (const {step, testCommand} of failedSteps) { + console.error(`##[error] The '${step}' test step finished with a non-zero exit code \n ${testCommand}`) } process.exit(1) } From 582b6d97e32beb18ab6418f6e2f489fd80b8952a Mon Sep 17 00:00:00 2001 From: aminya Date: Fri, 17 Jul 2020 22:28:19 -0500 Subject: [PATCH 14/14] update message about finding a single application to run the tests --- script/test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/test b/script/test index 77e481e298b4..8c66a2ae9259 100755 --- a/script/test +++ b/script/test @@ -49,15 +49,15 @@ const resourcePath = CONFIG.repositoryRootPath let executablePath if (process.platform === 'darwin') { const executablePaths = glob.sync(path.join(CONFIG.buildOutputPath, '*.app')) - assert(executablePaths.length === 1, `More than one application to run tests against was found. ${executablePaths.join(',')}`) + assert(executablePaths.length === 1, `A single application to run tests against was not found. ${executablePaths.join(',')}`) executablePath = path.join(executablePaths[0], 'Contents', 'MacOS', path.basename(executablePaths[0], '.app')) } else if (process.platform === 'linux') { const executablePaths = glob.sync(path.join(CONFIG.buildOutputPath, 'atom-*', 'atom')) - assert(executablePaths.length === 1, `More than one application to run tests against was found. ${executablePaths.join(',')}`) + assert(executablePaths.length === 1, `A single application to run tests against was not found. ${executablePaths.join(',')}`) executablePath = executablePaths[0] } else if (process.platform === 'win32') { const executablePaths = glob.sync(path.join(CONFIG.buildOutputPath, '**', 'atom*.exe')) - assert(executablePaths.length === 1, `More than one application to run tests against was found. ${executablePaths.join(',')}`) + assert(executablePaths.length === 1, `A single application to run tests against was not found. ${executablePaths.join(',')}`) executablePath = executablePaths[0] } else { throw new Error('##[error] Running tests on this platform is not supported.')