Parallel tests

script/test

@@ -49,18 +49,18 @@ const resourcePath = CONFIG.repositoryRootPath
 let executablePath
 if (process.platform === 'darwin') {
   const executablePaths = glob.sync(path.join(CONFIG.buildOutputPath, '*.app'))
-  assert(executablePaths.length === 1, `More than one application to run tests against was found. ${executablePaths.join(',')}`)
+  assert(executablePaths.length === 1, `A single application to run tests against was not found. ${executablePaths.join(',')}`)
   executablePath = path.join(executablePaths[0], 'Contents', 'MacOS', path.basename(executablePaths[0], '.app'))
 } else if (process.platform === 'linux') {
   const executablePaths = glob.sync(path.join(CONFIG.buildOutputPath, 'atom-*', 'atom'))
-  assert(executablePaths.length === 1, `More than one application to run tests against was found. ${executablePaths.join(',')}`)
+  assert(executablePaths.length === 1, `A single application to run tests against was not found. ${executablePaths.join(',')}`)
   executablePath = executablePaths[0]
 } else if (process.platform === 'win32') {
   const executablePaths = glob.sync(path.join(CONFIG.buildOutputPath, '**', 'atom*.exe'))
-  assert(executablePaths.length === 1, `More than one application to run tests against was found. ${executablePaths.join(',')}`)
+  assert(executablePaths.length === 1, `A single application to run tests against was not found. ${executablePaths.join(',')}`)
   executablePath = executablePaths[0]
 } else {
-  throw new Error('Running tests on this platform is not supported.')
+  throw new Error('##[error] Running tests on this platform is not supported.')
 }
 
 function prepareEnv (suiteName) {
@@ -91,24 +91,29 @@ function runCoreMainProcessTests (callback) {
 
   const testEnv = Object.assign({}, prepareEnv('core-main-process'), {ATOM_GITHUB_INLINE_GIT_EXEC: 'true'})
 
-  console.log('Executing core main process tests'.bold.green)
+  console.log('##[command] Executing core main process tests'.bold.green)
   const cp = childProcess.spawn(executablePath, testArguments, {stdio: 'inherit', env: testEnv})
   cp.on('error', error => { callback(error) })
-  cp.on('close', exitCode => { callback(null, {exitCode, step: 'core-main-process'}) })
+  cp.on('close', exitCode => { callback(null, {exitCode, step: 'core-main-process', testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`}) })
 }
 
-function runCoreRenderProcessTests (callback) {
-  const testPath = path.join(CONFIG.repositoryRootPath, 'spec')
-  const testArguments = [
-    '--resource-path', resourcePath,
-    '--test', testPath
-  ]
-  const testEnv = prepareEnv('core-render-process')
+// Build an array of functions, each running tests for a different rendering test
+const coreRenderProcessTestSuites = []
+const testPath = path.join(CONFIG.repositoryRootPath, 'spec')
+let testFiles = glob.sync(path.join(testPath, '*-spec.+(js|coffee|ts|jsx|tsx|mjs)'))
+for (let testFile of testFiles) {
+  coreRenderProcessTestSuites.push( function (callback) {
 
-  console.log('Executing core render process tests'.bold.green)
-  const cp = childProcess.spawn(executablePath, testArguments, {stdio: 'inherit', env: testEnv})
-  cp.on('error', error => { callback(error) })
-  cp.on('close', exitCode => { callback(null, {exitCode, step: 'core-render-process'}) })
+    const testEnv = prepareEnv('core-render-process')
+    console.log(`##[command] Executing core render process tests for ${testFile}`.bold.green)
+    const testArguments = [
+      '--resource-path', resourcePath,
+      '--test', testFile
+    ]
+    const cp = childProcess.spawn(executablePath, testArguments, {stdio: 'inherit', env: testEnv})
+    cp.on('error', error => { callback(error) })
+    cp.on('close', exitCode => { callback(null, {exitCode, step: `core-render-process for ${testFile}.`, testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`}) })
+  })
 }
 
 // Build an array of functions, each running tests for a different bundled package
@@ -140,17 +145,17 @@ for (let packageName in CONFIG.appMetadata.packageDependencies) {
     const nodeModulesPath = path.join(repositoryPackagePath, 'node_modules')
     let finalize = () => null
     if (require(pkgJsonPath).atomTestRunner) {
-      console.log(`Installing test runner dependencies for ${packageName}`.bold.green)
+      console.log(`##[command] Installing test runner dependencies for ${packageName}`.bold.green)
       if (fs.existsSync(nodeModulesPath)) {
         const backup = backupNodeModules(repositoryPackagePath)
         finalize = backup.restore
       } else {
         finalize = () => fs.removeSync(nodeModulesPath)
       }
       runApmInstall(repositoryPackagePath)
-      console.log(`Executing ${packageName} tests`.green)
+      console.log(`##[command] Executing ${packageName} tests`.green)
     } else {
-      console.log(`Executing ${packageName} tests`.bold.green)
+      console.log(`##[command] Executing ${packageName} tests`.bold.green)
     }
     const cp = childProcess.spawn(executablePath, testArguments, {env: testEnv})
 
@@ -164,11 +169,11 @@ for (let packageName in CONFIG.appMetadata.packageDependencies) {
     })
     cp.on('close', exitCode => {
       if (exitCode !== 0) {
-        console.log(`Package tests failed for ${packageName}:`.red)
+        console.log(`##[error] Package tests failed for ${packageName}:`.red)
         console.log(stderrOutput)
       }
       finalize()
-      callback(null, {exitCode, step: `package-${packageName}`})
+      callback(null, {exitCode, step: `package-${packageName}.`, testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`})
     })
   })
 }
@@ -178,75 +183,107 @@ function runBenchmarkTests (callback) {
   const testArguments = ['--benchmark-test', benchmarksPath]
   const testEnv = prepareEnv('benchmark')
 
-  console.log('Executing benchmark tests'.bold.green)
+  console.log('##[command] Executing benchmark tests'.bold.green)
   const cp = childProcess.spawn(executablePath, testArguments, {stdio: 'inherit', env: testEnv})
   cp.on('error', error => { callback(error) })
-  cp.on('close', exitCode => { callback(null, {exitCode, step: 'core-benchmarks'}) })
+  cp.on('close', exitCode => { callback(null, {exitCode, step: 'core-benchmarks', testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`}) })
 }
 
-let testSuitesToRun = requestedTestSuites() || testSuitesForPlatform(process.platform)
+let testSuitesToRun = requestedTestSuites(process.platform)
+
+function requestedTestSuites (platform) {
+  // env variable or argv options
+  let coreAll = process.env.ATOM_RUN_CORE_TESTS === 'true'
+  let coreMain = process.env.ATOM_RUN_CORE_MAIN_TESTS === 'true' || argv.coreMain
+  let coreRenderer = argv.coreRenderer || process.env.ATOM_RUN_CORE_RENDER_TESTS == 'true'
+  let coreRenderer1 = process.env.ATOM_RUN_CORE_RENDER_TESTS === '1'
+  let coreRenderer2 = process.env.ATOM_RUN_CORE_RENDER_TESTS === '2'
+  let packageAll = argv.package || process.env.ATOM_RUN_PACKAGE_TESTS == 'true'
+  let packages1 = process.env.ATOM_RUN_PACKAGE_TESTS === '1'
+  let packages2 = process.env.ATOM_RUN_PACKAGE_TESTS === '2'
+  let benchmark = argv.coreBenchmark
+
+  // Operating system overrides:
+  coreMain = coreMain || (platform === 'linux') || (platform === 'win32' && process.arch === 'x86')
+
+  // split package tests (used for macos in CI)
+  const PACKAGES_TO_TEST_IN_PARALLEL = 23
+  // split core render test (used for windows x64 in CI)
+  const CORE_RENDER_TO_TEST_IN_PARALLEL = 45
+
+  let suites = []
+  // Core tess
+  if (coreAll) {
+    suites.push(...[runCoreMainProcessTests, ...coreRenderProcessTestSuites])
+  } else {
+
+    // Core main tests
+    if (coreMain) {
+      suites.push(runCoreMainProcessTests)
+    }
+
+    // Core renderer tests
+    if (coreRenderer) {
+      suites.push(...coreRenderProcessTestSuites)
+    } else {
+      // split
+      if (coreRenderer1) {
+        suites.push(...coreRenderProcessTestSuites.slice(0, CORE_RENDER_TO_TEST_IN_PARALLEL))
+      }
+      if (coreRenderer2) {
+        suites.push(...coreRenderProcessTestSuites.slice(CORE_RENDER_TO_TEST_IN_PARALLEL))
+      }
+    }
 
-function requestedTestSuites () {
-  const suites = []
-  if (argv.coreMain) {
-    suites.push(runCoreMainProcessTests)
-  }
-  if (argv.coreRenderer) {
-    suites.push(runCoreRenderProcessTests)
-  }
-  if (argv.coreBenchmark) {
-    suites.push(runBenchmarkTests)
   }
-  if (argv.package) {
+
+  // Package tests
+  if (packageAll) {
     suites.push(...packageTestSuites)
+  } else {
+    // split
+    if (packages1) {
+      suites.push(...packageTestSuites.slice(0, PACKAGES_TO_TEST_IN_PARALLEL))
+    }
+    if (packages2) {
+      suites.push(...packageTestSuites.slice(PACKAGES_TO_TEST_IN_PARALLEL))
+    }
   }
-  return suites.length > 0 ? suites : null
-}
 
-function testSuitesForPlatform (platform) {
-  let suites = []
-  switch (platform) {
-    case 'darwin':
-      const PACKAGES_TO_TEST_IN_PARALLEL = 23
-
-      if (process.env.ATOM_RUN_CORE_TESTS === 'true') {
-        suites = [runCoreMainProcessTests, runCoreRenderProcessTests]
-      } else if (process.env.ATOM_RUN_PACKAGE_TESTS === '1') {
-        suites = packageTestSuites.slice(0, PACKAGES_TO_TEST_IN_PARALLEL)
-      } else if (process.env.ATOM_RUN_PACKAGE_TESTS === '2') {
-        suites = packageTestSuites.slice(PACKAGES_TO_TEST_IN_PARALLEL)
-      } else {
-        suites = [runCoreMainProcessTests, runCoreRenderProcessTests].concat(packageTestSuites)
-      }
-      break
-    case 'win32':
-      suites = (process.arch === 'x64') ? [runCoreMainProcessTests, runCoreRenderProcessTests] : [runCoreMainProcessTests]
-      break
-    case 'linux':
-      suites = [runCoreMainProcessTests]
-      break
-    default:
-      console.log(`Unrecognized platform: ${platform}`)
+  // Benchmark tests
+  if (benchmark) {
+    suites.push(runBenchmarkTests)
   }
 
   if (argv.skipMainProcessTests) {
     suites = suites.filter(suite => suite !== runCoreMainProcessTests)
   }
 
+  // Remove duplicates
+  suites = Array.from(new Set(suites))
+
+  if (suites.length == 0) {
+    throw new Error("No tests was requested")
+  }
+
   return suites
 }
 
-async.series(testSuitesToRun, function (err, results) {
+async.parallel(testSuitesToRun, function (err, results) {
   if (err) {
     console.error(err)
     process.exit(1)
   } else {
     const failedSteps = results.filter(({exitCode}) => exitCode !== 0)
 
-    for (const {step} of failedSteps) {
-      console.error(`Error! The '${step}' test step finished with a non-zero exit code`)
+    if (failedSteps.length > 0) {
+      console.warn("##[error] \n \n *** Reporting the errors that happened in all of the tests: *** \n \n")
+      for (const {step, testCommand} of failedSteps) {
+        console.error(`##[error] The '${step}' test step finished with a non-zero exit code \n ${testCommand}`)
+      }
+      process.exit(1)
     }
 
-    process.exit(failedSteps.length === 0 ? 0 : 1)
+    process.exit(0)
   }
 })

script/vsts/platforms/macos.yml

@@ -8,6 +8,7 @@ jobs:
       ReleaseVersion: $[ dependencies.GetReleaseVersion.outputs['Version.ReleaseVersion'] ]
       IsReleaseBranch: $[ dependencies.GetReleaseVersion.outputs['Version.IsReleaseBranch'] ]
       IsSignedZipBranch: $[ dependencies.GetReleaseVersion.outputs['Version.IsSignedZipBranch'] ]
+      RunCoreMainTests: true
     pool:
       vmImage: macos-10.14
 
@@ -25,6 +26,9 @@ jobs:
 
       - template: templates/build.yml
 
+      # core main tests
+      - template: templates/test.yml
+
       - script: |
           cp $(Build.SourcesDirectory)/out/*.zip $(Build.ArtifactStagingDirectory)
         displayName: Stage Artifacts
@@ -51,8 +55,8 @@ jobs:
     strategy:
       maxParallel: 3
       matrix:
-        core:
-          RunCoreTests: true
+        renderer:
+          RunCoreRendererTests: true
           RunPackageTests: false
         packages-1:
           RunCoreTests: false

script/vsts/platforms/templates/test.yml

@@ -21,6 +21,8 @@ steps:
       ATOM_JASMINE_REPORTER: list
       TEST_JUNIT_XML_ROOT: $(Common.TestResultsDirectory)/junit
       ATOM_RUN_CORE_TESTS: $(RunCoreTests)
+      ATOM_RUN_CORE_MAIN_TESTS: $(RunCoreMainTests)
+      ATOM_RUN_CORE_RENDER_TESTS: $(RunCoreRendererTests)
       ATOM_RUN_PACKAGE_TESTS: $(RunPackageTests)
     displayName: Run tests
     condition: and(succeeded(), ne(variables['Atom.SkipTests'], 'true'))

script/vsts/platforms/windows.yml

@@ -1,14 +1,16 @@
 jobs:
-  - job: Windows
+  - job: Windows_Build
     dependsOn: GetReleaseVersion
     timeoutInMinutes: 180
     strategy:
       maxParallel: 2
       matrix:
         x64:
           buildArch: x64
+          RunCoreMainTests: true
         x86:
           buildArch: x86
+          RunCoreMainTests: true
 
     pool:
       vmImage: vs2017-win2016
@@ -53,7 +55,7 @@ jobs:
           artifacts:
             - filename: atom$(FileID)-windows.zip
               dir: $(Build.SourcesDirectory)/out
-              condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+              condition: and( succeeded(), or( eq(variables['BUILD_ARCH'], 'x64'), ne(variables['Build.Reason'], 'PullRequest') ) )
             - filename: AtomSetup$(FileID).exe
               dir: $(Build.SourcesDirectory)/out
               condition: and(succeeded(), eq(variables['IsReleaseBranch'], 'true'))
@@ -67,3 +69,54 @@ jobs:
             - filename: RELEASES$(FileID)
               dir: $(Build.SourcesDirectory)/out
               condition: and(succeeded(), eq(variables['IsReleaseBranch'], 'true'))
+
+  - job: Windows_RendererTests
+    dependsOn: Windows_Build
+    timeoutInMinutes: 180
+    strategy:
+      maxParallel: 2
+      matrix:
+        x64_Renderer_Test1:
+          RunCoreMainTests: false
+          RunCoreRendererTests: 1
+          buildArch: x64
+        x64_Renderer_Test2:
+          RunCoreMainTests: false
+          RunCoreRendererTests: 2
+          buildArch: x64
+
+    pool:
+      vmImage: vs2017-win2016
+
+    variables:
+      AppName: $[ dependencies.GetReleaseVersion.outputs['Version.AppName'] ]
+      ReleaseVersion: $[ dependencies.GetReleaseVersion.outputs['Version.ReleaseVersion'] ]
+      IsReleaseBranch: $[ dependencies.GetReleaseVersion.outputs['Version.IsReleaseBranch'] ]
+      IsSignedZipBranch: $[ dependencies.GetReleaseVersion.outputs['Version.IsSignedZipBranch'] ]
+
+    steps:
+      - template: templates/preparation.yml
+
+      - template: templates/cache.yml
+        parameters:
+          OS: windows
+
+      - template: templates/bootstrap.yml
+
+      # Downloading the build artifacts
+      - pwsh: |
+          if ($env:BUILD_ARCH -eq "x64") {
+            $env:FileID="-x64"
+            echo "##vso[task.setvariable variable=FileID]$env:FileID" # Azure syntax
+          }
+        env:
+          BUILD_ARCH: $(buildArch)
+        displayName: Set FileID based on the arch
+
+      - template: templates/download-unzip.yml
+        parameters:
+          artifacts:
+            -  atom$(FileID)-windows.zip
+
+      #  Core renderer tests
+      - template: templates/test.yml

src/main-process/start.js

@@ -1,7 +1,7 @@
 const { app } = require('electron');
 const nslog = require('nslog');
 const path = require('path');
-const temp = require('temp').track();
+const temp = require('temp');
 const parseCommandLine = require('./parse-command-line');
 const startCrashReporter = require('../crash-reporter-start');
 const getReleaseChannel = require('../get-release-channel');