Add pipeline to run Pri-0 CoreCLR tests (#128)

Author: MichalStrehovsky

eng/pipelines/runtimelab.yml

@@ -85,5 +85,3 @@ jobs:
       testGroup: innerloop
       buildArgs: -s nativeaot+libs+installer -c $(_BuildConfig) /p:ArchiveTests=true
       extraStepsTemplate: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml
-      extraStepsParameters:
-        uploadTests: true

eng/pipelines/runtimelab/coreclr-tests.yml

@@ -0,0 +1,44 @@
+trigger: none
+
+jobs:
+#
+# Checkout repository
+#
+- template: /eng/pipelines/common/checkout-job.yml
+
+#
+# Build with Debug config and Checked runtimeConfiguration
+#
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/common/global-build-job.yml
+    buildConfig: Checked
+    platforms:
+    - Windows_NT_x64
+    jobParameters:
+      timeoutInMinutes: 720
+      testGroup: innerloop
+      buildArgs: -s nativeaot+libs+installer -lc release -rc checked
+      extraStepsTemplate: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml
+      extraStepsParameters:
+        smokeTestsOnly: false
+        runSingleFileTests: false
+
+
+#
+# Build with Release config and Release runtimeConfiguration
+#
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/common/global-build-job.yml
+    buildConfig: Release
+    platforms:
+    - Windows_NT_x64
+    jobParameters:
+      timeoutInMinutes: 720
+      testGroup: innerloop
+      buildArgs: -s nativeaot+libs+installer -c $(_BuildConfig) /p:ArchiveTests=true
+      extraStepsTemplate: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml
+      extraStepsParameters:
+        smokeTestsOnly: false
+        runSingleFileTests: false

eng/pipelines/runtimelab/runtimelab-post-build-steps.yml

@@ -4,21 +4,26 @@ parameters:
   osGroup: ''
   osSubgroup: ''
   uploadTests: false
+  smokeTestsOnly: true
+  runSingleFileTests: true
 
 steps:
   # Build coreclr native test output
   - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
-    - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) nativeaot skipstressdependencies $(buildConfigUpper) ${{ parameters.archType }} /p:NativeAotMultimodule=true /p:SmokeTestsOnly=true
+    - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) nativeaot skipstressdependencies $(buildConfigUpper) ${{ parameters.archType }} /p:NativeAotMultimodule=true /p:SmokeTestsOnly=${{ parameters.smokeTestsOnly }}
       displayName: Build tests
   - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
     - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) nativeaot skipstressdependencies $(buildConfigUpper) ${{ parameters.archType }} /p:SmokeTestsOnly=true
       displayName: Build tests
 
+  - ${{ if eq(parameters.runSingleFileTests, true) }}:
+    - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
+      - script: $(Build.SourcesDirectory)/src/tests/run$(scriptExt) runnativeaottests $(buildConfigUpper) ${{ parameters.archType }}
+        displayName: Run tests in single file mode
+    - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
+      - script: $(Build.SourcesDirectory)/src/tests/run$(scriptExt) --runnativeaottests $(buildConfigUpper) ${{ parameters.archType }}
+        displayName: Run tests in single file mode
+
   - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
-    - script: $(Build.SourcesDirectory)/src/tests/run$(scriptExt) runnativeaottests $(buildConfigUpper) ${{ parameters.archType }}
-      displayName: Run tests in single file mode
     - script: $(Build.SourcesDirectory)/src/tests/run$(scriptExt) runnativeaottests nativeaotmultimodule $(buildConfigUpper) ${{ parameters.archType }}
       displayName: Run tests in multifile mode
-  - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
-    - script: $(Build.SourcesDirectory)/src/tests/run$(scriptExt) --runnativeaottests $(buildConfigUpper) ${{ parameters.archType }}
-      displayName: Run tests

src/tests/Common/CLRTest.Execute.Batch.targets

@@ -312,6 +312,11 @@ set CLRTestExitCode=!ERRORLEVEL!
 if defined RunCrossGen (
   call :ReleaseLock
 )
+if defined RunNativeAot (
+  if NOT "%CLRTestNoCleanup%" == "" (
+    IF EXIST native rmdir /s /q native
+  )
+)
 $(BatchLinkerTestCleanupCmds)
       ]]></BatchCLRTestLaunchCmds>
 

src/tests/issues.targets

@@ -1197,6 +1197,12 @@
         <ExcludeList Include="$(XunitTestBinBase)/JIT/Methodical/Arrays/misc/_il_relinitializearray/*">
             <Issue>Implement general purpose RuntimeHelpers.InitializeArray</Issue>
         </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/Methodical/Arrays/misc/_il_dbginitializearray/*">
+            <Issue>Implement general purpose RuntimeHelpers.InitializeArray</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT\Methodical/explicit/rotate/_opt_dbgrotarg_float/*">
+            <Issue>Needs triage. Might be non-deterministic. Reproed without optimizations.</Issue>
+        </ExcludeList>
         <ExcludeList Include="$(XunitTestBinBase)/JIT/Methodical/eh/basics/throwinfilter_d/*">
             <Issue>Throwing from a filter block</Issue>
         </ExcludeList>
@@ -1320,6 +1326,12 @@
         <ExcludeList Include="$(XunitTestBinBase)/Loader/classloader/regressions/529206/vsw529206ModuleCctor/*">
             <Issue>Module constructors</Issue>
         </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/Loader/classloader/regressions/vsw529206/vsw529206ModuleCctor/*">
+            <Issue>Module constructors</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/Loader/classloader/regressions/asurt150271/test3/*">
+            <Issue>needs triage</Issue>
+        </ExcludeList>
         <ExcludeList Include="$(XunitTestBinBase)/Loader/classloader/TSAmbiguities/CollapsedInterfaces/HelloWorld/*">
             <Issue>Needs triage</Issue>
         </ExcludeList>
@@ -1344,6 +1356,9 @@
         <ExcludeList Include="$(XunitTestBinBase)/readytorun/**/*">
             <Issue>ReadyToRun tests</Issue>
         </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/reflection/Modifiers/modifiers/*">
+            <Issue>needs triage</Issue>
+        </ExcludeList>
         <ExcludeList Include="$(XunitTestBinBase)/reflection/SetValue/TrySetReadonlyStaticField/*">
             <Issue>Disallow setting readonly static fields</Issue>
         </ExcludeList>
@@ -1354,6 +1369,21 @@
             <Issue>EventPipe</Issue>
         </ExcludeList>
 
+        <!-- Release/optimized test failures -->
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/Regression/CLR-x86-JIT/V2.0-Beta2/b426654/b426654/*">
+            <Issue>Needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/Loader/classloader/methodoverriding/regressions/549411/exploit/*">
+            <Issue>Needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/Regression/CLR-x86-JIT/v2.1/DDB/B168384/LdfldaHack/*">
+            <Issue>Needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/Regression/JitBlue/DevDiv_754566/DevDiv_754566/*">
+            <Issue>Needs triage</Issue>
+        </ExcludeList>
+
+
         <!-- Catch(T) in shared code -->
         <ExcludeList Include="$(XunitTestBinBase)/JIT/Methodical\eh\generics\throwincatch_d\*" />
         <ExcludeList Include="$(XunitTestBinBase)/JIT/Methodical\eh\generics\throwincatch_do\*" />