TestScripts: Add support for running tests concurrently

Changes: * Change MiniDumper UI output so that it doesn't display any progress output to the console unless --verbose is passed in * Change ValidateHeapEntries to add --no-output option to not display any console output * Add null_stream as a wostream null device * Change Run powershell scripts to run multiple tests up to the number of CPU cores per test to speed up the running of the tests
shanepowell · Dec 4, 2022 · 2ba2031 · 2ba2031
1 parent 0a4d6b6
commit 2ba2031
Show file tree

Hide file tree

Showing 14 changed files with 362 additions and 77 deletions.
diff --git a/DbgHelpUtils/CMakeLists.txt b/DbgHelpUtils/CMakeLists.txt
@@ -333,6 +333,7 @@ set(Utility
     "hex_dump.h"
     "join.h"
     "memory_range.h"
+    "null_stream.h"
     "overload.h"
     "locale_number_formatting.cpp"
     "locale_number_formatting.h"

diff --git a/DbgHelpUtils/DbgHelpUtils.vcxproj b/DbgHelpUtils/DbgHelpUtils.vcxproj
@@ -342,6 +342,7 @@
     <ClInclude Include="memory_range.h" />
     <ClInclude Include="mini_dump_memory_stream.h" />
     <ClInclude Include="nt_heap.h" />
+    <ClInclude Include="null_stream.h" />
     <ClInclude Include="overload.h" />
     <ClInclude Include="page_range_descriptor.h" />
     <ClInclude Include="page_range_flags_utils.h" />

diff --git a/DbgHelpUtils/DbgHelpUtils.vcxproj.filters b/DbgHelpUtils/DbgHelpUtils.vcxproj.filters
@@ -959,6 +959,9 @@
     <ClInclude Include="heap_node_type.h">
       <Filter>Heap</Filter>
     </ClInclude>
+    <ClInclude Include="null_stream.h">
+      <Filter>Utility</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <Text Include="CMakeLists.txt" />

diff --git a/DbgHelpUtils/filesystem_utils.h b/DbgHelpUtils/filesystem_utils.h
@@ -1,5 +1,6 @@
 #pragma once
 #include <functional>
+#include <string_view>
 #include <vector>
 #include <experimental/generator>
 

diff --git a/DbgHelpUtils/null_stream.h b/DbgHelpUtils/null_stream.h
@@ -0,0 +1,27 @@
+#pragma once
+#include <ostream>
+
+class null_stream : public std::wostream
+{
+private:
+    class null_buffer : public std::wstreambuf
+    {
+    public:
+        int_type overflow( int_type const ch ) override
+        {
+            return ch;
+        }
+
+        // ReSharper disable once IdentifierTypo
+        std::streamsize xsputn( [[maybe_unused]] const char_type* s, std::streamsize const count ) override
+        {
+            return count;
+        }
+    } nb_;
+
+public:
+    null_stream()
+        : std::wostream{&nb_}
+    {
+    }
+};
diff --git a/MiniDumper/dump_mini_dump_heap_statistics.cpp b/MiniDumper/dump_mini_dump_heap_statistics.cpp
@@ -181,15 +181,21 @@ void dump_mini_dump_heap_statistics(std::wostream& log, mini_dump const& mini_du
     auto const loading_heap_statistics = L"Loading heap statistics..."s;
     std::wstring const move_back(loading_heap_statistics.size(), L'\b');
     std::wstring const clear(loading_heap_statistics.size(), L' ');
-    std::wcerr << loading_heap_statistics;
+    if(options.verbose_output())
+    {
+        std::wcerr << loading_heap_statistics;
+    }
 
     auto const hex_length = heaps.peb().machine_hex_printable_length();
     auto const is_x86_target = stream_stack_dump::is_x86_target_t{heaps.peb().is_x86_target()};
     auto const statistics = heaps.statistics();
 
-    std::wcerr << move_back;
-    std::wcerr << clear;
-    std::wcerr << move_back;
+    if(options.verbose_output())
+    {
+        std::wcerr << move_back;
+        std::wcerr << clear;
+        std::wcerr << move_back;
+    }
 
     log << L"Heap Statistics:\n";
     if(options.display_heap_statistic_view(heap_statistics_view::by_size_frequency_view))

diff --git a/MiniDumper/symbol_engine_ui.cpp b/MiniDumper/symbol_engine_ui.cpp
@@ -24,24 +24,30 @@ void symbol_engine_ui::deferred_symbol_load_partial([[maybe_unused]] std::wstrin
 void symbol_engine_ui::start_download(std::wstring_view const& module_name)
 {
     module_ = std::format(L"downloading {}: ", module_name);
-    std::wcerr << module_;
-    last_percent_.clear();
+    if(options_.verbose_output())
+    {
+        std::wcerr << module_;
+        last_percent_.clear();
+    }
 }
 
 void symbol_engine_ui::download_percent(unsigned const percent)
 {
-    if (!last_percent_.empty())
+    if (options_.verbose_output() && !last_percent_.empty())
     {
         std::wcerr << std::wstring(last_percent_.size(), L'\b');
     }
 
     last_percent_ = std::format(L"{}%", percent);
-    std::wcerr << last_percent_;
+    if(options_.verbose_output())
+    {
+        std::wcerr << last_percent_;
+    }
 }
 
 void symbol_engine_ui::download_complete()
 {
-    if (!last_percent_.empty() || !module_.empty())
+    if (options_.verbose_output() && (!last_percent_.empty() || !module_.empty()))
     {
         std::wstring const clear(last_percent_.size() + module_.size(), L'\b');
         std::wcerr << clear << std::wstring(last_percent_.size() + module_.size(), L' ') << clear;
@@ -52,7 +58,12 @@ void symbol_engine_ui::download_complete()
 
 std::wostream& symbol_engine_ui::log_stream() const
 {
-    return std::wcerr;
+    if(options_.verbose_output())
+    {
+        return std::wcerr;
+    }
+
+    return null_stream_;
 }
 
 bool symbol_engine_ui::symbol_load_debug() const

diff --git a/MiniDumper/symbol_engine_ui.h b/MiniDumper/symbol_engine_ui.h
@@ -2,6 +2,7 @@
 
 #include "dump_file_options.h"
 #include "DbgHelpUtils/i_symbol_load_callback.h"
+#include "DbgHelpUtils/null_stream.h"
 
 class symbol_engine_ui : public dlg_help_utils::dbg_help::i_symbol_load_callback
 {
@@ -21,4 +22,5 @@ class symbol_engine_ui : public dlg_help_utils::dbg_help::i_symbol_load_callback
     dump_file_options const& options_;
     std::wstring module_;
     std::wstring last_percent_;
+    mutable null_stream null_stream_;
 };
diff --git a/RunAllBuildHeapDumpTests.ps1 b/RunAllBuildHeapDumpTests.ps1
@@ -5,6 +5,7 @@ Param
     [switch] $CheckOnly,
     [switch] $GenerateHeapLogs,
     [switch] $LastReport,
+    [int] $ConcurrentLimit = 0,
     [switch] $Verbose,
     [switch] $ExitOnFailure
 )
@@ -40,19 +41,23 @@ if(!$LastReport)
 
     # run release / x64
     Write-Host "Running $Compiler Release x64 tests"
-    .\RunHeapDumpTests.ps1 -TestX86:$false -TestDebug:$false -CheckOnly:$CheckOnly -GenerateHeapLogs:$GenerateHeapLogs -Verbose:$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -ExitOnFailure:$ExitOnFailure
+    Write-Verbose "Command: \RunHeapDumpTests.ps1 -TestX86:`$false -TestDebug:`$false -CheckOnly:`$$CheckOnly -GenerateHeapLogs:`$$GenerateHeapLogs -Verbose:`$$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -ExitOnFailure:`$$ExitOnFailure -ConcurrentLimit:$ConcurrentLimit"
+    .\RunHeapDumpTests.ps1 -TestX86:$false -TestDebug:$false -CheckOnly:$CheckOnly -GenerateHeapLogs:$GenerateHeapLogs -Verbose:$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -ExitOnFailure:$ExitOnFailure -ConcurrentLimit:$ConcurrentLimit
 
     # run release / x86
     Write-Host "Running $Compiler Release x86 tests"
-    .\RunHeapDumpTests.ps1 -TestX86:$true -TestDebug:$false -CheckOnly:$CheckOnly -GenerateHeapLogs:$GenerateHeapLogs -Verbose:$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -ExitOnFailure:$ExitOnFailure
+    Write-Verbose "Command: \RunHeapDumpTests.ps1 -TestX86:`$true -TestDebug:`$false -CheckOnly:`$$CheckOnly -GenerateHeapLogs:`$$GenerateHeapLogs -Verbose:`$$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -ExitOnFailure:`$$ExitOnFailure -ConcurrentLimit:$ConcurrentLimit"
+    .\RunHeapDumpTests.ps1 -TestX86:$true -TestDebug:$false -CheckOnly:$CheckOnly -GenerateHeapLogs:$GenerateHeapLogs -Verbose:$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -ExitOnFailure:$ExitOnFailure -ConcurrentLimit:$ConcurrentLimit
 
     # run debug / x64
     Write-Host "Running $Compiler Debug x64 tests"
-    .\RunHeapDumpTests.ps1 -TestX86:$false -TestDebug:$true -CheckOnly:$CheckOnly -GenerateHeapLogs:$GenerateHeapLogs -Verbose:$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -ExitOnFailure:$ExitOnFailure
+    Write-Verbose "Command: \RunHeapDumpTests.ps1 -TestX86:`$false -TestDebug:`$true -CheckOnly:`$$CheckOnly -GenerateHeapLogs:`$$GenerateHeapLogs -Verbose:`$$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -ExitOnFailure:`$$ExitOnFailure -ConcurrentLimit:$ConcurrentLimit"
+    .\RunHeapDumpTests.ps1 -TestX86:$false -TestDebug:$true -CheckOnly:$CheckOnly -GenerateHeapLogs:$GenerateHeapLogs -Verbose:$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -ExitOnFailure:$ExitOnFailure -ConcurrentLimit:$ConcurrentLimit
 
     # run debug / x86
     Write-Host "Running $Compiler Debug x86 tests"
-    .\RunHeapDumpTests.ps1 -TestX86:$true -TestDebug:$true -CheckOnly:$CheckOnly -GenerateHeapLogs:$GenerateHeapLogs -Verbose:$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -ExitOnFailure:$ExitOnFailure
+    Write-Verbose "Command: \RunHeapDumpTests.ps1 -TestX86:`$true -TestDebug:`$true -CheckOnly:`$$CheckOnly -GenerateHeapLogs:`$$GenerateHeapLogs -Verbose:`$$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -ExitOnFailure:`$$ExitOnFailure -ConcurrentLimit:$ConcurrentLimit"
+    .\RunHeapDumpTests.ps1 -TestX86:$true -TestDebug:$true -CheckOnly:$CheckOnly -GenerateHeapLogs:$GenerateHeapLogs -Verbose:$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -ExitOnFailure:$ExitOnFailure -ConcurrentLimit:$ConcurrentLimit
 }
 
 
@@ -74,6 +79,7 @@ if($errorLines)
 {
     Write-Host "Complete with Errors:"
     Write-Host $errorLines
+    Write-Host "Completed with Errors"
 }
 else
 {

diff --git a/RunAllFailedTests.ps1 b/RunAllFailedTests.ps1
@@ -5,13 +5,92 @@ Param
     [switch] $GenerateHeapLogs,
     [switch] $LastReport,
     [string[]] $Filter,
+    [int] $ConcurrentLimit = 0,
     [switch] $Verbose
 )
 
+if($Verbose)
+{
+    $VerbosePreference = "continue"
+}
+
+if ($ConcurrentLimit -le 0)
+{
+    $ConcurrentLimit = $env:NUMBER_OF_PROCESSORS
+}
+
+$engine = Get-Process -id $pid | Get-Item
+
+Function GenerateTempReportFile($tempReportFiles)
+{
+    $tempFile = New-TemporaryFile
+    $tempReportFiles.Add($tempFile)
+    return $tempFile
+}
+
+Function RunTest($jobs, $tempReportFiles, $testArguments)
+{
+    if($ConcurrentLimit -eq 1)
+    {
+        $tempFile = GenerateTempReportFile $tempReportFiles
+        Write-Verbose "Command: .\RunHeapDumpTests.ps1 $testArguments -ResultFile:`"$($tempFile.FullName)`" -ConcurrentLimit:1"
+        Invoke-Expression -Command ".\RunHeapDumpTests.ps1 $testArguments -ResultFile:`"$($tempFile.FullName)`" -ConcurrentLimit:1"
+        return
+    }
+
+    # waiting for a job to complete to start another job
+    if(($jobs | Where-Object -FilterScript { -not $_.HasExited }).Count -ge $ConcurrentLimit)
+    {
+        Write-Verbose "Waiting to start job"
+        while(($jobs | Where-Object -FilterScript { -not $_.HasExited }).Count -ge $ConcurrentLimit)
+        {
+            Start-Sleep -Milliseconds 500
+        }
+        Write-Verbose "Wait complete"
+    }
+
+    $tempFile = GenerateTempReportFile $tempReportFiles
+    Write-Verbose "Command: $($engine.FullName) -NoLogo -NoProfile -NonInteractive -ExecutionPolicy RemoteSigned -File .\RunHeapDumpTests.ps1 $testArguments -ResultFile:`"$($tempFile.FullName)`""
+    $arguments = "-NoLogo -NoProfile -NonInteractive -ExecutionPolicy RemoteSigned -File .\RunHeapDumpTests.ps1 $testArguments -ResultFile:`"$($tempFile.FullName)`""
+
+    $pinfo = New-Object System.Diagnostics.ProcessStartInfo
+    $pinfo.FileName = $engine.FullName
+    $pinfo.UseShellExecute = $false
+    $pinfo.Arguments = $arguments
+    $pinfo.WorkingDirectory = Get-Location
+    $p = New-Object System.Diagnostics.Process
+    $p.StartInfo = $pinfo
+    $p.Start() | Out-Null
+
+    $jobs.Add($p)
+    Write-Verbose "Started job [$($p.Id)]"
+}
+
+Function RunTests($jobs, $tempReportFiles, $type, $name, $baseArguments)
+{
+    # run release / x64
+    Write-Host "Running $Compiler Release x64 test on $type $name"
+    $rv = RunTest $jobs $tempReportFiles "-TestX86:`$false -TestDebug:`$false $baseArguments"
+
+    # run release / x86
+    Write-Host "Running $Compiler Release x86 tests on $type $name"
+    $rv = RunTest $jobs $tempReportFiles "-TestX86:`$true -TestDebug:`$false $baseArguments"
+
+    # run debug / x64
+    Write-Host "Running $Compiler Debug x64 tests on $type $name"
+    $rv = RunTest $jobs $tempReportFiles "-TestX86:`$false -TestDebug:`$true $baseArguments"
+
+    # run debug / x86
+    Write-Host "Running $Compiler Debug x86 tests on $type $name"
+    $rv = RunTest $jobs $tempReportFiles "-TestX86:`$true -TestDebug:`$true $baseArguments"
+}
+
 if(!$LastReport)
 {
     Remove-Item $ResultFile -ErrorAction:SilentlyContinue | Out-Null
     $dmps = Get-ChildItem failed\*.dmp -Recurse
+    $tempReportFiles = New-Object System.Collections.ArrayList
+    $jobs = New-Object System.Collections.ArrayList
     foreach ($dmp in $dmps)
     {
         $CheckDumpHasStackTrace = $dmp.BaseName -match '_ust'
@@ -29,43 +108,45 @@ if(!$LastReport)
         elseif ($dmp.BaseName -match '_1$')
         {
             $name = $dmp.BaseName.SubString(0, $dmp.BaseName.Length - 2)
-            # run release / x64
-            Write-Host "Running $Compiler Release x64 test on set $name"
-            .\RunHeapDumpTests.ps1 -TestX86:$false -TestDebug:$false -CheckOnly -GenerateHeapLogs:$GenerateHeapLogs -Verbose:$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -CheckDumpFileBaseName $name -CheckDumpHasStackTrace:$CheckDumpHasStackTrace -SkipFakeOffsetCheck:$SkipFakeOffsetCheck -DumpFolder $dmp.Directory
-
-            # run release / x86
-            Write-Host "Running $Compiler Release x86 tests on set $name"
-            .\RunHeapDumpTests.ps1 -TestX86:$true -TestDebug:$false -CheckOnly -GenerateHeapLogs:$GenerateHeapLogs -Verbose:$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -CheckDumpFileBaseName $name -CheckDumpHasStackTrace:$CheckDumpHasStackTrace -SkipFakeOffsetCheck:$SkipFakeOffsetCheck -DumpFolder $dmp.Directory
+            $baseArguments = "-CheckOnly -GenerateHeapLogs:`$$GenerateHeapLogs -Verbose:`$$Verbose -Compiler:$Compiler -CheckDumpFileBaseName `"$name`" -CheckDumpHasStackTrace:`$$CheckDumpHasStackTrace -SkipFakeOffsetCheck:`$$SkipFakeOffsetCheck -DumpFolder `"$($dmp.Directory)`" -ConcurrentLimit:$ConcurrentLimit"
 
-            # run debug / x64
-            Write-Host "Running $Compiler Debug x64 tests on set $name"
-            .\RunHeapDumpTests.ps1 -TestX86:$false -TestDebug:$true -CheckOnly -GenerateHeapLogs:$GenerateHeapLogs -Verbose:$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -CheckDumpFileBaseName $name -CheckDumpHasStackTrace:$CheckDumpHasStackTrace -SkipFakeOffsetCheck:$SkipFakeOffsetCheck -DumpFolder $dmp.Directory
-
-            # run debug / x86
-            Write-Host "Running $Compiler Debug x86 tests on set $name"
-            .\RunHeapDumpTests.ps1 -TestX86:$true -TestDebug:$true -CheckOnly -GenerateHeapLogs:$GenerateHeapLogs -Verbose:$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -CheckDumpFileBaseName $name -CheckDumpHasStackTrace:$CheckDumpHasStackTrace -SkipFakeOffsetCheck:$SkipFakeOffsetCheck -DumpFolder $dmp.Directory
+            RunTests $jobs $tempReportFiles 'set' $name $baseArguments
         }
         else
         {
-            # run release / x64
-            Write-Host "Running $Compiler Release x64 test on single $dmp"
-            .\RunHeapDumpTests.ps1 -TestX86:$false -TestDebug:$false -CheckOnly -GenerateHeapLogs:$GenerateHeapLogs -Verbose:$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -CheckDumpFileBaseName $dmp.BaseName -CheckDumpHasStackTrace:$CheckDumpHasStackTrace -SkipFakeOffsetCheck:$SkipFakeOffsetCheck -SingleDumpOnly -DumpFolder $dmp.Directory
+            $baseArguments = "-CheckOnly -GenerateHeapLogs:`$$GenerateHeapLogs -Verbose:`$$Verbose -Compiler:$Compiler -CheckDumpFileBaseName `"$($dmp.BaseName)`" -CheckDumpHasStackTrace:`$$CheckDumpHasStackTrace -SkipFakeOffsetCheck:`$$SkipFakeOffsetCheck -SingleDumpOnly -DumpFolder `"$($dmp.Directory)`" -ConcurrentLimit:$ConcurrentLimit"
+            RunTests $jobs $tempReportFiles 'single' $dmp $baseArguments
+        }
+    }
 
-            # run release / x86
-            Write-Host "Running $Compiler Release x86 tests on single $dmp"
-            .\RunHeapDumpTests.ps1 -TestX86:$true -TestDebug:$false -CheckOnly -GenerateHeapLogs:$GenerateHeapLogs -Verbose:$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -CheckDumpFileBaseName $dmp.BaseName -CheckDumpHasStackTrace:$CheckDumpHasStackTrace -SkipFakeOffsetCheck:$SkipFakeOffsetCheck -SingleDumpOnly -DumpFolder $dmp.Directory
+    Write-Verbose "Waiting for all jobs to complete"
+    # wait for all jobs
+    foreach($p in $jobs)
+    {
+        $p.WaitForExit()
+    }
 
-            # run debug / x64
-            Write-Host "Running $Compiler Debug x64 tests on single $dmp"
-            .\RunHeapDumpTests.ps1 -TestX86:$false -TestDebug:$true -CheckOnly -GenerateHeapLogs:$GenerateHeapLogs -Verbose:$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -CheckDumpFileBaseName $dmp.BaseName -CheckDumpHasStackTrace:$CheckDumpHasStackTrace -SkipFakeOffsetCheck:$SkipFakeOffsetCheck -SingleDumpOnly -DumpFolder $dmp.Directory
+    Write-Verbose "Combining report output"
 
-            # run debug / x86
-            Write-Host "Running $Compiler Debug x86 tests on single $dmp"
-            .\RunHeapDumpTests.ps1 -TestX86:$true -TestDebug:$true -CheckOnly -GenerateHeapLogs:$GenerateHeapLogs -Verbose:$Verbose -ResultFile:$ResultFile -Compiler:$Compiler -CheckDumpFileBaseName $dmp.BaseName -CheckDumpHasStackTrace:$CheckDumpHasStackTrace -SkipFakeOffsetCheck:$SkipFakeOffsetCheck -SingleDumpOnly -DumpFolder $dmp.Directory
+    # Combine result files
+    foreach ($reportFile in $tempReportFiles)
+    {
+        if(Test-Path -Path $reportFile -PathType Leaf)
+        {
+            Write-Verbose "Append $reportFile to $ResultFile"
+            Get-Content $reportFile | Add-Content $ResultFile
+        } else {
+            Write-Error "Report File $reportFile doesn't exist!"
         }
     }
+
+    # wait for the above to complete it's file handling as the file may still be locked when we get to the remove temp files stage...
+    #Start-Sleep -Seconds 5
+
+    #$tempReportFiles | Remove-Item
 }
 
+Write-Verbose "Reading report output"
 
 $errorLines = ""
 $log = Get-Content $ResultFile
@@ -85,6 +166,7 @@ if($errorLines)
 {
     Write-Host "Complete with Errors:"
     Write-Host $errorLines
+    Write-Host "Completed with Errors"
 }
 else
 {