diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/RetryOrchestrator.cs b/src/Platform/Microsoft.Testing.Extensions.Retry/RetryOrchestrator.cs index 59df6004f2..947df825d5 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/RetryOrchestrator.cs +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/RetryOrchestrator.cs @@ -292,7 +292,7 @@ public async Task OrchestrateTestHostExecutionAsync() } else { - await outputDevice.DisplayAsync(this, new FormattedTextOutputDeviceData(string.Format(CultureInfo.InvariantCulture, ExtensionResources.TestSuiteCompletedSuccessfully, attemptCount)) { ForegroundColor = new SystemConsoleColor { ConsoleColor = ConsoleColor.Green } }); + await outputDevice.DisplayAsync(this, new FormattedTextOutputDeviceData(string.Format(CultureInfo.InvariantCulture, ExtensionResources.TestSuiteCompletedSuccessfully, attemptCount)) { ForegroundColor = new SystemConsoleColor { ConsoleColor = ConsoleColor.DarkGreen } }); } } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/AnsiTerminalTestProgressFrame.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/AnsiTerminalTestProgressFrame.cs index 6489512a6b..c238cce484 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/AnsiTerminalTestProgressFrame.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/AnsiTerminalTestProgressFrame.cs @@ -37,7 +37,7 @@ public void AppendTestWorkerProgress(TestProgressState progress, RenderedProgres terminal.Append('['); charsTaken++; - terminal.SetColor(TerminalColor.Green); + terminal.SetColor(TerminalColor.DarkGreen); terminal.Append('✓'); charsTaken++; string passedText = passed.ToString(CultureInfo.CurrentCulture); @@ -48,7 +48,7 @@ public void AppendTestWorkerProgress(TestProgressState progress, RenderedProgres terminal.Append('/'); charsTaken++; - terminal.SetColor(TerminalColor.Red); + terminal.SetColor(TerminalColor.DarkRed); terminal.Append('x'); charsTaken++; string failedText = failed.ToString(CultureInfo.CurrentCulture); @@ -59,7 +59,7 @@ public void AppendTestWorkerProgress(TestProgressState progress, RenderedProgres terminal.Append('/'); charsTaken++; - terminal.SetColor(TerminalColor.Yellow); + terminal.SetColor(TerminalColor.DarkYellow); terminal.Append('↓'); charsTaken++; string skippedText = skipped.ToString(CultureInfo.CurrentCulture); diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/NonAnsiTerminal.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/NonAnsiTerminal.cs index 11af53dc1e..78b26ac13d 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/NonAnsiTerminal.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/NonAnsiTerminal.cs @@ -2,7 +2,6 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Platform.Helpers; -using Microsoft.Testing.Platform.Resources; namespace Microsoft.Testing.Platform.OutputDevice.Terminal; @@ -10,143 +9,48 @@ namespace Microsoft.Testing.Platform.OutputDevice.Terminal; /// Non-ANSI terminal that writes text using the standard Console.Foreground color capabilities to stay compatible with /// standard Windows command line, and other command lines that are not capable of ANSI, or when output is redirected. /// -internal sealed class NonAnsiTerminal : ITerminal +internal sealed class NonAnsiTerminal : SimpleTerminal { - private readonly IConsole _console; private readonly ConsoleColor _defaultForegroundColor; - private bool _isBatching; - private object? _batchingLock; + private bool? _colorNotSupported; public NonAnsiTerminal(IConsole console) - { - _console = console; - _defaultForegroundColor = IsForegroundColorNotSupported() ? ConsoleColor.Black : _console.GetForegroundColor(); - } - -#pragma warning disable CA1416 // Validate platform compatibility - public int Width => _console.IsOutputRedirected ? int.MaxValue : _console.BufferWidth; - - public int Height => _console.IsOutputRedirected ? int.MaxValue : _console.BufferHeight; -#pragma warning restore CA1416 // Validate platform compatibility - - public void Append(char value) - => _console.Write(value); - - public void Append(string value) - => _console.Write(value); - - public void AppendLine() - => _console.WriteLine(); - - public void AppendLine(string value) - => _console.WriteLine(value); - - public void AppendLink(string path, int? lineNumber) - { - Append(path); - if (lineNumber.HasValue) - { - Append($":{lineNumber}"); - } - } + : base(console) + => _defaultForegroundColor = IsForegroundColorNotSupported() ? ConsoleColor.Black : console.GetForegroundColor(); - public void SetColor(TerminalColor color) + public override void SetColor(TerminalColor color) { if (IsForegroundColorNotSupported()) { return; } - _console.SetForegroundColor(ToConsoleColor(color)); + Console.SetForegroundColor(ToConsoleColor(color)); } - public void ResetColor() + public override void ResetColor() { if (IsForegroundColorNotSupported()) { return; } - _console.SetForegroundColor(_defaultForegroundColor); - } - - public void ShowCursor() - { - // nop + Console.SetForegroundColor(_defaultForegroundColor); } - public void HideCursor() - { - // nop - } - - // TODO: Refactor NonAnsiTerminal and AnsiTerminal such that we don't need StartUpdate/StopUpdate. - // It's much better if we use lock C# keyword instead of manually calling Monitor.Enter/Exit - // Using lock also ensures we don't accidentally have `await`s in between that could cause Exit to be on a different thread. - public void StartUpdate() + [SupportedOSPlatformGuard("android")] + [SupportedOSPlatformGuard("ios")] + [SupportedOSPlatformGuard("tvos")] + [SupportedOSPlatformGuard("browser")] + private bool IsForegroundColorNotSupported() { - if (_isBatching) - { - throw new InvalidOperationException(PlatformResources.ConsoleIsAlreadyInBatchingMode); - } - - bool lockTaken = false; - - // We store Console.Out in a field to make sure we will be doing - // the Monitor.Exit call on the same instance. - _batchingLock = Console.Out; - - // Note that we need to lock on System.Out for batching to work correctly. - // Consider the following scenario: - // 1. We call StartUpdate - // 2. We call a Write("A") - // 3. User calls Console.Write("B") from another thread. - // 4. We call a Write("C"). - // 5. We call StopUpdate. - // The expectation is that we see either ACB, or BAC, but not ABC. - // Basically, when doing batching, we want to ensure that everything we write is - // written continuously, without anything in-between. - // One option (and we used to do it), is that we append to a StringBuilder while batching - // Then at StopUpdate, we write the whole string at once. - // This works to some extent, but we cannot get it to work when SetColor kicks in. - // Console methods will internally lock on Console.Out, so we are locking on the same thing. - // This locking is the easiest way to get coloring to work correctly while preventing - // interleaving with user's calls to Console.Write methods. - // One extra note: - // It's very important to lock on Console.Out (the current Console.Out). - // Consider the following scenario: - // 1. SystemConsole captures the original Console.Out set by runtime. - // 2. Framework author sets his own Console.Out which wraps the original Console.Out. - // 3. Two threads are writing concurrently: - // - One thread is writing using Console.Write* APIs, which will use the Console.Out set by framework author. - // - The other thread is writing using NonAnsiTerminal. - // 4. **If** we lock the original Console.Out. The following may happen (subject to race) [NOT THE CURRENT CASE - imaginary situation if we lock on the original Console.Out]: - // - First thread enters the Console.Write, which will acquire the lock for the current Console.Out (set by framework author). - // - Second thread executes StartUpdate, and acquires the lock for the original Console.Out. - // - First thread continues in the Write implementation of the framework author, which tries to run Console.Write on the original Console.Out. - // - First thread can't make any progress, because the second thread is holding the lock already. - // - Second thread continues execution, and reaches into runtime code (ConsolePal.WriteFromConsoleStream - on Unix) which tries to acquire the lock for the current Console.Out (set by framework author). - // - (see https://github.com/dotnet/runtime/blob/8a9d492444f06df20fcc5dfdcf7a6395af18361f/src/libraries/System.Console/src/System/ConsolePal.Unix.cs#L963) - // - No thread can progress. - // - Basically, what happened is that the first thread acquires the lock for current Console.Out, then for the original Console.Out. - // - while the second thread acquires the lock for the original Console.Out, then for the current Console.Out. - // - That's a typical deadlock where two threads are acquiring two locks in reverse order. - // 5. By locking the *current* Console.Out, we avoid the situation described in 4. - Monitor.Enter(_batchingLock, ref lockTaken); - if (!lockTaken) - { - // Can this happen? :/ - throw new InvalidOperationException(); - } - - _isBatching = true; - } + _colorNotSupported ??= RuntimeInformation.IsOSPlatform(OSPlatform.Create("ANDROID")) || + RuntimeInformation.IsOSPlatform(OSPlatform.Create("IOS")) || + RuntimeInformation.IsOSPlatform(OSPlatform.Create("TVOS")) || + RuntimeInformation.IsOSPlatform(OSPlatform.Create("WASI")) || + RuntimeInformation.IsOSPlatform(OSPlatform.Create("BROWSER")); - public void StopUpdate() - { - Monitor.Exit(_batchingLock!); - _batchingLock = null; - _isBatching = false; + return _colorNotSupported.Value; } private ConsoleColor ToConsoleColor(TerminalColor color) => color switch @@ -170,111 +74,4 @@ public void StopUpdate() TerminalColor.White => ConsoleColor.White, _ => _defaultForegroundColor, }; - - public void EraseProgress() - { - // nop - } - - public void RenderProgress(TestProgressState?[] progress) - { - int count = 0; - foreach (TestProgressState? p in progress) - { - if (p == null) - { - continue; - } - - count++; - - string durationString = HumanReadableDurationFormatter.Render(p.Stopwatch.Elapsed); - - int passed = p.PassedTests; - int failed = p.FailedTests; - int skipped = p.SkippedTests; - - // Use just ascii here, so we don't put too many restrictions on fonts needing to - // properly show unicode, or logs being saved in particular encoding. - Append('['); - SetColor(TerminalColor.DarkGreen); - Append('+'); - Append(passed.ToString(CultureInfo.CurrentCulture)); - ResetColor(); - - Append('/'); - - SetColor(TerminalColor.DarkRed); - Append('x'); - Append(failed.ToString(CultureInfo.CurrentCulture)); - ResetColor(); - - Append('/'); - - SetColor(TerminalColor.DarkYellow); - Append('?'); - Append(skipped.ToString(CultureInfo.CurrentCulture)); - ResetColor(); - Append(']'); - - Append(' '); - Append(p.AssemblyName); - - if (p.TargetFramework != null || p.Architecture != null) - { - Append(" ("); - if (p.TargetFramework != null) - { - Append(p.TargetFramework); - Append('|'); - } - - if (p.Architecture != null) - { - Append(p.Architecture); - } - - Append(')'); - } - - TestDetailState? activeTest = p.TestNodeResultsState?.GetRunningTasks(1).FirstOrDefault(); - if (!RoslynString.IsNullOrWhiteSpace(activeTest?.Text)) - { - Append(" - "); - Append(activeTest.Text); - Append(' '); - } - - Append(durationString); - - AppendLine(); - } - - // Do not render empty lines when there is nothing to show. - if (count > 0) - { - AppendLine(); - } - } - - public void StartBusyIndicator() - { - // nop - } - - public void StopBusyIndicator() - { - // nop - } - - [SupportedOSPlatformGuard("android")] - [SupportedOSPlatformGuard("ios")] - [SupportedOSPlatformGuard("tvos")] - [SupportedOSPlatformGuard("browser")] - private static bool IsForegroundColorNotSupported() - => RuntimeInformation.IsOSPlatform(OSPlatform.Create("ANDROID")) || - RuntimeInformation.IsOSPlatform(OSPlatform.Create("IOS")) || - RuntimeInformation.IsOSPlatform(OSPlatform.Create("TVOS")) || - RuntimeInformation.IsOSPlatform(OSPlatform.Create("WASI")) || - RuntimeInformation.IsOSPlatform(OSPlatform.Create("BROWSER")); } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/SimpleAnsiTerminal.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/SimpleAnsiTerminal.cs new file mode 100644 index 0000000000..12b4f4eee0 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/SimpleAnsiTerminal.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Helpers; + +namespace Microsoft.Testing.Platform.OutputDevice.Terminal; + +/// +/// Simple terminal that uses 4-bit ANSI for colors but does not move cursor and does not do other fancy stuff to stay compatible with CI systems like AzDO. +/// The colors are set on start of every line to properly color multiline strings in AzDO output. +/// +internal sealed class SimpleAnsiTerminal : SimpleTerminal +{ + private string? _foregroundColor; + private bool _prependColor; + + public SimpleAnsiTerminal(IConsole console) + : base(console) + { + } + + public override void Append(string value) + { + // Previous write appended line, so we need to prepend color. + if (_prependColor) + { + Console.Write(_foregroundColor); + // This line is not adding new line at the end, so we don't need to prepend color on next line. + _prependColor = false; + } + + Console.Write(SetColorPerLine(value)); + } + + public override void AppendLine(string value) + { + // Previous write appended line, so we need to prepend color. + if (_prependColor) + { + Console.Write(_foregroundColor); + } + + Console.WriteLine(SetColorPerLine(value)); + // This call appended new line so the next write to console needs to prepend color. + _prependColor = true; + } + + public override void SetColor(TerminalColor color) + { + string setColor = $"{AnsiCodes.CSI}{(int)color}{AnsiCodes.SetColor}"; + _foregroundColor = setColor; + Console.Write(setColor); + // This call set the color for current line, no need to prepend on next write. + _prependColor = false; + } + + public override void ResetColor() + { + _foregroundColor = null; + _prependColor = false; + Console.Write(AnsiCodes.SetDefaultColor); + } + + private string? SetColorPerLine(string value) + => _foregroundColor == null ? value : value.Replace("\n", $"\n{_foregroundColor}"); +} diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/SimpleTerminalBase.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/SimpleTerminalBase.cs new file mode 100644 index 0000000000..aefad9bc28 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/SimpleTerminalBase.cs @@ -0,0 +1,223 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Resources; + +namespace Microsoft.Testing.Platform.OutputDevice.Terminal; + +internal abstract class SimpleTerminal : ITerminal +{ + private object? _batchingLock; + private bool _isBatching; + + public SimpleTerminal(IConsole console) + => Console = console; + +#pragma warning disable CA1416 // Validate platform compatibility + public int Width => Console.IsOutputRedirected ? int.MaxValue : Console.BufferWidth; + + public int Height => Console.IsOutputRedirected ? int.MaxValue : Console.BufferHeight; + + protected IConsole Console { get; } + + public void Append(char value) + => Console.Write(value); + + public virtual void Append(string value) + => Console.Write(value); + + public void AppendLine() + => Console.WriteLine(); + + public virtual void AppendLine(string value) + => Console.WriteLine(value); + + public void AppendLink(string path, int? lineNumber) + { + Append(path); + if (lineNumber.HasValue) + { + Append($":{lineNumber}"); + } + } + + public void EraseProgress() + { + // nop + } + + public void HideCursor() + { + // nop + } + + public void RenderProgress(TestProgressState?[] progress) + { + int count = 0; + foreach (TestProgressState? p in progress) + { + if (p == null) + { + continue; + } + + count++; + + string durationString = HumanReadableDurationFormatter.Render(p.Stopwatch.Elapsed); + + int passed = p.PassedTests; + int failed = p.FailedTests; + int skipped = p.SkippedTests; + + // Use just ascii here, so we don't put too many restrictions on fonts needing to + // properly show unicode, or logs being saved in particular encoding. + Append('['); + SetColor(TerminalColor.DarkGreen); + Append('+'); + Append(passed.ToString(CultureInfo.CurrentCulture)); + ResetColor(); + + Append('/'); + + SetColor(TerminalColor.DarkRed); + Append('x'); + Append(failed.ToString(CultureInfo.CurrentCulture)); + ResetColor(); + + Append('/'); + + SetColor(TerminalColor.DarkYellow); + Append('?'); + Append(skipped.ToString(CultureInfo.CurrentCulture)); + ResetColor(); + Append(']'); + + Append(' '); + Append(p.AssemblyName); + + if (p.TargetFramework != null || p.Architecture != null) + { + Append(" ("); + if (p.TargetFramework != null) + { + Append(p.TargetFramework); + Append('|'); + } + + if (p.Architecture != null) + { + Append(p.Architecture); + } + + Append(')'); + } + + TestDetailState? activeTest = p.TestNodeResultsState?.GetRunningTasks(1).FirstOrDefault(); + if (!RoslynString.IsNullOrWhiteSpace(activeTest?.Text)) + { + Append(" - "); + Append(activeTest.Text); + Append(' '); + } + + Append(durationString); + + AppendLine(); + } + + // Do not render empty lines when there is nothing to show. + if (count > 0) + { + AppendLine(); + } + } + + public void ShowCursor() + { + // nop + } + + public void StartBusyIndicator() + { + // nop + } + + // TODO: Refactor NonAnsiTerminal and AnsiTerminal such that we don't need StartUpdate/StopUpdate. + // It's much better if we use lock C# keyword instead of manually calling Monitor.Enter/Exit + // Using lock also ensures we don't accidentally have `await`s in between that could cause Exit to be on a different thread. + public void StartUpdate() + { + if (_isBatching) + { + throw new InvalidOperationException(PlatformResources.ConsoleIsAlreadyInBatchingMode); + } + + bool lockTaken = false; + + // We store Console.Out in a field to make sure we will be doing + // the Monitor.Exit call on the same instance. + _batchingLock = System.Console.Out; + + // Note that we need to lock on System.Out for batching to work correctly. + // Consider the following scenario: + // 1. We call StartUpdate + // 2. We call a Write("A") + // 3. User calls Console.Write("B") from another thread. + // 4. We call a Write("C"). + // 5. We call StopUpdate. + // The expectation is that we see either ACB, or BAC, but not ABC. + // Basically, when doing batching, we want to ensure that everything we write is + // written continuously, without anything in-between. + // One option (and we used to do it), is that we append to a StringBuilder while batching + // Then at StopUpdate, we write the whole string at once. + // This works to some extent, but we cannot get it to work when SetColor kicks in. + // Console methods will internally lock on Console.Out, so we are locking on the same thing. + // This locking is the easiest way to get coloring to work correctly while preventing + // interleaving with user's calls to Console.Write methods. + // One extra note: + // It's very important to lock on Console.Out (the current Console.Out). + // Consider the following scenario: + // 1. SystemConsole captures the original Console.Out set by runtime. + // 2. Framework author sets his own Console.Out which wraps the original Console.Out. + // 3. Two threads are writing concurrently: + // - One thread is writing using Console.Write* APIs, which will use the Console.Out set by framework author. + // - The other thread is writing using NonAnsiTerminal. + // 4. **If** we lock the original Console.Out. The following may happen (subject to race) [NOT THE CURRENT CASE - imaginary situation if we lock on the original Console.Out]: + // - First thread enters the Console.Write, which will acquire the lock for the current Console.Out (set by framework author). + // - Second thread executes StartUpdate, and acquires the lock for the original Console.Out. + // - First thread continues in the Write implementation of the framework author, which tries to run Console.Write on the original Console.Out. + // - First thread can't make any progress, because the second thread is holding the lock already. + // - Second thread continues execution, and reaches into runtime code (ConsolePal.WriteFromConsoleStream - on Unix) which tries to acquire the lock for the current Console.Out (set by framework author). + // - (see https://github.com/dotnet/runtime/blob/8a9d492444f06df20fcc5dfdcf7a6395af18361f/src/libraries/System.Console/src/System/ConsolePal.Unix.cs#L963) + // - No thread can progress. + // - Basically, what happened is that the first thread acquires the lock for current Console.Out, then for the original Console.Out. + // - while the second thread acquires the lock for the original Console.Out, then for the current Console.Out. + // - That's a typical deadlock where two threads are acquiring two locks in reverse order. + // 5. By locking the *current* Console.Out, we avoid the situation described in 4. + Monitor.Enter(_batchingLock, ref lockTaken); + if (!lockTaken) + { + // Can this happen? :/ + throw new InvalidOperationException(); + } + + _isBatching = true; + } + + public void StopBusyIndicator() + { + // nop + } + + public void StopUpdate() + { + Monitor.Exit(_batchingLock!); + _batchingLock = null; + _isBatching = false; + } + + public abstract void SetColor(TerminalColor color); + + public abstract void ResetColor(); +} diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs index 905fc50137..e55fb70898 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs @@ -77,12 +77,20 @@ public TerminalTestReporter(IConsole console, TerminalTestReporterOptions option } else { - // Autodetect. - (bool consoleAcceptsAnsiCodes, bool _, uint? originalConsoleMode) = NativeMethods.QueryIsScreenAndTryEnableAnsiColorCodes(); - _originalConsoleMode = originalConsoleMode; - terminalWithProgress = consoleAcceptsAnsiCodes || _options.ForceAnsi is true - ? new TestProgressStateAwareTerminal(new AnsiTerminal(console, _options.BaseDirectory), showProgress, writeProgressImmediatelyAfterOutput: true, updateEvery: ansiUpdateCadenceInMs) - : new TestProgressStateAwareTerminal(new NonAnsiTerminal(console), showProgress, writeProgressImmediatelyAfterOutput: false, updateEvery: nonAnsiUpdateCadenceInMs); + if (_options.UseCIAnsi) + { + // We are told externally that we are in CI, use simplified ANSI mode. + terminalWithProgress = new TestProgressStateAwareTerminal(new SimpleAnsiTerminal(console), showProgress, writeProgressImmediatelyAfterOutput: true, updateEvery: nonAnsiUpdateCadenceInMs); + } + else + { + // We are not in CI, or in CI non-compatible with simple ANSI, autodetect terminal capabilities + (bool consoleAcceptsAnsiCodes, bool _, uint? originalConsoleMode) = NativeMethods.QueryIsScreenAndTryEnableAnsiColorCodes(); + _originalConsoleMode = originalConsoleMode; + terminalWithProgress = consoleAcceptsAnsiCodes || _options.ForceAnsi is true + ? new TestProgressStateAwareTerminal(new AnsiTerminal(console, _options.BaseDirectory), showProgress, writeProgressImmediatelyAfterOutput: true, updateEvery: ansiUpdateCadenceInMs) + : new TestProgressStateAwareTerminal(new NonAnsiTerminal(console), showProgress, writeProgressImmediatelyAfterOutput: false, updateEvery: nonAnsiUpdateCadenceInMs); + } } _terminalWithProgress = terminalWithProgress; @@ -179,7 +187,7 @@ private void AppendTestRunSummary(ITerminal terminal) bool allTestsWereSkipped = totalTests == 0 || totalTests == totalSkippedTests; bool anyTestFailed = totalFailedTests > 0; bool runFailed = anyTestFailed || notEnoughTests || allTestsWereSkipped || _wasCancelled; - terminal.SetColor(runFailed ? TerminalColor.Red : TerminalColor.Green); + terminal.SetColor(runFailed ? TerminalColor.DarkRed : TerminalColor.DarkGreen); terminal.Append(PlatformResources.TestRunSummary); terminal.Append(' '); @@ -248,7 +256,7 @@ private void AppendTestRunSummary(ITerminal terminal) terminal.AppendLine(totalText); if (colorizeFailed) { - terminal.SetColor(TerminalColor.Red); + terminal.SetColor(TerminalColor.DarkRed); } terminal.AppendLine(failedText); @@ -260,7 +268,7 @@ private void AppendTestRunSummary(ITerminal terminal) if (colorizePassed) { - terminal.SetColor(TerminalColor.Green); + terminal.SetColor(TerminalColor.DarkGreen); } terminal.AppendLine(passedText); @@ -272,7 +280,7 @@ private void AppendTestRunSummary(ITerminal terminal) if (colorizeSkipped) { - terminal.SetColor(TerminalColor.Yellow); + terminal.SetColor(TerminalColor.DarkYellow); } terminal.AppendLine(skippedText); @@ -294,7 +302,7 @@ private static void AppendAssemblyResult(ITerminal terminal, bool succeeded, int { if (!succeeded) { - terminal.SetColor(TerminalColor.Red); + terminal.SetColor(TerminalColor.DarkRed); // If the build failed, we print one of three red strings. string text = (countErrors > 0, countWarnings > 0) switch { @@ -308,13 +316,13 @@ private static void AppendAssemblyResult(ITerminal terminal, bool succeeded, int } else if (countWarnings > 0) { - terminal.SetColor(TerminalColor.Yellow); + terminal.SetColor(TerminalColor.DarkYellow); terminal.Append($"succeeded with {countWarnings} warning(s)"); terminal.ResetColor(); } else { - terminal.SetColor(TerminalColor.Green); + terminal.SetColor(TerminalColor.DarkGreen); terminal.Append(PlatformResources.PassedLowercase); terminal.ResetColor(); } @@ -442,9 +450,9 @@ private void RenderTestCompleted( TerminalColor color = outcome switch { - TestOutcome.Error or TestOutcome.Fail or TestOutcome.Canceled or TestOutcome.Timeout => TerminalColor.Red, - TestOutcome.Skipped => TerminalColor.Yellow, - TestOutcome.Passed => TerminalColor.Green, + TestOutcome.Error or TestOutcome.Fail or TestOutcome.Canceled or TestOutcome.Timeout => TerminalColor.DarkRed, + TestOutcome.Skipped => TerminalColor.DarkYellow, + TestOutcome.Passed => TerminalColor.DarkGreen, _ => throw new NotSupportedException(), }; string outcomeText = outcome switch @@ -492,7 +500,7 @@ private static void FormatInnerExceptions(ITerminal terminal, FlatException[] ex for (int i = 1; i < exceptions.Length; i++) { - terminal.SetColor(TerminalColor.Red); + terminal.SetColor(TerminalColor.DarkRed); terminal.Append(SingleIndentation); terminal.Append("--->"); FormatErrorMessage(terminal, exceptions, TestOutcome.Error, i); @@ -510,7 +518,7 @@ private static void FormatErrorMessage(ITerminal terminal, FlatException[] excep return; } - terminal.SetColor(TerminalColor.Red); + terminal.SetColor(TerminalColor.DarkRed); if (firstStackTrace is null) { @@ -539,7 +547,7 @@ private static void FormatExpectedAndActual(ITerminal terminal, string? expected return; } - terminal.SetColor(TerminalColor.Red); + terminal.SetColor(TerminalColor.DarkRed); terminal.Append(SingleIndentation); terminal.AppendLine(PlatformResources.Expected); AppendIndentedLine(terminal, expected, DoubleIndentation); @@ -781,7 +789,7 @@ internal void WriteErrorMessage(string assembly, string? targetFramework, string _terminalWithProgress.WriteToTerminal(terminal => { - terminal.SetColor(TerminalColor.Red); + terminal.SetColor(TerminalColor.DarkRed); if (padding == null) { terminal.AppendLine(text); @@ -801,7 +809,7 @@ internal void WriteWarningMessage(string assembly, string? targetFramework, stri asm.AddWarning(text); _terminalWithProgress.WriteToTerminal(terminal => { - terminal.SetColor(TerminalColor.Yellow); + terminal.SetColor(TerminalColor.DarkYellow); if (padding == null) { terminal.AppendLine(text); @@ -906,7 +914,7 @@ public void AppendTestDiscoverySummary(ITerminal terminal) terminal.AppendLine(); } - terminal.SetColor(runFailed ? TerminalColor.Red : TerminalColor.Green); + terminal.SetColor(runFailed ? TerminalColor.DarkRed : TerminalColor.DarkGreen); if (assemblies.Count <= 1) { terminal.Append(string.Format(CultureInfo.CurrentCulture, PlatformResources.TestDiscoverySummarySingular, totalTests)); diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporterOptions.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporterOptions.cs index 478453758d..bb610c1026 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporterOptions.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporterOptions.cs @@ -47,6 +47,12 @@ internal sealed class TerminalTestReporterOptions /// public bool UseAnsi { get; init; } + /// + /// Gets a value indicating whether we are running in compatible CI, and should use simplified ANSI renderer, which colors output, but does not move cursor. + /// Setting to false will disable this option. + /// + public bool UseCIAnsi { get; init; } + /// /// Gets a value indicating whether we should force ANSI escape codes. When true the ANSI is used without auto-detecting capabilities of the console. This is needed only for testing. /// diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs index ac46b81654..d47d80daf1 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs @@ -118,6 +118,9 @@ await _policiesService.RegisterOnAbortCallbackAsync( _isListTests = _commandLineOptions.IsOptionSet(PlatformCommandLineProvider.DiscoverTestsOptionKey); _isServerMode = _commandLineOptions.IsOptionSet(PlatformCommandLineProvider.ServerOptionKey); bool noAnsi = _commandLineOptions.IsOptionSet(TerminalTestReporterCommandLineOptionsProvider.NoAnsiOption); + + // TODO: Replace this with proper CI detection that we already have in telemetry. https://github.com/microsoft/testfx/issues/5533#issuecomment-2838893327 + bool inCI = string.Equals(_environment.GetEnvironmentVariable("TF_BUILD"), "true", StringComparison.OrdinalIgnoreCase) || string.Equals(_environment.GetEnvironmentVariable("GITHUB_ACTIONS"), "true", StringComparison.OrdinalIgnoreCase); bool noProgress = _commandLineOptions.IsOptionSet(TerminalTestReporterCommandLineOptionsProvider.NoProgressOption); // _runtimeFeature.IsHotReloadEnabled is not set to true here, even if the session will be HotReload, @@ -155,6 +158,7 @@ await _policiesService.RegisterOnAbortCallbackAsync( ShowPassedTests = showPassed, MinimumExpectedTests = PlatformCommandLineProvider.GetMinimumExpectedTests(_commandLineOptions), UseAnsi = !noAnsi, + UseCIAnsi = inCI, ShowActiveTests = true, ShowProgress = shouldShowProgress, }); diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/ConsoleTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/ConsoleTests.cs index 31f4b9b9b4..e035572d0f 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/ConsoleTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/ConsoleTests.cs @@ -43,7 +43,7 @@ private async Task ConsoleTestsCoreAsync(string tfm, string? environmentVariable }; } - TestHostResult testHostResult = await testHost.ExecuteAsync("--no-ansi --ignore-exit-code 8", environmentVariables); + TestHostResult testHostResult = await testHost.ExecuteAsync("--ignore-exit-code 8", environmentVariables); testHostResult.AssertExitCodeIs(ExitCodes.Success); testHostResult.AssertOutputContains("ABCDEF123"); } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/ExitOnProcessExitTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/ExitOnProcessExitTests.cs index 8e7ae8bb7f..d432b5e603 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/ExitOnProcessExitTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/ExitOnProcessExitTests.cs @@ -37,7 +37,7 @@ public void ExitOnProcessExit_Succeed(string tfm) } } - if (startTime.Elapsed.TotalSeconds > 55) + if (startTime.Elapsed.TotalSeconds > 60) { throw new Exception("Process PID not found in 60 seconds"); } @@ -48,7 +48,8 @@ public void ExitOnProcessExit_Succeed(string tfm) startTime = Stopwatch.StartNew(); while (!process.HasExited) { - if (startTime.Elapsed.TotalSeconds > 55) + Thread.Sleep(1000); + if (startTime.Elapsed.TotalSeconds > 60) { throw new Exception("Process did not exit in 60 seconds"); } @@ -83,7 +84,7 @@ public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture. using Microsoft.Testing.Platform.Extensions.Messages; using Microsoft.Testing.Platform.Extensions.TestFramework; -if (args.Length == 0) +if (!args.Contains("--exit-on-process-exit")) { int currentPid = Process.GetCurrentProcess().Id; var currentEntryPoint = Path.Combine(Path.GetDirectoryName(Assembly.GetEntryAssembly()!.Location)!, Path.GetFileNameWithoutExtension(Assembly.GetEntryAssembly()!.Location) @@ -93,7 +94,7 @@ public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture. Environment.SetEnvironmentVariable("WaitTestHost", mutexName); ProcessStartInfo processStartInfo = new(); processStartInfo.FileName = currentEntryPoint; - processStartInfo.Arguments = $"--exit-on-process-exit {currentPid}"; + processStartInfo.Arguments = $"--exit-on-process-exit {currentPid} --no-progress --no-ansi"; processStartInfo.UseShellExecute = false; var process = Process.Start(processStartInfo); while (!Mutex.TryOpenExisting(mutexName, out Mutex? _)) diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs index 0e3467d86c..1cce4e85cc 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs @@ -29,6 +29,8 @@ public void AppendStackFrameFormatsStackTraceLineCorrectly() #if NETCOREAPP StringAssert.Contains(terminal.Output, " at Microsoft.Testing.Platform.UnitTests.TerminalTestReporterTests.AppendStackFrameFormatsStackTraceLineCorrectly() in "); #else + // This is caused by us using portable symbols, and .NET Framework 4.6.2, once we update to .NET Framework 4.7.2 the path to file will be included in the stacktrace and this won't be necessary. + // See first point here: https://learn.microsoft.com/en-us/dotnet/core/diagnostics/symbols#support-for-portable-pdbs StringAssert.Contains(terminal.Output, " at Microsoft.Testing.Platform.UnitTests.TerminalTestReporterTests.AppendStackFrameFormatsStackTraceLineCorrectly()"); #endif // Line number without the respective file @@ -64,13 +66,216 @@ public void StackTraceRegexCapturesLines(string stackTraceLine, string expected) } [TestMethod] - public void OutputFormattingIsCorrect() + public void NonAnsiTerminal_OutputFormattingIsCorrect() { var stringBuilderConsole = new StringBuilderConsole(); var terminalReporter = new TerminalTestReporter(stringBuilderConsole, new TerminalTestReporterOptions { ShowPassedTests = () => true, + + // Like --no-ansi in commandline, should disable ANSI altogether. + UseAnsi = false, + + ShowAssembly = false, + ShowAssemblyStartAndComplete = false, + ShowProgress = () => false, + }); + + DateTimeOffset startTime = DateTimeOffset.MinValue; + DateTimeOffset endTime = DateTimeOffset.MaxValue; + terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false); + + string targetFramework = "net8.0"; + string architecture = "x64"; + string assembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\assembly.dll" : "/mnt/work/assembly.dll"; + string folder = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\" : "/mnt/work/"; + string folderNoSlash = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work" : "/mnt/work"; + string folderLink = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:/work/" : "mnt/work/"; + string folderLinkNoSlash = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:/work" : "mnt/work"; + + terminalReporter.AssemblyRunStarted(assembly, targetFramework, architecture); + string standardOutput = "Hello!"; + string errorOutput = "Oh no!"; + + terminalReporter.TestCompleted(assembly, targetFramework, architecture, testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(10), + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); + terminalReporter.TestCompleted(assembly, targetFramework, architecture, testNodeUid: "SkippedTest1", "SkippedTest1", TestOutcome.Skipped, TimeSpan.FromSeconds(10), + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); + // timed out + canceled + failed should all report as failed in summary + terminalReporter.TestCompleted(assembly, targetFramework, architecture, testNodeUid: "TimedoutTest1", "TimedoutTest1", TestOutcome.Timeout, TimeSpan.FromSeconds(10), + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); + terminalReporter.TestCompleted(assembly, targetFramework, architecture, testNodeUid: "CanceledTest1", "CanceledTest1", TestOutcome.Canceled, TimeSpan.FromSeconds(10), + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); + terminalReporter.TestCompleted(assembly, targetFramework, architecture, testNodeUid: "FailedTest1", "FailedTest1", TestOutcome.Fail, TimeSpan.FromSeconds(10), + informativeMessage: null, errorMessage: "Tests failed", exception: new StackTraceException(@$" at FailingTest() in {folder}codefile.cs:line 10"), expected: "ABC", actual: "DEF", standardOutput, errorOutput); + terminalReporter.ArtifactAdded(outOfProcess: true, assembly, targetFramework, architecture, testName: null, @$"{folder}artifact1.txt"); + terminalReporter.ArtifactAdded(outOfProcess: false, assembly, targetFramework, architecture, testName: null, @$"{folder}artifact2.txt"); + terminalReporter.AssemblyRunCompleted(assembly, targetFramework, architecture, exitCode: null, outputData: null, errorData: null); + terminalReporter.TestExecutionCompleted(endTime); + + string output = stringBuilderConsole.Output; + + string expected = $""" + passed PassedTest1 (10s 000ms) + Standard output + Hello! + Error output + Oh no! + skipped SkippedTest1 (10s 000ms) + Standard output + Hello! + Error output + Oh no! + failed (canceled) TimedoutTest1 (10s 000ms) + Standard output + Hello! + Error output + Oh no! + failed (canceled) CanceledTest1 (10s 000ms) + Standard output + Hello! + Error output + Oh no! + failed FailedTest1 (10s 000ms) + Tests failed + Expected + ABC + Actual + DEF + at FailingTest() in {folder}codefile.cs:10 + Standard output + Hello! + Error output + Oh no! + + Out of process file artifacts produced: + - {folder}artifact1.txt + In process file artifacts produced: + - {folder}artifact2.txt + + Test run summary: Failed! - {assembly} (net8.0|x64) + total: 5 + failed: 3 + succeeded: 1 + skipped: 1 + duration: 3652058d 23h 59m 59s 999ms + + """; + + Assert.AreEqual(expected, ShowEscape(output)); + } + + [TestMethod] + public void SimpleAnsiTerminal_OutputFormattingIsCorrect() + { + var stringBuilderConsole = new StringBuilderConsole(); + var terminalReporter = new TerminalTestReporter(stringBuilderConsole, new TerminalTestReporterOptions + { + ShowPassedTests = () => true, + // Like if we autodetect that we are in CI (e.g. by looking at TF_BUILD, and we don't disable ANSI. + UseAnsi = true, + UseCIAnsi = true, + ForceAnsi = true, + + ShowAssembly = false, + ShowAssemblyStartAndComplete = false, + ShowProgress = () => false, + }); + + DateTimeOffset startTime = DateTimeOffset.MinValue; + DateTimeOffset endTime = DateTimeOffset.MaxValue; + terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false); + + string targetFramework = "net8.0"; + string architecture = "x64"; + string assembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\assembly.dll" : "/mnt/work/assembly.dll"; + string folder = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\" : "/mnt/work/"; + string folderNoSlash = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work" : "/mnt/work"; + string folderLink = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:/work/" : "mnt/work/"; + string folderLinkNoSlash = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:/work" : "mnt/work"; + + terminalReporter.AssemblyRunStarted(assembly, targetFramework, architecture); + string standardOutput = "Hello!"; + string errorOutput = "Oh no!"; + + terminalReporter.TestCompleted(assembly, targetFramework, architecture, testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(10), + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); + terminalReporter.TestCompleted(assembly, targetFramework, architecture, testNodeUid: "SkippedTest1", "SkippedTest1", TestOutcome.Skipped, TimeSpan.FromSeconds(10), + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); + // timed out + canceled + failed should all report as failed in summary + terminalReporter.TestCompleted(assembly, targetFramework, architecture, testNodeUid: "TimedoutTest1", "TimedoutTest1", TestOutcome.Timeout, TimeSpan.FromSeconds(10), + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); + terminalReporter.TestCompleted(assembly, targetFramework, architecture, testNodeUid: "CanceledTest1", "CanceledTest1", TestOutcome.Canceled, TimeSpan.FromSeconds(10), + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); + terminalReporter.TestCompleted(assembly, targetFramework, architecture, testNodeUid: "FailedTest1", "FailedTest1", TestOutcome.Fail, TimeSpan.FromSeconds(10), + informativeMessage: null, errorMessage: "Tests failed", exception: new StackTraceException(@$" at FailingTest() in {folder}codefile.cs:line 10"), expected: "ABC", actual: "DEF", standardOutput, errorOutput); + terminalReporter.ArtifactAdded(outOfProcess: true, assembly, targetFramework, architecture, testName: null, @$"{folder}artifact1.txt"); + terminalReporter.ArtifactAdded(outOfProcess: false, assembly, targetFramework, architecture, testName: null, @$"{folder}artifact2.txt"); + terminalReporter.AssemblyRunCompleted(assembly, targetFramework, architecture, exitCode: null, outputData: null, errorData: null); + terminalReporter.TestExecutionCompleted(endTime); + + string output = stringBuilderConsole.Output; + + string expected = $""" + ␛[32mpassed␛[m PassedTest1␛[90m ␛[90m(10s 000ms)␛[m + ␛[90m Standard output + ␛[90m Hello! + ␛[90m Error output + ␛[90m Oh no! + ␛[m␛[33mskipped␛[m SkippedTest1␛[90m ␛[90m(10s 000ms)␛[m + ␛[90m Standard output + ␛[90m Hello! + ␛[90m Error output + ␛[90m Oh no! + ␛[m␛[31mfailed (canceled)␛[m TimedoutTest1␛[90m ␛[90m(10s 000ms)␛[m + ␛[90m Standard output + ␛[90m Hello! + ␛[90m Error output + ␛[90m Oh no! + ␛[m␛[31mfailed (canceled)␛[m CanceledTest1␛[90m ␛[90m(10s 000ms)␛[m + ␛[90m Standard output + ␛[90m Hello! + ␛[90m Error output + ␛[90m Oh no! + ␛[m␛[31mfailed␛[m FailedTest1␛[90m ␛[90m(10s 000ms)␛[m + ␛[31m Tests failed + ␛[m␛[31m Expected + ␛[31m ABC + ␛[31m Actual + ␛[31m DEF + ␛[m␛[90m at FailingTest() in {folder}codefile.cs:10␛[90m + ␛[m␛[90m Standard output + ␛[90m Hello! + ␛[90m Error output + ␛[90m Oh no! + ␛[m + Out of process file artifacts produced: + - {folder}artifact1.txt + In process file artifacts produced: + - {folder}artifact2.txt + + ␛[31mTest run summary: Failed!␛[90m - ␛[m{folder}assembly.dll (net8.0|x64) + ␛[m total: 5 + ␛[31m failed: 3 + ␛[m succeeded: 1 + skipped: 1 + duration: 3652058d 23h 59m 59s 999ms + + """; + + Assert.AreEqual(expected, ShowEscape(output)); + } + + [TestMethod] + public void AnsiTerminal_OutputFormattingIsCorrect() + { + var stringBuilderConsole = new StringBuilderConsole(); + var terminalReporter = new TerminalTestReporter(stringBuilderConsole, new TerminalTestReporterOptions + { + ShowPassedTests = () => true, + // Like if we autodetect that we are in ANSI capable terminal. UseAnsi = true, + UseCIAnsi = false, ForceAnsi = true, ShowAssembly = false, @@ -113,29 +318,29 @@ public void OutputFormattingIsCorrect() string output = stringBuilderConsole.Output; string expected = $""" - ␛[92mpassed␛[m PassedTest1␛[90m ␛[90m(10s 000ms)␛[m + ␛[32mpassed␛[m PassedTest1␛[90m ␛[90m(10s 000ms)␛[m ␛[90m Standard output Hello! Error output Oh no! - ␛[m␛[93mskipped␛[m SkippedTest1␛[90m ␛[90m(10s 000ms)␛[m + ␛[m␛[33mskipped␛[m SkippedTest1␛[90m ␛[90m(10s 000ms)␛[m ␛[90m Standard output Hello! Error output Oh no! - ␛[m␛[91mfailed (canceled)␛[m TimedoutTest1␛[90m ␛[90m(10s 000ms)␛[m + ␛[m␛[31mfailed (canceled)␛[m TimedoutTest1␛[90m ␛[90m(10s 000ms)␛[m ␛[90m Standard output Hello! Error output Oh no! - ␛[m␛[91mfailed (canceled)␛[m CanceledTest1␛[90m ␛[90m(10s 000ms)␛[m + ␛[m␛[31mfailed (canceled)␛[m CanceledTest1␛[90m ␛[90m(10s 000ms)␛[m ␛[90m Standard output Hello! Error output Oh no! - ␛[m␛[91mfailed␛[m FailedTest1␛[90m ␛[90m(10s 000ms)␛[m - ␛[91m Tests failed - ␛[m␛[91m Expected + ␛[m␛[31mfailed␛[m FailedTest1␛[90m ␛[90m(10s 000ms)␛[m + ␛[31m Tests failed + ␛[m␛[31m Expected ABC Actual DEF @@ -149,28 +354,30 @@ Oh no! - ␛[90m␛]8;;file:///{folderLink}artifact1.txt␛\{folder}artifact1.txt␛]8;;␛\␛[m In process file artifacts produced: - ␛[90m␛]8;;file:///{folderLink}artifact2.txt␛\{folder}artifact2.txt␛]8;;␛\␛[m - - ␛[91mTest run summary: Failed!␛[90m - ␛[m␛[90m␛]8;;file:///{folderLinkNoSlash}␛\{folder}assembly.dll␛]8;;␛\␛[m (net8.0|x64) + + ␛[31mTest run summary: Failed!␛[90m - ␛[m␛[90m␛]8;;file:///{folderLinkNoSlash}␛\{folder}assembly.dll␛]8;;␛\␛[m (net8.0|x64) ␛[m total: 5 - ␛[91m failed: 3 + ␛[31m failed: 3 ␛[m succeeded: 1 skipped: 1 duration: 3652058d 23h 59m 59s 999ms - + """; Assert.AreEqual(expected, ShowEscape(output)); } [TestMethod] - public void OutputProgressFrameIsCorrect() + public void AnsiTerminal_OutputProgressFrameIsCorrect() { var stringBuilderConsole = new StringBuilderConsole(); var stopwatchFactory = new StopwatchFactory(); var terminalReporter = new TerminalTestReporter(stringBuilderConsole, new TerminalTestReporterOptions { ShowPassedTests = () => true, + // Like if we autodetect that we are in ANSI capable terminal. UseAnsi = true, + UseCIAnsi = false, ForceAnsi = true, ShowActiveTests = true, @@ -233,29 +440,29 @@ public void OutputProgressFrameIsCorrect() // Note: The progress is drawn after each completed event. string expected = $""" - {busyIndicatorString}␛[?25l␛[92mpassed␛[m PassedTest1␛[90m ␛[90m(10s 000ms)␛[m + {busyIndicatorString}␛[?25l␛[32mpassed␛[m PassedTest1␛[90m ␛[90m(10s 000ms)␛[m ␛[90m Standard output Hello! Error output Oh no! ␛[m - [␛[92m✓1␛[m/␛[91mx0␛[m/␛[93m↓0␛[m] assembly.dll (net8.0|x64)␛[2147483640G(1m 31s) + [␛[32m✓1␛[m/␛[31mx0␛[m/␛[33m↓0␛[m] assembly.dll (net8.0|x64)␛[2147483640G(1m 31s) SkippedTest1␛[2147483640G(1m 31s) InProgressTest1␛[2147483640G(1m 31s) InProgressTest2␛[2147483643G(31s) InProgressTest3␛[2147483644G(1s) ␛[7F - ␛[J␛[93mskipped␛[m SkippedTest1␛[90m ␛[90m(10s 000ms)␛[m + ␛[J␛[33mskipped␛[m SkippedTest1␛[90m ␛[90m(10s 000ms)␛[m ␛[90m Standard output Hello! Error output Oh no! ␛[m - [␛[92m✓1␛[m/␛[91mx0␛[m/␛[93m↓1␛[m] assembly.dll (net8.0|x64)␛[2147483640G(1m 31s) + [␛[32m✓1␛[m/␛[31mx0␛[m/␛[33m↓1␛[m] assembly.dll (net8.0|x64)␛[2147483640G(1m 31s) InProgressTest1␛[2147483640G(1m 31s) InProgressTest2␛[2147483643G(31s) InProgressTest3␛[2147483644G(1s) - + """; Assert.AreEqual(expected, ShowEscape(output)); diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/TestHost.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/TestHost.cs index 32352b750b..57f4880b4f 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/TestHost.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/TestHost.cs @@ -87,8 +87,10 @@ public async Task ExecuteAsync( .ExecuteAsync(async () => { CommandLine commandLine = new(); + // Disable ANSI rendering so tests have easier time parsing the output. + // Disable progress so tests don't mix progress with overall progress, and with test process output. int exitCode = await commandLine.RunAsyncAndReturnExitCodeAsync( - $"{FullName} {finalArguments}", + $"{FullName} --no-ansi --no-progress {finalArguments}", environmentVariables: environmentVariables, workingDirectory: null, cleanDefaultEnvironmentVariableIfCustomAreProvided: true,