diff --git a/src/native/watchdog/watchdog.cpp b/src/native/watchdog/watchdog.cpp index 1dc6f74fb06b6b..fec9674cd28366 100644 --- a/src/native/watchdog/watchdog.cpp +++ b/src/native/watchdog/watchdog.cpp @@ -35,8 +35,8 @@ int main(const int argc, const char *argv[]) return EXIT_FAILURE; } - const long timeout_sec = strtol(argv[1], nullptr, 10); - int exit_code = run_timed_process(timeout_sec * 1000L, argc-2, &argv[2]); + const long timeout_ms = strtol(argv[1], nullptr, 10); + int exit_code = run_timed_process(timeout_ms, argc-2, &argv[2]); printf("App Exit Code: %d\n", exit_code); return exit_code; diff --git a/src/tests/Common/CLRTest.Execute.Bash.targets b/src/tests/Common/CLRTest.Execute.Bash.targets index 5cc68ec20eff07..0623040dc12d80 100644 --- a/src/tests/Common/CLRTest.Execute.Bash.targets +++ b/src/tests/Common/CLRTest.Execute.Bash.targets @@ -290,7 +290,7 @@ fi "$CORE_ROOT/corerun" $(CoreRunArgs) ${__DotEnvArg} - "$CORE_ROOT/watchdog" 300 + "$CORE_ROOT/watchdog" $_WatcherTimeoutMilliseconds "%CORE_ROOT%\corerun.exe" $(CoreRunArgs) %__DotEnvArg% - "%CORE_ROOT%\watchdog.exe" 300 + "%CORE_ROOT%\watchdog.exe" %_WatcherTimeoutMilliseconds% workItemStats = File.ReadLines(statsCsvPath); + // If the final results log file is present, then we can assume everything + // went fine, and it's ready to go without any further processing. We just + // check the stats csv file to know how many tests were run, and display a + // brief summary of the work item. + + if (File.Exists(finalLogPath)) + { + Console.WriteLine($"[XUnitLogChecker]: Item '{wrapperName}' did" + + " complete successfully!"); + return SUCCESS; + } + + // If we're here, then that means we've got something to fix. + // First, read the stats csv file. If it doesn't exist, then we can + // assume something went very badly and will likely cause more issues + // later on, so we exit now. + + if (!File.Exists(statsCsvPath)) + { + Console.WriteLine("[XUnitLogChecker]: An error occurred. No stats csv" + + $" was found. The expected name would be '{statsCsvPath}'."); + return FAILURE; + } + + // Declaring the enumerable to contain the log lines first because we + // might not be able to read on the first try due to locked resources + // on Windows. We will retry for up to one minute when this case happens. + IEnumerable? workItemStats = TryReadFile(statsCsvPath); + + if (workItemStats is null) + { + Console.WriteLine("[XUnitLogChecker]: Timed out trying to read the" + + $" stats file '{statsCsvPath}'."); + return FAILURE; + } // The first value at the top of the csv represents the amount of tests // that were expected to be run. @@ -93,25 +129,17 @@ static int Main(string[] args) .Select(x => Int32.Parse(x)) .ToArray(); - // If the final results log file is present, then we can assume everything - // went fine, and it's ready to go without any further processing. We just - // check the stats csv file to know how many tests were run, and display a - // brief summary of the work item. - - if (File.Exists(finalLogPath)) - { - Console.WriteLine($"[XUnitLogChecker]: Item '{wrapperName}' did" - + " complete successfully!"); - - PrintWorkItemSummary(numExpectedTests, workItemEndStatus); - return SUCCESS; - } - // Here goes the main core of the XUnit Log Checker :) Console.WriteLine($"[XUnitLogChecker]: Item '{wrapperName}' did not" + " finish running. Checking and fixing the log..."); - FixTheXml(tempLogPath); + bool success = FixTheXml(tempLogPath); + if (!success) + { + Console.WriteLine("[XUnitLogChecker]: Fixing the log failed."); + return FAILURE; + } + PrintWorkItemSummary(numExpectedTests, workItemEndStatus); // Rename the temp log to the final log, so that Helix can use it without @@ -120,6 +148,33 @@ static int Main(string[] args) return SUCCESS; } + static IEnumerable TryReadFile(string filePath) + { + IEnumerable? fileContents = null; + Stopwatch fileReadStopwatch = Stopwatch.StartNew(); + + while (fileReadStopwatch.ElapsedMilliseconds < 60000) + { + // We were able to read the file, so we can finish this loop. + if (fileContents is not null) + break; + + try + { + fileContents = File.ReadLines(filePath); + } + catch (IOException ioEx) + { + Console.WriteLine("[XUnitLogChecker]: Could not read the" + + $" file {filePath}. Retrying..."); + + // Give it a couple seconds before trying again. + Thread.Sleep(2000); + } + } + return fileContents; + } + static void PrintWorkItemSummary(int numExpectedTests, int[] workItemEndStatus) { Console.WriteLine($"\n{workItemEndStatus[0]}/{numExpectedTests} tests run."); @@ -128,17 +183,25 @@ static void PrintWorkItemSummary(int numExpectedTests, int[] workItemEndStatus) Console.WriteLine($"* {workItemEndStatus[3]} tests skipped.\n"); } - static void FixTheXml(string xFile) + static bool FixTheXml(string xFile) { var tags = new Stack(); string tagText = string.Empty; + IEnumerable? logLines = TryReadFile(xFile); + + if (logLines is null) + { + Console.WriteLine("[XUnitLogChecker]: Timed out trying to read the" + + $" log file '{xFile}'."); + return false; + } // Flag to ensure we don't process tag-like-looking things while reading through // a test's output. bool inOutput = false; bool inCData = false; - foreach (string line in File.ReadLines(xFile)) + foreach (string line in logLines) { // Get all XML tags found in the current line and sort them in order // of appearance. @@ -212,6 +275,7 @@ static void FixTheXml(string xFile) if (tags.Count == 0) { Console.WriteLine($"[XUnitLogChecker]: XUnit log file '{xFile}' was A-OK!"); + return true; } // Write the missing closings for all the opened tags we found. @@ -226,6 +290,7 @@ static void FixTheXml(string xFile) } Console.WriteLine("[XUnitLogChecker]: XUnit log file has been fixed!"); + return true; } static TagResult[] GetOrderedTagMatches(Match[] openingTags, Match[] closingTags)