Explore challenging intermittent nightly test failures

pwiz_tools/Skyline/Model/DiaUmpireDdaConverter.cs

@@ -111,6 +111,7 @@ public override bool Run(IProgressMonitor progressMonitor, IProgressStatus statu
 
                     // CONSIDER: read the file description to see what settings were used to generate the file;
                     // if the same settings were used, we can re-use the file, else regenerate
+/* temp removal - is this the trouble spot causing memory errors? TODO restore this?
                     if (MsDataFileImpl.IsValidFile(outputFilepath))
                     {
                         var outputFileConfig = DiaUmpire.Config.GetConfigFromDiaUmpireOutput(outputFilepath);
@@ -139,7 +140,8 @@ public override bool Run(IProgressMonitor progressMonitor, IProgressStatus statu
                             }
                         }
                     }
-
+end temp removal */
+
                     if (File.Exists(outputFilepath))
                         FileEx.SafeDelete(outputFilepath);
 

pwiz_tools/Skyline/Program.cs

@@ -96,6 +96,7 @@ public static bool SkylineOffscreen
         } // Set true to move Skyline windows offscreen.
 
         public static bool DemoMode { get; set; }                   // Set to true in demo mode (main window is full screen and pauses at screenshots)
+        public static bool ReportTutorialTestProgress { get; set; } // When True, write PauseForScreenshot messages to console even if offscreen and even if not pausing for screenshots
         public static bool NoVendorReaders { get; set; }            // Set true to avoid calling vendor readers.
         public static bool UseOriginalURLs { get; set; }            // Set true to use original URLs for downloading tools instead of our S3 copies
         public static bool IsPassZero { get { return NoVendorReaders; } }   // Currently the only time NoVendorReaders gets set is pass0

pwiz_tools/Skyline/SkylineTester/TabBuild.cs

@@ -196,15 +196,17 @@ public static bool CreateBuildCommands(
 
                 commandShell.Add("#@ Checking out {0} source files...\n", branchName);
                 commandShell.Add("# Checking out {0} source files...", branchName);
+
                 // Add the --progress flag for richer logging - git leaves out most progress info when it isn't writing to an actual terminal
+                // Add the "-c http.postBuffer=524288000" argument to help with clone on unstable networks
                 if (branchName.Contains("master"))
                 {
-                    commandShell.AddWithRetry("{0} clone {1} --recurse-submodules --progress {2}", git.Quote(), branchUrl.Quote(), buildRoot.Quote());
+                    commandShell.AddWithRetry("{0} -c http.postBuffer=524288000 clone {1} --recurse-submodules --progress {2}", git.Quote(), branchUrl.Quote(), buildRoot.Quote());
                 }
                 else
                 {
                     var branch = branchUrl.Split(new[] {"tree/"}, StringSplitOptions.None)[1];
-                    commandShell.AddWithRetry("{0} clone {1} --recurse-submodules --progress -b {2} {3}", git.Quote(), GetMasterUrl().Quote(), branch.Quote(), buildRoot.Quote());
+                    commandShell.AddWithRetry("{0} -c http.postBuffer=524288000 clone {1} --recurse-submodules --progress -b {2} {3}", git.Quote(), GetMasterUrl().Quote(), branch.Quote(), buildRoot.Quote());
                 }
             }
 
@@ -214,7 +216,7 @@ public static bool CreateBuildCommands(
             foreach (int architecture in architectures)
             {
                 commandShell.Add("#@ Building Skyline {0} bit...\n", architecture);
-                commandShell.Add("{0} {1} {2} --i-agree-to-the-vendor-licenses {3} nolog",
+                commandShell.Add("{0} {1} {2} variant=release debug-symbols=on --i-agree-to-the-vendor-licenses {3} nolog",
                     Path.Combine(buildRoot, @"pwiz_tools\build-apps.bat").Quote(),
                     architecture,
                     runBuildTests ? "pwiz_tools/Skyline" : "pwiz_tools/Skyline//Skyline.exe",

pwiz_tools/Skyline/TestFunctional/AgilentCeOptimizationTest.cs

@@ -27,7 +27,7 @@
 namespace pwiz.SkylineTestFunctional
 {
     [TestClass]
-    public class AgilentCeOptimizationTest : AbstractFunctionalTest
+    public class AgilentCeOptimizationTest : AbstractFunctionalTestEx
     {
         [TestMethod]
         public void TestAgilentCeOptimization()
@@ -38,10 +38,7 @@ public void TestAgilentCeOptimization()
 
         protected override void DoTest()
         {
-            RunUI(() =>
-            { 
-                SkylineWindow.OpenFile(TestFilesDir.GetTestPath("AgilentCeTest.sky"));
-            });
+            OpenDocument(TestFilesDir.GetTestPath("AgilentCeTest.sky"));
             RunDlg<ImportResultsDlg>(SkylineWindow.ImportResults, importResults=>{
                 importResults.OptimizationName = ExportOptimize.CE;
                 importResults.NamedPathSets = new[]

pwiz_tools/Skyline/TestFunctional/PermuteIsotopeModificationsTest.cs

@@ -32,7 +32,7 @@ namespace pwiz.SkylineTestFunctional
     /// https://skyline.ms/announcements/home/support/thread.view?rowId=52220
     /// </summary>
     [TestClass]
-    public class PermuteIsotopeModificationsTest : AbstractFunctionalTest
+    public class PermuteIsotopeModificationsTest : AbstractFunctionalTestEx
     {
         [TestMethod]
         public void TestPermuteIsotopeModifications()
@@ -43,7 +43,7 @@ public void TestPermuteIsotopeModifications()
 
         protected override void DoTest()
         {
-            RunUI(()=>SkylineWindow.OpenFile(TestFilesDir.GetTestPath("PermuteIsotopeModificationsTest.sky")));
+            OpenDocument(TestFilesDir.GetTestPath("PermuteIsotopeModificationsTest.sky"));
             // This document contains two peptides
             Assert.AreEqual(2, SkylineWindow.Document.MoleculeCount);
             Assert.AreEqual(2, SkylineWindow.Document.MoleculeTransitionGroupCount);

pwiz_tools/Skyline/TestRunner/Program.cs

@@ -1444,6 +1444,26 @@ private static bool RunTestPasses(
                     runTests.Log("\r\n");
                 }
 
+                if (asNightly) 
+                {
+                    // Run procdump if installed
+                    var process = new Process();
+                    process.StartInfo.FileName = "procdump.exe";
+                    var dmpDirQuote = string.IsNullOrEmpty(dmpDir) ? string.Empty : $@" ""{dmpDir}""";
+                    process.StartInfo.Arguments = $@"-accepteula -ma -mk -h -64 TestRunner{dmpDirQuote}";
+                    runTests.Log($@"# Attempting to launch 'procdump.exe {process.StartInfo.Arguments}', to try to diagnose hangs in TestRunner\r\n");
+                    try
+                    {
+
+                        process.Start();
+                    }
+                    catch (Exception)
+                    {
+                        // ignored, presumably ProcDump is not installed    
+                        runTests.Log("# ProcDump did not launch. You may wish to install SysInternals ProcDump to try to diagnose hangs in TestRunner. It will be launched automatically if available.\r\n");
+                    }
+                }
+
                 if (commandLineArgs.ArgAsBool("clipboardcheck"))
                 {
                     runTests.TestContext.Properties["ClipboardCheck"] = "TestRunner clipboard check";

pwiz_tools/Skyline/TestTutorial/Ms1FullScanFilteringTutorial.cs

@@ -46,6 +46,7 @@
 using pwiz.Skyline.Util;
 using pwiz.Skyline.Util.Extensions;
 using ZedGraph;
+using pwiz.Skyline;
 
 namespace pwiz.SkylineTestTutorial
 {
@@ -58,6 +59,7 @@ public class Ms1FullScanFilteringTutorial : AbstractFunctionalTestEx
         [TestMethod, MinidumpLeakThreshold(15)]
         public void TestMs1Tutorial()
         {
+Program.ReportTutorialTestProgress = true; // Write screenshot info to console, trying to figure out where this hangs
             // Set true to look at tutorial screenshots.
 //            IsPauseForScreenShots = true;
 //            IsCoverShotMode = true;

pwiz_tools/Skyline/TestUtil/TestFunctional.cs

@@ -1308,6 +1308,12 @@ private void PauseForScreenShot(string description, int? pageNum, Type formType,
                 var form = TryWaitForOpenForm(formType);
                 Assert.IsNotNull(form);
             }
+
+            if (Program.ReportTutorialTestProgress)
+            {
+                Console.WriteLine($@"# tutorial test progress: {description}  - p. {pageNum}");
+            }
+
             if (Program.SkylineOffscreen)
                 return;
 

pwiz_tools/Skyline/Util/Statistics.cs

@@ -1342,24 +1342,34 @@ public static double QNthItem(IList<double> list, int elementIndex)
 
         private static void Split(IList<double> list, double value, ref int left, ref int right)
         {
-            // Left and right scan until the pointers cross
-            do
+            try
             {
-                while (list[left] < value)
-                    left++;
-                while (value < list[right])
-                    right--;
 
-                if (left <= right)
+                // Left and right scan until the pointers cross
+                do
                 {
-                    double temp = list[left];
-                    list[left] = list[right];
-                    list[right] = temp;
+                    while (list[left] < value)
+                        left++;
+                    while (value < list[right])
+                        right--;
 
-                    left++;
-                    right--;
-                }
-            } while (left <= right);
+                    if (left <= right)
+                    {
+                        double temp = list[left];
+                        list[left] = list[right];
+                        list[right] = temp;
+
+                        left++;
+                        right--;
+                    }
+                } while (left <= right);
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine(e);
+                Console.WriteLine($@"left={left} right={right} size={list.Count}");
+                throw;
+            }
         }
 
         /// <summary>

pwiz_tools/build-apps.bat

@@ -30,15 +30,9 @@ if "%ALL_ARGS: REGISTER=%" neq "%ALL_ARGS%" (
 )
 if "%ALL_ARGS: debug=%" neq "%ALL_ARGS%" (
     set OPTIMIZATION=debug
-    if "%ALL_ARGS: debug-symbols=%" == "%ALL_ARGS%" (
-        set ALL_ARGS=%ALL_ARGS: debug=%
-    )
 )
 if "%ALL_ARGS: DEBUG=%" neq "%ALL_ARGS%" (
     set OPTIMIZATION=debug
-    if "%ALL_ARGS: DEBUG-SYMBOLS=%" == "%ALL_ARGS%" (
-        set ALL_ARGS=%ALL_ARGS: DEBUG=%
-    )
 )
 if "%ALL_ARGS: nolog=%" neq "%ALL_ARGS%" (
     set NOLOG=1