Retry up to 10 times when error requesting Koina predictions

pwiz_tools/Skyline/FileUI/PeptideSearch/SearchControl.cs

@@ -28,6 +28,7 @@
 using pwiz.Common.SystemUtil;
 using pwiz.Skyline.Alerts;
 using pwiz.Skyline.Util;
+using pwiz.Skyline.Util.Extensions;
 
 namespace pwiz.Skyline.FileUI.PeptideSearch
 {
@@ -142,8 +143,11 @@ protected void UpdateSearchEngineProgress(IProgressStatus status)
             }
 
             // look at the last 10 lines for the same message and if found do not relog the same message
-            if (_progressTextItems.Skip(Math.Max(0, _progressTextItems.Count - 10)).Any(entry => entry.Message == message))
+            if (Enumerable.Range(Math.Max(0, _progressTextItems.Count - 10), Math.Min(10, _progressTextItems.Count))
+                .Any(i => _progressTextItems[i].Message == message))
+            {
                 return;
+            }
 
             if (message.EndsWith(@"%") && double.TryParse(message.Substring(0, message.Length - 1), out _))
             {
@@ -229,8 +233,9 @@ private void showTimestampsCheckbox_CheckedChanged(object sender, EventArgs e)
         private void RefreshProgressTextbox()
         {
             txtSearchProgress.Clear();
-            foreach (var entry in _progressTextItems)
-                txtSearchProgress.AppendText($@"{entry.ToString(showTimestampsCheckbox.Checked)}{Environment.NewLine}");
+            txtSearchProgress.AppendText(TextUtil.LineSeparate(_progressTextItems.Select(entry
+                => entry.ToString(showTimestampsCheckbox.Checked))) + Environment.NewLine);
+
         }
 
         public string LogText => txtSearchProgress.Text;
@@ -247,7 +252,7 @@ public virtual UpdateProgressResponse UpdateProgress(IProgressStatus status)
             if (InvokeRequired)
                 Invoke(new MethodInvoker(() => UpdateProgress(status)));
             else
-                UpdateSearchEngineProgress(status.ChangeMessage(status.Message));
+                UpdateSearchEngineProgress(status);
 
             return UpdateProgressResponse.normal;
         }

pwiz_tools/Skyline/Model/Koina/KoinaResources.resx

@@ -160,4 +160,7 @@
   <data name="GraphSpectrum_UpdateUI_No_spectral_library_match" xml:space="preserve">
     <value>No spectral library match</value>
   </data>
+  <data name="KoinaModel_PredictBatches_Error___0__retrying__1_____2__" xml:space="preserve">
+    <value>Error: {0} retrying {1}/{2}.</value>
+  </data>
 </root>

pwiz_tools/Skyline/Model/Koina/Models/KoinaIntensityModel.cs

@@ -19,6 +19,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Text;
@@ -28,6 +29,7 @@
 using pwiz.Skyline.Model.DocSettings;
 using pwiz.Skyline.Properties;
 using pwiz.Skyline.Util;
+using pwiz.Skyline.Util.Extensions;
 using static Inference.ModelInferRequest.Types;
 
 namespace pwiz.Skyline.Model.Koina.Models
@@ -271,6 +273,12 @@ public KoinaPrecursorInput(string peptideSequence, int precursorCharge, float no
                 public int PrecursorCharge { get; }
 
                 public float NormalizedCollisionEnergy { get; }
+
+                public override string ToString()
+                {
+                    return TextUtil.SpaceSeparate(PeptideSequence + Transition.GetChargeIndicator(PrecursorCharge),
+                        TextUtil.ColonSeparate(@"NCE", NormalizedCollisionEnergy.ToString(CultureInfo.CurrentCulture)));
+                }
             }
         }
 

pwiz_tools/Skyline/Model/Koina/Models/KoinaModel.cs

@@ -411,15 +411,39 @@ public TKoinaOut PredictBatches(GRPCInferenceService.GRPCInferenceServiceClient
             var koinaOutput = new List<TKoinaOut>();
             for (int i = 0; i < inputsList.Count; ++i)
                 koinaOutput.Add(null);
-
+            int retryCount = Settings.Default.KoinaRetryCount;
             ParallelEx.For(0, inputsList.Count, batchIndex =>
             {
                 var koinaIn = inputsList[batchIndex];
 
                 if (progressMonitor.UpdateProgress(localProgressStatus) == UpdateProgressResponse.cancel)
                     return;
-
-                koinaOutput[batchIndex] = Predict(predictionClient, koinaIn, token);
+                var koinaExceptions = new List<KoinaException>();
+                for (int attempt = 0;; attempt++)
+                {
+                    try
+                    {
+                        koinaOutput[batchIndex] = Predict(predictionClient, koinaIn, token);
+                        break;
+                    }
+                    catch (KoinaException koinaException)
+                    {
+                        koinaExceptions.Add(koinaException);
+                        if (attempt >= retryCount)
+                        {
+                            throw new AggregateException(koinaExceptions);
+                        }
+                        lock (progressMonitor)
+                        {
+                            string message = string.Format(
+                                KoinaResources.KoinaModel_PredictBatches_Error___0__retrying__1_____2__,
+                                koinaException.Message,
+                                attempt + 1, retryCount);
+                            localProgressStatus = localProgressStatus.ChangeMessage(message);
+                            progressMonitor.UpdateProgress(localProgressStatus);
+                        }
+                    }
+                }
 
                 lock(progressMonitor)
                 {
@@ -752,7 +776,7 @@ public static IEnumerable<IEnumerable<T>> EnumerateBatches<T>(IList<T> data, int
             while (dataIndex < data.Count)
             {
                 var size = Math.Min(data.Count - dataIndex, batchSize);
-                yield return data.Skip(dataIndex).Take(size);
+                yield return Enumerable.Range(dataIndex, size).Select(i => data[i]);
                 dataIndex += size;
             }
         }

pwiz_tools/Skyline/Model/Koina/Models/KoinaRetentionTimeModel.cs

@@ -211,6 +211,10 @@ public KoinaPeptideInput(string peptideSequence)
                 }
 
                 public string PeptideSequence { get; private set; }
+                public override string ToString()
+                {
+                    return PeptideSequence;
+                }
             }
         }
 

pwiz_tools/Skyline/Properties/Settings.settings

@@ -881,5 +881,8 @@
     <Setting Name="RelativeAbundanceLogScale" Type="System.Boolean" Scope="User">
       <Value Profile="(Default)">True</Value>
     </Setting>
+    <Setting Name="KoinaRetryCount" Type="System.Int32" Scope="Application">
+      <Value Profile="(Default)">10</Value>
+    </Setting>
   </Settings>
 </SettingsFile>

pwiz_tools/Skyline/app.config

@@ -856,10 +856,10 @@
                 <value>False</value>
             </setting>
             <setting name="FeatureFindingMinIntensityPPM" serializeAs="String">
-                <value>100.0</value>
+                <value>10.0</value>
             </setting>
             <setting name="FeatureFindingMinIdotP" serializeAs="String">
-                <value>0.9</value>
+                <value>0.98</value>
             </setting>
             <setting name="FeatureFindingSignalToNoise" serializeAs="String">
                 <value>3</value>
@@ -890,6 +890,9 @@
             <setting name="InstalledVersion" serializeAs="String">
                 <value />
             </setting>
+            <setting name="KoinaRetryCount" serializeAs="String">
+                <value>10</value>
+            </setting>
         </pwiz.Skyline.Properties.Settings>
     </applicationSettings>
   <system.data>