FitScore class, now calculates plantwide scores either from stored da…

…ta or from a provided simualted dataset
equinor · Dec 5, 2023 · 58a8f48 · 58a8f48
1 parent 50d1197
commit 58a8f48
Show file tree

Hide file tree

Showing 7 changed files with 196 additions and 29 deletions.
diff --git a/Dynamic/Identification/FitScore.cs b/Dynamic/Identification/FitScore.cs
@@ -1,13 +1,182 @@
 using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Reflection;
 using System.Text;
 
-namespace TimeSeriesAnalysis.Dynamic.Identification
+using TimeSeriesAnalysis.Dynamic;
+
+namespace TimeSeriesAnalysis
 {
     public class FitScore
     {
+
+
+        /// <summary>
+        /// Calculate a "fit score" between two signals. 
+        /// </summary>
+        /// <param name="meas"></param>
+        /// <param name="sim"></param>
+        /// <returns> a fit score that is maximum 100 percent, but can also go negative if fit is poor</returns>
+        public static double Calc(double[] meas, double[] sim)
+        {
+            if (meas == null)
+                return double.NaN;
+            if (sim == null)
+                return double.NaN;
+
+            double dev = Deviation(meas, sim);
+            double devFromSelf = DeviationFromAvg(meas);
+
+            double fitScore = 0;
+            if ( !Double.IsNaN(dev))
+            {
+                fitScore = (double)(1 - dev / devFromSelf) * 100;
+            }
+            else
+            {
+                fitScore = double.NaN;
+            }
+            return fitScore;
+        }
+
+        /// <summary>
+        /// Determines a score for how well a simulation fits with inputData. This can 
+        /// be compared over time to see any signs of degredation of the plant over time
+        /// 
+        /// </summary>
+        /// <param name="plantSimObj"></param>
+        /// <param name="inputData"></param>
+        /// <param name="simData"></param>
+        /// <returns></returns>
+        static public double GetPlantWideSimulated(PlantSimulator plantSimObj, TimeSeriesDataSet inputData, 
+            TimeSeriesDataSet simData)
+        {
+            const string disturbanceSignalPrefix = "_D";
+            List<double> fitScores = new List<double>();
+            foreach (var modelName in plantSimObj.modelDict.Keys)
+            {
+                double[] measY = null;
+                double[] simY = null;
+
+                var modelObj = plantSimObj.modelDict[modelName];
+                var outputName = modelObj.GetOutputID();
+                var outputIdentName = modelObj.GetOutputIdentID();
+                if (outputName == "" || outputName == null)
+                    continue;
+
+                if (plantSimObj.modelDict[modelName].GetProcessModelType() == ModelType.PID)
+                {
+                    if (inputData.ContainsSignal(outputName))
+                    {
+                        measY = inputData.GetValues(outputName);
+                    }
+
+                }
+                else //if (plantSimObj.modelDict[modelName].GetProcessModelType() == ModelType.SubProcess)
+                {
+                    if (outputIdentName != null)
+                    {
+                        if (inputData.ContainsSignal(outputIdentName))
+                        {
+                            measY = inputData.GetValues(outputIdentName);
+                        }
+                        else if (simData.ContainsSignal(outputIdentName))
+                        {
+                            measY = simData.GetValues(outputIdentName);
+                        }
+                    }
+                    // add in fit of process output, but only if "additive output signal" is not a
+                    // locally identified "_D_" signal, as then output matches 100%  always (any model error is put into disturbance signal as well)
+                    else if (modelObj.GetAdditiveInputIDs() != null)
+                    {
+                        if (!modelObj.GetAdditiveInputIDs()[0].StartsWith(disturbanceSignalPrefix))
+                        {
+                            measY = inputData.GetValues(outputName);
+                        }
+                    }
+                    else if (inputData.ContainsSignal(outputName))
+                    {
+                        measY = inputData.GetValues(outputName);
+                    }
+                }
+                if (simData.ContainsSignal(outputName))
+                { 
+                    simY = simData.GetValues(outputName);
+                }
+
+                if (measY != null && simY != null)
+                {
+                    var curFitScore = FitScore.Calc(measY, simY);
+
+                    if (curFitScore != double.NaN)
+                    {
+                        fitScores.Add(curFitScore);
+                    }
+                }
+
+            }
+            if (!fitScores.Any())
+                return double.NaN;
+            else
+            {
+                return fitScores.Average();
+            }
+
+        }
+
+
+
+        /// <summary>
+        /// Get the average stored fit-scores in the "paramters.Fitting" object, 
+        /// indicating how well the plant on average described/aligned with the dataset when it was fitted. 
+        /// </summary>
+        /// <param name="plantSimObj"></param>
+        /// <returns></returns>
+        static public double GetPlantWideStored(PlantSimulator plantSimObj)
+        {
+            List<double> fitScores = new List<double>();
+            foreach (var modelName in plantSimObj.modelDict.Keys)
+            {
+                double curFitScore = 0;
+                if (plantSimObj.modelDict[modelName].GetProcessModelType() == ModelType.PID)
+                {
+                    var modelObj = (PidModel)plantSimObj.modelDict[modelName];
+                    if (modelObj.pidParameters.Fitting != null)
+                    {
+                        curFitScore = modelObj.pidParameters.Fitting.FitScorePrc;
+                    }
+                }
+                else if (plantSimObj.modelDict[modelName].GetProcessModelType() == ModelType.SubProcess)
+                {
+                    var modelObj = (UnitModel)plantSimObj.modelDict[modelName];
+                    if (modelObj.modelParameters.Fitting != null)
+                    {
+                        curFitScore = modelObj.modelParameters.Fitting.FitScorePrc;
+                    }
+                }
+                if (curFitScore != 0)
+                {
+                    fitScores.Add(curFitScore);
+                }
+
+            }
+            if (!fitScores.Any())
+                return double.NaN;
+            else
+            {
+                return fitScores.Average();
+            }
+        }
+
+
+        /// <summary>
+        /// Get the absolute average deviation between the referene and model signals
+        /// </summary>
+        /// <param name="reference"></param>
+        /// <param name="model"></param>
+        /// <returns></returns>
         private static double Deviation(double[] reference, double[] model)
         {
             if (reference == null)
@@ -32,7 +201,7 @@ private static double Deviation(double[] reference, double[] model)
         }
 
         private static double DeviationFromAvg(double[] signal)
-        { 
+        {
             if (signal == null) return double.NaN;
             if (signal.Length == 0) return double.NaN;
 
@@ -48,29 +217,7 @@ private static double DeviationFromAvg(double[] signal)
             {
                 ret += Math.Abs(signal[i] - avg.Value);
             }
-            return ret/N;
-        }
-
-        public static double Calc(double[] meas, double[] sim)
-        {
-            if (meas == null)
-                return double.NaN;
-            if (sim == null)
-                return double.NaN;
-
-            double dev = Deviation(meas, sim);
-            double devFromSelf = DeviationFromAvg(meas);
-
-            double fitScore = 0;
-            if ( !Double.IsNaN(dev))
-            {
-                fitScore = (double)(1 - dev / devFromSelf) * 100;
-            }
-            else
-            {
-                fitScore = double.NaN;
-            }
-            return fitScore;
+            return ret / N;
         }
 
 

diff --git a/Dynamic/Identification/FittingInfo.cs b/Dynamic/Identification/FittingInfo.cs
@@ -2,7 +2,6 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
-using TimeSeriesAnalysis.Dynamic.Identification;
 using TimeSeriesAnalysis.Utility;
 
 namespace TimeSeriesAnalysis.Dynamic

diff --git a/Dynamic/Identification/PidIdentifier.cs b/Dynamic/Identification/PidIdentifier.cs
@@ -7,7 +7,6 @@
 using TimeSeriesAnalysis;
 using TimeSeriesAnalysis.Utility;
 using TimeSeriesAnalysis.Dynamic;
-using TimeSeriesAnalysis.Dynamic.Identification;
 
 namespace TimeSeriesAnalysis.Dynamic
 {
@@ -588,7 +587,7 @@ private double[] GetErrorTerm(UnitDataSet dataSet, PidFilter pidFilter)
             pidParam.Fitting.NFittingBadDataPoints = regressResults.NfittingBadDataPoints;
             pidParam.Fitting.RsqDiff = regressResults.Rsq;
             pidParam.Fitting.ObjFunValDiff = regressResults.ObjectiveFunctionValue;
-            pidParam.Fitting.FitScorePrc = FitScore.Calc(dataSet.U.GetColumn(0), U_sim.GetColumn(0));
+            pidParam.Fitting.FitScorePrc = SignificantDigits.Format(FitScore.Calc(dataSet.U.GetColumn(0), U_sim.GetColumn(0)), nDigits);
 
             pidParam.Fitting.ObjFunValAbs  = vec.SumOfSquareErr(dataSet.U.GetColumn(0), U_sim.GetColumn(0), 0);
             pidParam.Fitting.RsqAbs = vec.RSquared(dataSet.U.GetColumn(0), U_sim.GetColumn(0), null, 0) * 100;

diff --git a/Dynamic/Interfaces/ISimulateableModel.cs b/Dynamic/Interfaces/ISimulateableModel.cs
@@ -83,6 +83,13 @@ public interface ISimulatableModel
         /// <returns></returns>
         string GetOutputID();
 
+        /// <summary>
+        /// Get the ID of the "OutputIdent" signal
+        /// </summary>
+        /// <returns></returns>
+        string GetOutputIdentID();
+
+
         /// <summary>
         /// Set the output ID
         /// </summary>

diff --git a/Dynamic/Interfaces/ModelBaseClass.cs b/Dynamic/Interfaces/ModelBaseClass.cs
@@ -240,6 +240,18 @@ public virtual string GetOutputID()
                 return this.outputID;
         }
 
+
+        /// <summary>
+        /// returns the ID of the signal the output is identified against
+        /// </summary>
+        /// <returns> may return <c>null</c> if output is not set</returns>
+        public virtual string GetOutputIdentID()
+        {
+            return this.outputIdentID;
+        }
+
+
+
         /// <summary>
         /// Get the length of the output vector
         /// </summary>

diff --git a/TimeSeriesAnalysis.Tests/Tests/PlantSimulatorSISOTests.cs b/TimeSeriesAnalysis.Tests/Tests/PlantSimulatorSISOTests.cs
@@ -236,6 +236,9 @@ public static void CommonAsserts(TimeSeriesDataSet inputData,TimeSeriesDataSet s
                 Assert.IsTrue(firstTwoValuesDiff < 0.01, "system should start up in steady-state");
                 Assert.IsTrue(lastTwoValuesDiff < 0.01, "system should end up in stedy-state");
             }
+
+
+
             Assert.AreEqual(simData.GetLength(), simData.GetTimeStamps().Count(), "number of timestamps should match number of data points in sim");
             Assert.AreEqual(simData.GetTimeStamps().Last(), inputData.GetTimeStamps().Last(),"datasets should end at same timestamp");
 

diff --git a/TimeSeriesAnalysis.csproj b/TimeSeriesAnalysis.csproj
@@ -14,7 +14,7 @@
     <RunAnalyzersDuringLiveAnalysis>False</RunAnalyzersDuringLiveAnalysis>
     <RepositoryUrl>https://github.com/equinor/timeseriesanalysis</RepositoryUrl>
     <PackageReadmeFile>readme.md</PackageReadmeFile>
-    <Version>1.2.57</Version>
+    <Version>1.2.58</Version>
     <Company>Equinor</Company>
     <Authors>Equinor</Authors>
 	<IncludeSymbols>true</IncludeSymbols>