From 602e8920aa0378d6585d2904726c1cbc121c58a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnar=20Sk=C3=A5lheim?= Date: Thu, 30 Nov 2023 14:58:42 +0100 Subject: [PATCH] Simple skeleton for GainSchedIdentify --- Dynamic/GainScheduling/GainSchedDataSet.cs | 396 ++++++ Dynamic/GainScheduling/GainSchedSimulator.cs | 281 ++++ Dynamic/GainScheduling/GainSchedWarnings.cs | 20 + Dynamic/Identification/GainSchedIdentifier.cs | 1194 +++++++++++++++++ .../GainSchedTimeDelayIdentifier.cs | 245 ++++ .../Identification/GainScheddentWarings.cs | 96 ++ Dynamic/SimulatableModels/GainSchedModel.cs | 8 +- .../SimulatableModels/GainSchedParameters.cs | 8 +- Dynamic/UnitSimulator/ModelType.cs | 2 +- .../Tests/GainSchedSimulatorTests.cs | 142 ++ .../Tests/PlantSimulatorMISOTests.cs | 34 +- 11 files changed, 2400 insertions(+), 26 deletions(-) create mode 100644 Dynamic/GainScheduling/GainSchedDataSet.cs create mode 100644 Dynamic/GainScheduling/GainSchedSimulator.cs create mode 100644 Dynamic/GainScheduling/GainSchedWarnings.cs create mode 100644 Dynamic/Identification/GainSchedIdentifier.cs create mode 100644 Dynamic/Identification/GainSchedTimeDelayIdentifier.cs create mode 100644 Dynamic/Identification/GainScheddentWarings.cs create mode 100644 TimeSeriesAnalysis.Tests/Tests/GainSchedSimulatorTests.cs diff --git a/Dynamic/GainScheduling/GainSchedDataSet.cs b/Dynamic/GainScheduling/GainSchedDataSet.cs new file mode 100644 index 00000000..92ca8df3 --- /dev/null +++ b/Dynamic/GainScheduling/GainSchedDataSet.cs @@ -0,0 +1,396 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using TimeSeriesAnalysis.Utility; + +namespace TimeSeriesAnalysis.Dynamic +{ + /// + /// The data for a porition of a process, containg only one output and one or multiple inputs that influence it + /// + public class GainSchedDataSet + { + /// + /// list of warings during identification + /// + public List Warnings { get; set; } + /// + /// Name + /// + public string ProcessName { get; } + /// + /// Timestamps + /// + public DateTime[] Times { get; set; } + /// + /// Output Y (measured) + /// + public double[] Y_meas { get; set; } + /// + /// Output Y (simulated) + /// + public double[] Y_sim { get; set; } + + /// + /// Input U(simulated) - in the case of PID-control + /// + public double[,] U_sim { get; set; } + + /// + /// Setpoint - (if sub-process includes a PID-controller) + /// + public double[] Y_setpoint { get; set; } = null; + + /// + /// Additve output disturbance D (Y = X+ D) + /// + public double[] D { get; set; } + + /// + /// Input U (given) + /// + public double[,] U { get; set; } + + /// + /// Indices that are ignored in Y during fitting. + /// + public List IndicesToIgnore = null; + + /// + /// Some systems for storing data do not support "NaN", but instead some other magic + /// value is reserved for indicating that a value is bad or missing. + /// + public double BadDataID { get; set; } = -9999; + + + /// + /// Constructor for data set without inputs - for "autonomous" processes such as sinusoids, + /// rand walks or other disturbancs. + /// + /// optional internal name of dataset + public GainSchedDataSet(string name = null) + { + this.Warnings = new List(); + this.Y_meas = null; + this.U = null; + this.ProcessName = name; + } + + /// + /// Create a copy of an existing data set + /// + /// + public GainSchedDataSet(GainSchedDataSet otherDataSet) + { + this.ProcessName = otherDataSet.ProcessName + "copy"; + + if (otherDataSet.Y_meas == null) + this.Y_meas = null; + else + this.Y_meas = otherDataSet.Y_meas.Clone() as double[]; + if (otherDataSet.Y_setpoint == null) + this.Y_setpoint = null; + else + this.Y_setpoint = otherDataSet.Y_setpoint.Clone() as double[]; + + if (otherDataSet.Y_sim == null) + { + this.Y_sim = null; + } + else + { + this.Y_sim = otherDataSet.Y_sim.Clone() as double[]; + } + if (otherDataSet.U == null) + this.U = null; + else + this.U = otherDataSet.U.Clone() as double[,]; + if (otherDataSet.U_sim == null) + this.U_sim = null; + else + this.U_sim = otherDataSet.U_sim.Clone() as double[,]; + if (otherDataSet.Times == null) + this.Times = null; + else + this.Times = otherDataSet.Times.Clone() as DateTime[]; + if (otherDataSet.D == null) + this.D = null; + else + this.D = otherDataSet.D.Clone() as double[]; + + if (otherDataSet.IndicesToIgnore == null) + this.IndicesToIgnore = null; + else + this.IndicesToIgnore = new List(otherDataSet.IndicesToIgnore); + + this.BadDataID = otherDataSet.BadDataID; + } + + /// + /// Create a downsampled copy of an existing data set + /// + /// + /// factor by which to downsample the original dataset + public GainSchedDataSet(GainSchedDataSet originalDataSet, int downsampleFactor) + { + this.ProcessName = originalDataSet.ProcessName + "downsampledFactor" + downsampleFactor; + + this.Y_meas = Vec.Downsample(originalDataSet.Y_meas, downsampleFactor); + this.Y_setpoint = Vec.Downsample(originalDataSet.Y_setpoint, downsampleFactor); + this.Y_sim = Vec.Downsample(originalDataSet.Y_sim, downsampleFactor); + this.U = Array2D.Downsample(originalDataSet.U, downsampleFactor); + this.U_sim = Array2D.Downsample(originalDataSet.U_sim, downsampleFactor); + this.Times = Vec.Downsample(originalDataSet.Times, downsampleFactor); + } + + public int GetNumDataPoints () + { + if (U != null) + return U.GetNRows(); + else if (Times != null) + return Times.Length; + else if (Y_meas != null) + return Y_meas.Length; + else if (Y_setpoint!= null) + return Y_setpoint.Length; + else + return 0; + } + + /// Tags indices to be removed if either of the output is outside the range defined by + /// [Y_min,Y_max], an input is outside [u_min, umax] or if any data matches badDataId + /// + public void DetermineIndicesToIgnore(FittingSpecs fittingSpecs) + { + if (fittingSpecs == null) + { + return; + } + var newIndToExclude = new List(); + var vec = new Vec(); + + // find values below minimum for each input + if (fittingSpecs.Y_min_fit.HasValue) + { + if (!Double.IsNaN(fittingSpecs.Y_min_fit.Value) && fittingSpecs.Y_min_fit.Value != BadDataID + && !Double.IsNegativeInfinity(fittingSpecs.Y_min_fit.Value)) + { + var indices = + vec.FindValues(Y_meas, fittingSpecs.Y_min_fit.Value, VectorFindValueType.SmallerThan, IndicesToIgnore); + newIndToExclude.AddRange(indices); + } + } + if (fittingSpecs.Y_max_fit.HasValue) + { + if (!Double.IsNaN(fittingSpecs.Y_max_fit.Value) && fittingSpecs.Y_max_fit.Value != BadDataID + && !Double.IsPositiveInfinity(fittingSpecs.Y_max_fit.Value)) + { + var indices = + vec.FindValues(Y_meas, fittingSpecs.Y_max_fit.Value, VectorFindValueType.BiggerThan, IndicesToIgnore); + newIndToExclude.AddRange(indices); + } + } + // find values below minimum for each input + if (fittingSpecs.U_min_fit != null) + { + for (int idx = 0; idx < Math.Min(fittingSpecs.U_min_fit.Length, U.GetNColumns()); idx++) + { + if (Double.IsNaN(fittingSpecs.U_min_fit[idx]) || fittingSpecs.U_min_fit[idx] == BadDataID + || Double.IsNegativeInfinity(fittingSpecs.U_min_fit[idx])) + continue; + var indices = + vec.FindValues(U.GetColumn(idx), fittingSpecs.U_min_fit[idx], VectorFindValueType.SmallerThan, IndicesToIgnore); + newIndToExclude.AddRange(indices); + } + } + if (fittingSpecs.U_max_fit != null) + { + for (int idx = 0; idx < Math.Min(fittingSpecs.U_max_fit.Length, U.GetNColumns()); idx++) + { + if (Double.IsNaN(fittingSpecs.U_max_fit[idx]) || fittingSpecs.U_max_fit[idx] == BadDataID + || Double.IsNegativeInfinity(fittingSpecs.U_max_fit[idx])) + continue; + var indices = + vec.FindValues(U.GetColumn(idx), fittingSpecs.U_max_fit[idx], + VectorFindValueType.BiggerThan, IndicesToIgnore); + newIndToExclude.AddRange(indices); + } + } + if (newIndToExclude.Count > 0) + { + var result = Vec.Sort(newIndToExclude.ToArray(), VectorSortType.Ascending); + newIndToExclude = result.ToList(); + var newIndToExcludeDistinct = newIndToExclude.Distinct(); + newIndToExclude = newIndToExcludeDistinct.ToList(); + } + + if (IndicesToIgnore != null) + { + if (newIndToExclude.Count > 0) + { + IndicesToIgnore.AddRange(newIndToExclude); + } + } + else + { + IndicesToIgnore = newIndToExclude; + } + } + + + /// + /// Tags indices to be removed if either of the inputs are outside the range defined by + /// [uMinFit,uMaxFit]. + /// + /// uMinFit,uMaxFit may include NaN or BadDataID for values if no max/min applies to the specific input + /// + /// vector of minimum values for each element in U + /// vector of maximum values for each element in U + public void SetInputUFitMaxAndMin(double[] uMinFit, double[] uMaxFit) + { + if ((uMinFit == null && uMaxFit == null) || this.U == null) + return; + + var newIndToExclude = new List(); + var vec = new Vec(); + // find values below minimum for each input + if (uMinFit != null) + { + for (int idx = 0; idx < Math.Min(uMinFit.Length,U.GetNColumns()); idx++) + { + if (Double.IsNaN(uMinFit[idx]) || uMinFit[idx] == BadDataID || Double.IsNegativeInfinity(uMinFit[idx])) + continue; + var indices = + vec.FindValues(U.GetColumn(idx), uMinFit[idx], VectorFindValueType.SmallerThan, IndicesToIgnore); + newIndToExclude.AddRange(indices); + } + } + if (uMaxFit != null) + { + for (int idx = 0; idx < Math.Min(uMaxFit.Length, U.GetNColumns()); idx++) + { + if (Double.IsNaN(uMaxFit[idx]) || uMaxFit[idx] == BadDataID || Double.IsNegativeInfinity(uMaxFit[idx])) + continue; + var indices = + vec.FindValues(U.GetColumn(idx), uMaxFit[idx], VectorFindValueType.BiggerThan, IndicesToIgnore); + newIndToExclude.AddRange(indices); + } + } + if (newIndToExclude.Count > 0) + { + var result = Vec.Sort(newIndToExclude.ToArray(), VectorSortType.Ascending); + newIndToExclude = result.ToList(); + var newIndToExcludeDistinct = newIndToExclude.Distinct(); + newIndToExclude = newIndToExcludeDistinct.ToList(); + } + + if (IndicesToIgnore != null) + { + if (newIndToExclude.Count > 0) + { + IndicesToIgnore.AddRange(newIndToExclude); + } + } + IndicesToIgnore = newIndToExclude; + } + + /// + /// Gets the time between samples in seconds, returns zero if times are not set + /// + /// + public double GetTimeBase() + { + if (Times != null) + { + if (Times.Length > 2) + return (Times.Last() - Times.First()).TotalSeconds / (Times.Length - 1); + else + return 0; + } + return 0; + } + public void CreateTimeStamps(double timeBase_s, DateTime? t0 = null) + { + if (t0 == null) + { + t0 = new DateTime(2010, 1, 1);//intended for testing + } + + var times = new List(); + //times.Add(t0.Value); + DateTime time = t0.Value; + for (int i = 0; i < GetNumDataPoints(); i++) + { + times.Add(time); + time = time.AddSeconds(timeBase_s); + } + this.Times = times.ToArray(); + } + + /// + /// Create a dataset for single-input system from two signals that have separate but overlapping + /// time-series(each given as value-date tuples) + /// + /// tuple of values and dates describing u + /// tuple of values and dates describing y + /// name of dataset + public GainSchedDataSet((double[], DateTime[]) u, (double[], DateTime[]) y_meas, string name = null/*, + int? timeBase_s=null*/) + { + var jointTime = Vec.Intersect(u.Item2.ToList(),y_meas.Item2.ToList()); + var indU = Vec.GetIndicesOfValues(u.Item2.ToList(), jointTime); + var indY = Vec.GetIndicesOfValues(y_meas.Item2.ToList(), jointTime); + this.Times = jointTime.ToArray(); + + this.Y_meas = Vec.GetValuesAtIndices(y_meas.Item1,indY); + var newU = Vec.GetValuesAtIndices(u.Item1, indU); + this.U = Array2D.CreateFromList(new List { newU }); + this.ProcessName = name; + } + + /// + /// Get the time spanned by the dataset + /// + /// The time spanned by the dataset, or null if times are not set + public TimeSpan GetTimeSpan() + { + if (this.Times == null) + { + return new TimeSpan(); + } + if (this.Times.Length == 0) + { + return new TimeSpan(); + } + return Times.Last() - Times.First(); + } + /// + /// Get the average value of each input in the dataset. + /// This is useful when defining model local around a working point. + /// + /// an array of averages, each corrsponding to one column of U. + /// Returns null if it was not possible to calculate averages + public double[] GetAverageU() + { + if (U == null) + { + return null; + } + List averages = new List(); + + for (int i = 0; i < U.GetNColumns(); i++) + { + double? avg = (new Vec(BadDataID)).Mean(U.GetColumn(i)); + if (!avg.HasValue) + return null; + averages.Add(avg.Value); + } + return averages.ToArray(); + } + + + } +} diff --git a/Dynamic/GainScheduling/GainSchedSimulator.cs b/Dynamic/GainScheduling/GainSchedSimulator.cs new file mode 100644 index 00000000..5d3a612e --- /dev/null +++ b/Dynamic/GainScheduling/GainSchedSimulator.cs @@ -0,0 +1,281 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using TimeSeriesAnalysis; +using TimeSeriesAnalysis.Utility; +using System.Diagnostics; + + + +namespace TimeSeriesAnalysis.Dynamic +{ + /// + /// Stand-alone simulation of any ISimulatableModel model. + /// + /// + public class GainSchedSimulator + { + GainSchedModel model; + /// + /// Constructor + /// + /// + public GainSchedSimulator(GainSchedModel model) + { + this.model = model; + } + + /// + /// Simulation is written to ymeas instead of ysim. This is useful when creating generic datasets for + /// testing/test driven development. + /// + /// + /// optionally adds noise to the "measured" y (for testing purposes) + public void SimulateYmeas(ref GainSchedDataSet processDataSet, double noiseAmplitude=0) + { + Simulate(ref processDataSet,true); + if (noiseAmplitude > 0) + { + // use a specific seed here, to avoid potential issues with "random GainSched tests" and not-repeatable + // errors. + Random rand = new Random(1232); + + for (int k = 0; k < processDataSet.GetNumDataPoints(); k++) + { + processDataSet.Y_meas[k] += (rand.NextDouble()-0.5)*2* noiseAmplitude; + } + } + } + + + /// + /// Co-simulate a process model and pid-controller(both Y_sim and U_sim) + /// + /// the + /// the process will read the .Y_set and .Times and + /// possibly .Dc>, + /// and write simulated inputs to U_sim and Y_sim + /// pid-controller looks at e[k-1] if true, otherwise e[k] + /// write data to processDataSet.Y_meas + /// instead of processDataSet.Y_sim + /// Returns true if able to simulate, otherwise false (simulation is written into processDataSet ) + public bool CoSimulate + ( PidModel pid, ref GainSchedDataSet processDataSet, bool pidActsOnPastStep=true, bool writeResultToYmeasInsteadOfYsim = false) + { + processDataSet.Y_sim = null; + processDataSet.U_sim = null; + + if (processDataSet.Y_setpoint == null) + { + return false; + } + if (processDataSet.Y_setpoint.Length == 0) + { + return false; + } + if (pid.GetModelParameters().Fitting != null) + { + if (pid.GetModelParameters().Fitting.WasAbleToIdentify == false) + { + return false; + } + } + if (model.GetModelParameters().Fitting != null) + { + if (model.GetModelParameters().Fitting.WasAbleToIdentify == false) + { + return false; + } + } + + int N = processDataSet.GetNumDataPoints(); + double[] Y = Vec.Fill(0, N); + double[] U = Vec.Fill(0, N); + double y0,u0,y, u; + + // initalize PID/model together + if (processDataSet.Y_meas != null) + { + y0 = processDataSet.Y_meas[0]; + } + else + { + y0 = processDataSet.Y_setpoint[0]; + } + double x0 = y0; + if (processDataSet.D != null) + { + x0 -= processDataSet.D[0]; + } + + // this assumes that the disturbance is zero? + u0 = model.GetSteadyStateInput(x0).Value; + double umax = pid.GetModelParameters().Scaling.GetUmax(); + double umin = pid.GetModelParameters().Scaling.GetUmin(); + + if (u0 >umin && u0(); + } + processDataSet.Warnings.Add(GainSchedWarnings.FailedToInitializeGainSchedModel); + Debug.WriteLine("Failed to initalize gain scheduling model."); + u = umin + (umax-umin)/2; + } + + double timeBase_s = processDataSet.GetTimeBase(); + double y_prev = y0; + + for (int rowIdx = 0; rowIdx < N; rowIdx++) + { + if (Double.IsNaN(u)) + { + return false; + } + + if (pidActsOnPastStep) + { + double[] pidInputs = new double[] { y_prev, processDataSet.Y_setpoint[Math.Max(0,rowIdx-1)] }; + u = pid.Iterate(pidInputs, timeBase_s, processDataSet.BadDataID)[0]; + double x = model.Iterate(new double[] { u }, timeBase_s, processDataSet.BadDataID)[0]; + y = x; + if (processDataSet.D != null) + { + y += processDataSet.D[rowIdx]; + } + } + else + { // pid acts on current step + double x = model.Iterate(new double[] { u }, timeBase_s, processDataSet.BadDataID)[0]; + y = x; + if (processDataSet.D != null) + { + y += processDataSet.D[rowIdx]; + } + double[] pidInputs = new double[] { y, processDataSet.Y_setpoint[rowIdx] }; + u = pid.Iterate(pidInputs, timeBase_s, processDataSet.BadDataID)[0]; + } + + if (Double.IsNaN(u)) + { + Debug.WriteLine("pid.iterate returned NaN!"); + } + y_prev = y; + Y[rowIdx] = y; + U[rowIdx] = u; + } + processDataSet.U_sim = Array2D.CreateFromList(new List { U }); + if (writeResultToYmeasInsteadOfYsim) + { + processDataSet.Y_meas = Y; + } + else + { + processDataSet.Y_sim = Y; + } + return true ; + } + + /// + /// Simulates the output of the model based on the processDataSet.U provided, by default the output is + /// written back to processDataSet.Y_sim or processDataSet.Y_meas + /// By default this method adds to Y_sim o Y_meas if they already contain values. + /// + /// dataset containing the inputs U to be simulated + /// if true, output is written to processDataSet.ymeas instead of processDataSet.ysim + /// (default is false)if true, output overwrites any data in processDataSet.ymeas or processDataSet.ysim + /// Returns the simulate y if able to simulate,otherwise null + public double[] Simulate(ref GainSchedDataSet processDataSet, + bool writeResultToYmeasInsteadOfYsim = false, bool doOverwriteY=false) + { + int N = processDataSet.GetNumDataPoints(); ; + double[] output = Vec.Fill(0, N); + double timeBase_s = processDataSet.GetTimeBase(); + + if (processDataSet.D != null) + { + for (int rowIdx = 0; rowIdx < N; rowIdx++) + { + output[rowIdx] += processDataSet.D[rowIdx]; + } + } + if (processDataSet.U != null) + { + if (processDataSet.IndicesToIgnore == null) + { + for (int rowIdx = 0; rowIdx < N; rowIdx++) + { + output[rowIdx] += model.Iterate(processDataSet.U.GetRow(rowIdx), + timeBase_s, processDataSet.BadDataID)[0]; + } + } + else + { + int lastGoodRowIdx = -1; + for (int rowIdx = 0; rowIdx < N; rowIdx++) + { + if (processDataSet.IndicesToIgnore.Contains(rowIdx)) + { + if (lastGoodRowIdx >= 0) + { + output[rowIdx] += model.Iterate(processDataSet.U.GetRow(lastGoodRowIdx), + timeBase_s, processDataSet.BadDataID)[0]; + } + else + { + output[rowIdx] = processDataSet.BadDataID;//NB! not "pluss" + } + } + else + { + lastGoodRowIdx = rowIdx; + output[rowIdx] += model.Iterate(processDataSet.U.GetRow(rowIdx), + timeBase_s, processDataSet.BadDataID)[0]; + } + } + } + } + else + { + for (int rowIdx = 0; rowIdx < N; rowIdx++) + { + output[rowIdx] += model.Iterate(null, timeBase_s,processDataSet.BadDataID)[0]; + } + } + var vec = new Vec(processDataSet.BadDataID); + if (writeResultToYmeasInsteadOfYsim) + { + if (processDataSet.Y_meas == null|| doOverwriteY) + { + processDataSet.Y_meas = output; + } + else + { + processDataSet.Y_meas = vec.Add(processDataSet.Y_meas, output); + } + } + else + { + if (processDataSet.Y_sim == null || doOverwriteY) + { + processDataSet.Y_sim = output; + } + else + { + processDataSet.Y_sim = vec.Add(processDataSet.Y_sim, output); + } + } + return output; + } + } +} diff --git a/Dynamic/GainScheduling/GainSchedWarnings.cs b/Dynamic/GainScheduling/GainSchedWarnings.cs new file mode 100644 index 00000000..9e61598f --- /dev/null +++ b/Dynamic/GainScheduling/GainSchedWarnings.cs @@ -0,0 +1,20 @@ +namespace TimeSeriesAnalysis.Dynamic +{ + /// + /// Enum of recognized warning or error states during identification of process model + /// + public enum GainSchedWarnings + { + /// + /// No errors or warnings + /// + Nothing = 0, + + /// + /// Failed to initalize the gain scheduling model + /// + FailedToInitializeGainSchedModel = 1, + + + } +} diff --git a/Dynamic/Identification/GainSchedIdentifier.cs b/Dynamic/Identification/GainSchedIdentifier.cs new file mode 100644 index 00000000..f5fc23d3 --- /dev/null +++ b/Dynamic/Identification/GainSchedIdentifier.cs @@ -0,0 +1,1194 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using System.Diagnostics; + +using TimeSeriesAnalysis; +using TimeSeriesAnalysis.Utility; +using System.Net.Http.Headers; +using System.Data; +using System.Reflection; +using System.ComponentModel.Design; + +namespace TimeSeriesAnalysis.Dynamic +{ + /// + /// Identifier of the "Default" process model - a dynamic process model with time-constant, time-delay, + /// linear process gain and optional (nonlinear)curvature process gains. + /// + /// This model class is sufficent for real-world linear or weakly nonlinear dynamic systems, yet also introduces the fewest possible + /// parameters to describe the system in an attempt to avoiding over-fitting/over-parametrization + /// + /// + /// The "default" process model is identified using a linear-in-parameters paramterization(paramters a,b,c), so that it can be solved by linear regression + /// and identification should thus be both fast and stable. The issue with the parametriation(a,b,c) is that the meaning of each paramter is less + /// inutitive, for instance the time constant depends on a, but linear gain depends on both a and b, while curvature depends on a and c. + /// Looking at the unceratinty of each parameter to determine if the model should be dynamic or static or what the uncertainty of the time constant is, + /// is very hard, and this observation motivates re-paramtrizing the model after identification. + /// + /// + /// When assessing and simulating the model, parmaters are converted into more intuitive paramters "time constant", "linear gains" and "curvature gain" + /// which are a different parametrization. The GainSchedIdentifier, GainSchedModel and GainSchedParamters classes handle this transition seamlessly to the user. + /// Uncertainty is expressed in terms of this more intuitive parametrization, to allow for a more intuitive assessment of the parameters. + /// + /// + /// Another advantage of the paramterization, is that the model internally separates between stedy-state and transient state, you can at any instance + /// "turn off" dynamics and request the steady-state model output for the current input. This is useful if you have transient data that you want to + /// analyze in the steady-state, as you can then fit the model to all available data-points without having to select what data points you beleive are at + /// steady state, then you can disable dynamic terms to do a static analysis of the dynamic model. + /// + /// + /// Time-delay is an integer parameter, and finding the time-delay alongside continous paramters + /// turns the identification problem into a linear mixed-integer problem. + /// The time delay identification is done by splitting the time-delay estimation from continous parameter + /// identification, turning the solver into a sequential optimization solver. + /// This logic to re-run estimation for multiple time-delays and selecting the best estiamte of time delay + /// is deferred to + /// + /// + /// Since the aim is to identify transients/dynamics, the regression is done on model differences rather than absolute values + /// + /// + /// make static? + public class GainSchedIdentifier + { + const double obFunDiff_MinImprovement = 0.0001; + const double rSquaredDiff_MinImprovement = 0.001; + const int nDigits = 5;// number of significant digits in result parameters + const bool doGainSchedyUNorm = true;// if set true, then Unorm is always one + /// + /// Default Constructor + /// + public GainSchedIdentifier() + { + } + + /// + /// Identifies the "Default" process model that best fits the dataSet given + /// + /// The dataset containing the ymeas and U that is to be fitted against, + /// a new y_sim is also added + /// optional fitting specs object for tuning data + /// the identified model parameters and some information about the fit + public GainSchedModel Identify(ref GainSchedDataSet dataSet, + FittingSpecs fittingSpecs = null) + { + return Identify_Internal(ref dataSet, fittingSpecs); + } + + /// + /// Identifies the "Default" process model that best fits the dataSet given, but no time-constants + /// + /// The dataset containing the ymeas and U that is to be fitted against, + /// a new element y_sim is also added to this dataset + /// Optionally sets the local working point for the inputs + /// around which the model is to be designed(can be set to null) + /// normalizing paramter for u-u0 (its range) + /// the identified model parameters and some information about the fit + public GainSchedModel IdentifyStatic(ref GainSchedDataSet dataSet, FittingSpecs fittingSpecs = null) + { + return Identify_Internal(ref dataSet, fittingSpecs, false); + } + + /// + /// Identifies the "Default" process model that best fits the dataSet given, but disables curvatures + /// + /// The dataset containing the ymeas and U that is to be fitted against, + /// a new y_sim is also added + /// if set to false, estimation of time delays are disabled + /// the identified model parameters and some information about the fit + public GainSchedModel IdentifyLinear(ref GainSchedDataSet dataSet, FittingSpecs fittingSpecs, bool doEstimateTimeDelay = true) + { + return Identify_Internal(ref dataSet, fittingSpecs, true, false, doEstimateTimeDelay); + } + + /// + /// Identifies the "Default" process model based on differences y[k]-y[k-1] that best fits the dataSet given, but disables curvatures + /// + /// The dataset containing the ymeas and U that is to be fitted against, + /// a new y_sim is also added + /// if set to false, estimation of time delays are disabled + /// Optionally sets the local working point for the inputs + /// around which the model is to be designed(can be set to null) + /// normalizing paramter for u-u0 (its range) + /// the identified model parameters and some information about the fit + public GainSchedModel IdentifyLinearDiff(ref GainSchedDataSet dataSet, FittingSpecs fittingSpecs, bool doEstimateTimeDelay = true) + { + var diffDataSet = new GainSchedDataSet(dataSet); + ConvertDatasetToDiffForm(ref diffDataSet); + var model = Identify_Internal(ref diffDataSet, fittingSpecs, true, false, doEstimateTimeDelay); + + if (model.modelParameters.Fitting.WasAbleToIdentify) + { + var simulator = new GainSchedSimulator(model); + simulator.Simulate(ref dataSet, default, true);// overwrite any y_sim + model.SetFittedDataSet(dataSet); + } + return model; + + } + + + /// + /// Identifies the "Default" process model that best fits the dataSet given, but disables curvatures and time-constants + /// + /// The dataset containing the ymeas and U that is to be fitted against, + /// a new y_sim is also added + /// if set to false, modeling does not identify time-delays + /// Optionally sets the local working point for the inputs + /// around which the model is to be designed(can be set to null) + /// normalizing paramter for u-u0 (its range) + /// the identified model parameters and some information about the fit + public GainSchedModel IdentifyLinearAndStatic(ref GainSchedDataSet dataSet, FittingSpecs fittingSpecs, bool doEstimateTimeDelay = true) + { + return Identify_Internal(ref dataSet, fittingSpecs, false, false, doEstimateTimeDelay); + } + + /// + /// Identifies the process model that best fits the dataSet given by minimizing differences y[k]-y[k-1], but disables curvatures and time-constants. + /// + /// The dataset containing the ymeas and U that is to be fitted against, + /// a new y_sim is also added + /// if set to false, modeling does not identify time-delays + /// Optionally sets the local working point for the inputs + /// around which the model is to be designed(can be set to null) + /// normalizing paramter for u-u0 (its range) + /// the identified model parameters and some information about the fit + public GainSchedModel IdentifyLinearAndStaticDiff(ref GainSchedDataSet dataSet, FittingSpecs fittingSpecs, bool doEstimateTimeDelay = true) + { + ConvertDatasetToDiffForm(ref dataSet); + return Identify_Internal(ref dataSet, fittingSpecs, false, false, doEstimateTimeDelay); + } + + private void ConvertDatasetToDiffForm(ref GainSchedDataSet dataSet) + { + Vec vec = new Vec(dataSet.BadDataID); + double[] Y_meas_old = new double[dataSet.Y_meas.Length]; + dataSet.Y_meas.CopyTo(Y_meas_old, 0); + dataSet.Y_meas = vec.Diff(Y_meas_old); + + double[,] U_old = new double[dataSet.U.GetNRows(), dataSet.U.GetNColumns()]; + for (int colIdx = 0; colIdx < dataSet.U.GetNColumns(); colIdx++) + { + Matrix.ReplaceColumn(dataSet.U, colIdx, vec.Diff(dataSet.U.GetColumn(colIdx))); + } + } + + + /// + /// Identifies the "Default" process model that best fits the dataSet given + /// + /// The dataset containing the ymeas and U that is to be fitted against, + /// a new y_sim is also added + + /// the identified model parameters and some information about the fit + private GainSchedModel Identify_Internal(ref GainSchedDataSet dataSet, FittingSpecs fittingSpecs, + bool doUseDynamicModel = true, bool doEstimateCurvature = true, bool doEstimateTimeDelay = true) + { + var vec = new Vec(dataSet.BadDataID); + + dataSet.DetermineIndicesToIgnore(fittingSpecs); + + var constantInputInds = new List(); + var correlatedInputInds = new List(); + bool doNonzeroU0 = true;// should be: true + double FilterTc_s = 0;// experimental: by default set to zero. + bool assumeThatYkminusOneApproxXkminusOne = true;// by default this should be set to true + + + double[] u0 = null; + double[] uNorm = null; + + bool hasU0 = false; + if (fittingSpecs == null) + { + hasU0 = false; + } + else if (fittingSpecs.u0 == null) + { + hasU0 = false; + } + else + { + hasU0 = true; + u0 = fittingSpecs.u0; + } + + bool hasUNorm = false; + if (fittingSpecs == null) + { + hasUNorm = false; + } + else if (fittingSpecs.uNorm == null) + { + hasUNorm = false; + } + else + { + hasUNorm = true; + uNorm = fittingSpecs.uNorm; + } + + if (!hasU0) + { + u0 = Vec.Fill(dataSet.U.GetNColumns(), 0); + if (doNonzeroU0) + { + u0 = SignificantDigits.Format(dataSet.GetAverageU(), nDigits); + } + } + if (!hasUNorm) + { + uNorm = Vec.Fill(1, dataSet.U.GetNColumns()); + for (int k = 0; k < dataSet.U.GetNColumns(); k++) + { + var u = dataSet.U.GetColumn(k); + if (!doGainSchedyUNorm) + { + uNorm[k] = Math.Max(Math.Abs(vec.Max(u) - u0[k]), Math.Abs(vec.Min(u) - u0[k])); + if (Double.IsInfinity(uNorm[k])) + { + uNorm[k] = 1; + } + } + //uNorm[k] = Math.Max(Math.Abs(vec.Max(u)), Math.Abs(vec.Min(u))); + if (vec.Max(u) == vec.Min(u))// input is constant + { + constantInputInds.Add(k); + uNorm[k] = 1; + } + if (uNorm[k] == 0)// avoid div by zero + { + constantInputInds.Add(k); + uNorm[k] = 1; + } + } + uNorm = SignificantDigits.Format(uNorm, nDigits); + } + + TimeSpan span = dataSet.GetTimeSpan(); + double maxExpectedTc_s = span.TotalSeconds / 4; + GainSchedTimeDelayIdentifier processTimeDelayIdentifyObj = + new GainSchedTimeDelayIdentifier(dataSet.GetTimeBase(), maxExpectedTc_s); + + // logic for all curves off an all curves on, treated as special cases + bool[] allCurvesDisabled = new bool[u0.Count()]; + bool[] allCurvesEnabled = new bool[u0.Count()]; + for (int i = 0; i < u0.Count(); i++) + { + allCurvesDisabled[i] = false; + allCurvesEnabled[i] = true; + } + ///////////////////////////////////////////////////////////////// + // Try turning off the dynamic parts and just see the static model + // + var modelList = new List(); + int timeDelayIdx = 0; + GainSchedParameters modelParams_StaticAndNoCurvature = + EstimateProcessForAGivenTimeDelay + (timeDelayIdx, dataSet, false, allCurvesDisabled, + FilterTc_s, u0, uNorm, assumeThatYkminusOneApproxXkminusOne); + modelList.Add(modelParams_StaticAndNoCurvature); + ///////////////////////////////////////////////////////////////// + // BEGIN WHILE loop to model process for different time delays + bool continueIncreasingTimeDelayEst = true; + timeDelayIdx = 0; + GainSchedParameters modelParams = null; + while (continueIncreasingTimeDelayEst) + { + modelParams = null; + GainSchedParameters modelParams_noCurvature = + EstimateProcessForAGivenTimeDelay + (timeDelayIdx, dataSet, doUseDynamicModel, allCurvesDisabled, + FilterTc_s, u0, uNorm, assumeThatYkminusOneApproxXkminusOne); + modelList.Add(modelParams_noCurvature); + if (doEstimateCurvature && modelParams_noCurvature.Fitting.WasAbleToIdentify) + { + GainSchedParameters modelParams_allCurvature = + EstimateProcessForAGivenTimeDelay + (timeDelayIdx, dataSet, doUseDynamicModel, allCurvesEnabled, + FilterTc_s, u0, uNorm, assumeThatYkminusOneApproxXkminusOne); + + + // only try rest of curvature models if it seems like it will help + // this is done to save on processing in case of many inputs (3 inputs= 8 identification runs) + //if (modelParams_noCurvature.FittingObjFunVal -modelParams_allCurvature.FittingObjFunVal > fitMinImprovement) + if (modelParams_allCurvature.Fitting.WasAbleToIdentify && + modelParams_allCurvature.Fitting.RsqDiff - modelParams_noCurvature.Fitting.RsqDiff > rSquaredDiff_MinImprovement) + { + List allNonZeroCombinations = GetAllNonzeroBitArrays(u0.Count()); + foreach (bool[] curCurveEnabledConfig in allNonZeroCombinations) + { + GainSchedParameters modelParams_withCurvature = + EstimateProcessForAGivenTimeDelay + (timeDelayIdx, dataSet, doUseDynamicModel, curCurveEnabledConfig, + FilterTc_s, u0, uNorm, assumeThatYkminusOneApproxXkminusOne); + modelList.Add(modelParams_withCurvature); + } + modelList.Add(modelParams_allCurvature); + modelParams = ChooseBestModel(modelParams_noCurvature, modelList); + } + else + { + modelParams = modelParams_noCurvature; + } + } + else + { + modelParams = modelParams_noCurvature; + } + if (!continueIncreasingTimeDelayEst) + continue; + // logic to deal with while loop + timeDelayIdx++; + processTimeDelayIdentifyObj.AddRun(modelParams); + continueIncreasingTimeDelayEst = processTimeDelayIdentifyObj. + DetermineIfContinueIncreasingTimeDelay(timeDelayIdx); + // fail-to-safe + if (timeDelayIdx * dataSet.GetTimeBase() > maxExpectedTc_s) + { + modelParams.AddWarning(GainScheddentWarnings.TimeDelayAtMaximumConstraint); + continueIncreasingTimeDelayEst = false; + } + if (doEstimateTimeDelay == false) + continueIncreasingTimeDelayEst = false; + + // use for debugging + /* bool doDebugPlotting = false;//should normally be false. + if (doDebugPlotting) + { + var debugModel = new GainSchedModel(modelParams, dataSet); + var sim = new GainSchedSimulator(debugModel); + var y_sim = sim.Simulate(ref dataSet); + Plot.FromList(new List {y_sim,dataSet.Y_meas},new List { "y1=ysim", "y1=ymeas" }, + (int)dataSet.TimeBase_s, "iteration:"+ timeDelayIdx,default,"debug_it_" + timeDelayIdx); + }*/ + } + + // the the time delay which caused the smallest object function value + int bestTimeDelayIdx = processTimeDelayIdentifyObj.ChooseBestTimeDelay( + out List timeDelayWarnings); + GainSchedParameters modelParameters = + (GainSchedParameters)processTimeDelayIdentifyObj.GetRun(bestTimeDelayIdx); + // use static and no curvature model as fallback if more complex models failed + if (!modelParameters.Fitting.WasAbleToIdentify && modelParams_StaticAndNoCurvature.Fitting.WasAbleToIdentify) + { + modelParameters = modelParams_StaticAndNoCurvature; + timeDelayWarnings.Add(ProcessTimeDelayIdentWarnings.FallbackToLinearStaticModel); + } + // check if the static model is better than the dynamic model. + else if (modelParameters.Fitting.WasAbleToIdentify && modelParams_StaticAndNoCurvature.Fitting.WasAbleToIdentify) + { + + // on real-world data it is sometimes obsrevd that RsqAbs and ObjFunAbs is NaN. + // to be robust, check for nan and use any of the four different metrics in order of trust + + if (!Double.IsNaN(modelParams_StaticAndNoCurvature.Fitting.RsqAbs) && + !Double.IsNaN(modelParameters.Fitting.RsqAbs)) + { + if (modelParams_StaticAndNoCurvature.Fitting.RsqAbs > modelParameters.Fitting.RsqAbs) + modelParameters = modelParams_StaticAndNoCurvature; + } + else if (!Double.IsNaN(modelParams_StaticAndNoCurvature.Fitting.RsqDiff) && + !Double.IsNaN(modelParameters.Fitting.RsqDiff)) + { + if (modelParams_StaticAndNoCurvature.Fitting.RsqDiff > modelParameters.Fitting.RsqDiff) + modelParameters = modelParams_StaticAndNoCurvature; + } + else if (!Double.IsNaN(modelParams_StaticAndNoCurvature.Fitting.ObjFunValDiff) && + !Double.IsNaN(modelParameters.Fitting.ObjFunValDiff)) + { + if (modelParams_StaticAndNoCurvature.Fitting.ObjFunValDiff < modelParameters.Fitting.ObjFunValDiff) + modelParameters = modelParams_StaticAndNoCurvature; + } + else if (!Double.IsNaN(modelParams_StaticAndNoCurvature.Fitting.ObjFunValAbs) && + !Double.IsNaN(modelParameters.Fitting.ObjFunValAbs)) + { + if (modelParams_StaticAndNoCurvature.Fitting.ObjFunValAbs < modelParameters.Fitting.ObjFunValAbs) + modelParameters = modelParams_StaticAndNoCurvature; + } + + + } + modelParameters.TimeDelayEstimationWarnings = timeDelayWarnings; + + if (constantInputInds.Count > 0) + { + modelParameters.AddWarning(GainScheddentWarnings.ConstantInputU); + } + if (correlatedInputInds.Count > 0) + { + modelParameters.AddWarning(GainScheddentWarnings.CorrelatedInputsU); + } + + modelParameters.FittingSpecs = fittingSpecs; + + + // END While loop + ///////////////////////////////////////////////////////////////// + var model = new GainSchedModel(modelParameters, dataSet); + + // simulate + if (modelParameters.Fitting.WasAbleToIdentify) + { + var simulator = new GainSchedSimulator(model); + simulator.Simulate(ref dataSet, default, true);// overwrite any y_sim + model.SetFittedDataSet(dataSet); + } + + return model; + } + + + // for three inputs, return every combination of true false + // except false-false-false and true-true-true, (but the other six) + private List GetAllNonzeroBitArrays(int size) + { + List list = new List(); + + var ints = new List(); + for (int i = 1; i < Math.Max(size * size-1, 2-1); i++) + { + var boolArray = new bool[size]; + BitArray bitArray = new BitArray(new int[] { i }); + for (int k = 0; k < size; k++) + { + boolArray[k] = bitArray.Get(k); + } + list.Add(boolArray); + } + return list; + } + + private GainSchedParameters ChooseBestModel(GainSchedParameters fallbackModel, List allModels) + { + GainSchedParameters bestModel = fallbackModel; + // models will be arranged from least to most numbre of curvature terms + // in case of doubt, do not add in extra curvature that does not significantly improve the objective function + foreach (GainSchedParameters curModel in allModels) + { + // Rsquared: higher is better + double RsqFittingDiff_improvement = curModel.Fitting.RsqDiff - bestModel.Fitting.RsqDiff; + double RsqFittingAbs_improvement = curModel.Fitting.RsqAbs - bestModel.Fitting.RsqAbs; + // objective function: lower is better + double objFunDiff_improvement = bestModel.Fitting.ObjFunValDiff - curModel.Fitting.ObjFunValDiff;// positive if curmodel improves on the current best + double objFunAbs_improvement = bestModel.Fitting.ObjFunValAbs - curModel.Fitting.ObjFunValAbs;// positive if curmodel improves on the current best + + if (Double.IsNaN(RsqFittingAbs_improvement) || Double.IsNaN(objFunAbs_improvement)) + { + if (objFunDiff_improvement >= obFunDiff_MinImprovement && + RsqFittingDiff_improvement >= rSquaredDiff_MinImprovement && + curModel.Fitting.WasAbleToIdentify + ) + { + bestModel = curModel; + } + } + else if (Double.IsNaN(RsqFittingDiff_improvement) || Double.IsNaN(objFunDiff_improvement)) + { + if (objFunAbs_improvement >= 0 && + RsqFittingAbs_improvement >= 0 && + curModel.Fitting.WasAbleToIdentify + ) + { + bestModel = curModel; + } + } + else + { + if (objFunDiff_improvement >= obFunDiff_MinImprovement && + RsqFittingDiff_improvement >= rSquaredDiff_MinImprovement && + objFunAbs_improvement >= 0 && + RsqFittingAbs_improvement >= 0 && + curModel.Fitting.WasAbleToIdentify + ) + { + bestModel = curModel; + } + } + } + return bestModel; + } + + private GainSchedParameters EstimateProcessForAGivenTimeDelay + (int timeDelay_samples, GainSchedDataSet dataSet, + bool useDynamicModel, bool[] doEstimateCurvature, + double FilterTc_s, double[] u0, double[] uNorm, bool assumeThatYkminusOneApproxXkminusOne + ) + { + Vec vec = new Vec(dataSet.BadDataID); + var inputIndicesToRegularize = new List { 1 }; + double[] ycur, yprev = null, dcur, dprev = null; + List ucurList = new List(); + string solverID = ""; + if (assumeThatYkminusOneApproxXkminusOne) + solverID += "v1."; + else + solverID += "v2."; + int idxEnd = dataSet.Y_meas.Length - 1; + int idxStart = timeDelay_samples + 1; + if (useDynamicModel) + { + solverID += "Dynamic"; + for (int colIdx = 0; colIdx < dataSet.U.GetNColumns(); colIdx++) + { + ucurList.Add(Vec.SubArray(dataSet.U.GetColumn(colIdx), + idxStart - timeDelay_samples, idxEnd - timeDelay_samples)); + } + ycur = Vec.SubArray(dataSet.Y_meas, idxStart, idxEnd); + yprev = Vec.SubArray(dataSet.Y_meas, idxStart - 1, idxEnd - 1); + dcur = null; + dprev = null; + if (dataSet.D != null) + { + dcur = Vec.SubArray(dataSet.D, idxStart, idxEnd); + dprev = Vec.SubArray(dataSet.D, idxStart - 1, idxEnd - 1); + } + } + else + { + // NB! for static model, y and u are not shifted by one sample! + idxStart = timeDelay_samples; + solverID += "Static"; + for (int colIdx = 0; colIdx < dataSet.U.GetNColumns(); colIdx++) + { + ucurList.Add(Vec.SubArray(dataSet.U.GetColumn(colIdx), + idxStart - timeDelay_samples, idxEnd - timeDelay_samples)); + } + ycur = Vec.SubArray(dataSet.Y_meas, idxStart, idxEnd); + dcur = Vec.SubArray(dataSet.D, idxStart, idxEnd); + } + + var indUbad = new List(); + for (int colIdx = 0; colIdx < dataSet.U.GetNColumns(); colIdx++) + { + indUbad = indUbad.Union(SysIdBadDataFinder.GetAllBadIndicesPlussNext(dataSet.U.GetColumn(colIdx), + dataSet.BadDataID)).ToList(); + } + List indYcurBad = vec.FindValues(ycur, dataSet.BadDataID, VectorFindValueType.NaN); + + List yIndicesToIgnore = new List(); + if (dataSet.IndicesToIgnore != null) + if (dataSet.IndicesToIgnore.Count > 0) + { + yIndicesToIgnore = new List(dataSet.IndicesToIgnore); + } + + // the above code misses the special case that y_prev[0] is bad, as it only looks at y_cur + if (useDynamicModel) + { + if (Double.IsNaN(yprev[0]) || yprev[0] == dataSet.BadDataID) + { + yIndicesToIgnore.Add(0); + } + } + yIndicesToIgnore = yIndicesToIgnore.Union(indUbad).ToList(); + yIndicesToIgnore = yIndicesToIgnore.Union(Index.AppendTrailingIndices(indYcurBad)).ToList(); + if (dataSet.IndicesToIgnore != null) + { + var indicesMinusOne = Index.Max(Index.Subtract(dataSet.IndicesToIgnore.ToArray(), 1), 0).Distinct(); + yIndicesToIgnore = yIndicesToIgnore.Union( + Index.AppendTrailingIndices(indicesMinusOne.ToList())).ToList(); + } + yIndicesToIgnore.Sort(); + + if (FilterTc_s > 0) + { + LowPass yLp = new LowPass(dataSet.GetTimeBase()); + LowPass yLpPrev = new LowPass(dataSet.GetTimeBase()); + LowPass uLp = new LowPass(dataSet.GetTimeBase()); + ycur = yLp.Filter(ycur, FilterTc_s);//todo:disturbance + yprev = yLpPrev.Filter(yprev, FilterTc_s);//todo:disturbance + } + + RegressionResults regResults; + double timeConstant_s = Double.NaN; + double[] linearProcessGains = Vec.Fill(Double.NaN, ucurList.Count); + double[] processCurvatures = Vec.Fill(Double.NaN, ucurList.Count); + + List phiIndicesToRegularize = new List(); + + if (useDynamicModel) + { + // Tc *xdot = x[k-1] + B*u[k-1] + // Tc/Ts *(x[k]-x[k-1]) = x[k-1]*B*u[k-1] + + // a first order differential equation + // x[k] = a*x[k-1]+b*u[k-1] + + // has steady state + // x_ss = b/(1-a) + // and the time-constant is related to a as + // a = 1/(1+Ts/Tc) + // where Ts is the sampling time + + // y[k]= a * y[k-1] + b*u + // where a is related to time constants and sampling rate by a = 1/(1 + Ts/Tc) + // ==> TimeConstant Tc = Ts*/(1/a +1) + // and "b" is related to steady state gain and "a" by : b = ProcessGain*(1-a) + // == > ProcessGain = b/(1-a) + + // actually, the a and b are related to internal states x, while y is a measurement + // that is subject to noise and disturbances, and it is important to formulate the identification + // so that noise and disturances do not influence Tc estimates. + // one alternative is to pre-filter y, another is to formulate problems so that noise if averaged out by + + // ---------------------------------------- + //formulation2:(seems to be more robust to disturbances) + //try to formulate the y in terms of ycur-yprev + // y[k] = x[x] + d[k] + + + // ---------------------------------------- + // to improve the perforamnce in estimating process dynamics when distrubances are in effect + //formulation2: without the assumption y[k-1] (approx=) x[k-1] + + // to guess at the process disturbances : + + // either you need to subtract d for Y or you need to add it to Ymod + double[] x_mod_cur_raw = new double[0]; + // -----------------v1 ------------- + // APPROXIMATE x[k-1] = y[k-1] then (NOTE THAT THIS MAY NOT BE A GOOD ASSUMPTION!) + // x[k] = a*y[k-1]+ b*u[k] + // y[k]= a*y[k-1] + b*u[k] + d[k] means that + // y[k]-a*y[k-1]-d[k]= b*u + // y[k]-a*y[k-1]-(1-a)*y[k-1]-d[k]=-(1-a)*y[k-1]+ b*u (subtract -(1-a)y[k-1] on both sides) + // y[k]-y[k-1]-d[k]=-(1-a)*y[k-1] b*u + // y[k]-y[k-1]-d[k]=(a-1)*y[k-1] + b*u (RESULTING FORMUALE TO BE IDENTIFIED) + // note that the above is not completely correct, model appears better if + // subtracting Y_ols = vec.Subtract(deltaY, vec.Subtract(dcur,dprev)); + // rather than //Y_ols = vec.Subtract(deltaY, dcur); + if (assumeThatYkminusOneApproxXkminusOne) + { + double[] deltaY = vec.Subtract(ycur, yprev); + double[] phi1_ols = yprev; + double[] Y_ols = deltaY; + if (dcur != null && dprev!= null) + { + solverID += "(d subtracted)"; + phi1_ols = vec.Subtract(yprev, dprev); + Y_ols = vec.Subtract(deltaY, vec.Subtract(dcur, dprev)); + } + double[,] phi_ols2D = new double[ycur.Length, ucurList.Count + 1]; + if (doEstimateCurvature.Contains(true)) + { + int nCurvatures = 0;// + for (int i = 0; i < doEstimateCurvature.Count(); i++) + { + if (doEstimateCurvature[i]) + nCurvatures++; + } + phi_ols2D = new double[ycur.Length, ucurList.Count + nCurvatures + 1]; + } + phi_ols2D.WriteColumn(0, phi1_ols); + for (int curIdx = 0; curIdx < ucurList.Count; curIdx++) + { + double uNormCur = uNorm[curIdx]; + if (Double.IsInfinity(uNormCur) || Double.IsNaN(uNormCur) || uNormCur == 0) + { + phi_ols2D.WriteColumn(curIdx + 1, Vec.Fill(0, ucurList[curIdx].Length)); + } + else + { + phi_ols2D.WriteColumn(curIdx + 1, vec.Subtract(ucurList[curIdx], u0[curIdx])); + } + } + if (doEstimateCurvature.Contains(true)) + { + int curCurvature = 0; + for (int curIdx = 0; curIdx < doEstimateCurvature.Count(); curIdx++) + { + if (!doEstimateCurvature[curIdx]) + continue; + double uNormCur = 1; + if (uNorm != null) + { + if (curIdx.Fill(0, ucurList[curIdx].Length)); + } + else + { + phi_ols2D.WriteColumn(curCurvature + ucurList.Count + 1, + vec.Div(vec.Pow(vec.Subtract(ucurList[curIdx], u0[curIdx]), 2), uNormCur)); + } + curCurvature++; + } + } + double[][] phi_ols = phi_ols2D.Transpose().Convert2DtoJagged(); + regResults = vec.RegressRegularized(Y_ols, phi_ols, yIndicesToIgnore.ToArray(), phiIndicesToRegularize); + } + // ----------------- v2 ----------- + // APPROXIMATE x[k-1] = y[k-1]-d[k-1] + // y[k] = a * (y[k-1]-d[k-1]) + b*u[k] + d[k] + // y[k]-a*y[k-1]-d[k]= -a*d[k-1] + b*u + // y[k]-a*y[k-1]-(1-a)*y[k-1]-d[k]=-(1-a)*y[k-1]+a*d[k-1]+ b*u (subtract -(1-a)y[k-1] on both sides) + // y[k] -y[k-1]-d[k] = (a-1)*y[k-1] -a*d[k-1] +b*u (an extra "-a*d[k-1]" term) + // or a better formulation may be + // y[k]=a*(y[k-1]-d[k-1]) + b*u[k] + d[k] +q + // y[k]-d[k]=a*(y[k-1]-d[k-1])+ b*u[k] + q + else + { + double[] phi1_ols = vec.Subtract(yprev, dprev); + double[,] phi_ols2D = new double[ycur.Length, ucurList.Count + 1]; + phi_ols2D.WriteColumn(0, phi1_ols); + for (int curIdx = 0; curIdx < ucurList.Count; curIdx++) + { + phi_ols2D.WriteColumn(curIdx + 1, vec.Subtract(ucurList[curIdx], u0[curIdx])); + } + double[][] phi_ols = phi_ols2D.Transpose().Convert2DtoJagged(); + double[] Y_ols = ycur; + if (dcur != null) + { + solverID += "(d subtracted)"; + vec.Subtract(ycur, dcur); + } + regResults = vec.RegressRegularized(Y_ols, phi_ols, yIndicesToIgnore.ToArray(), inputIndicesToRegularize); + } + + if (regResults.Param != null) + { + double a; + if (assumeThatYkminusOneApproxXkminusOne) + { + a = regResults.Param[0] + 1; + } + else + { + a = regResults.Param[0]; + } + double[] b = Vec.SubArray(regResults.Param, 1, regResults.Param.Length - 2); + double[] c = null; + if (doEstimateCurvature.Contains(true)) + { + b = Vec.SubArray(regResults.Param, 1, ucurList.Count); + c = Vec.SubArray(regResults.Param, ucurList.Count+1, regResults.Param.Length - 2); + } + + if (a > 1) + a = 0; + + // the estimation finds "a" in the difference equation + // a = 1/(1 + Ts/Tc) + // so that + // Tc = Ts/(1/a-1) + + if (a != 0) + timeConstant_s = dataSet.GetTimeBase() / (1 / a - 1); + else + timeConstant_s = 0; + if (timeConstant_s < 0) + timeConstant_s = 0; + + if (a != 0) + linearProcessGains = vec.Div(b, 1 - a); + else + linearProcessGains = b; + + if (doEstimateCurvature.Contains(true) && c != null) + { + processCurvatures = Vec.Fill(0, ucurList.Count); + int curCurvature = 0; + for (int curU = 0; curU < doEstimateCurvature.Count(); curU++) + { + if (!doEstimateCurvature[curU]) + continue; + processCurvatures[curU] = c[curCurvature] / (1 - a); + + if (uNorm != null) + { + processCurvatures[curU] = processCurvatures[curU] * uNorm[curU]; + } + curCurvature++; + } + } + else + { + processCurvatures = null; + } + } + } + else//static model + { + // y[k] = Kc*u[k]+ P0 + double[,] inputs2D = new double[ycur.Length, ucurList.Count]; + for (int curIdx = 0; curIdx < ucurList.Count; curIdx++) + { + inputs2D.WriteColumn(curIdx, vec.Subtract(ucurList[curIdx], u0[curIdx])); + } + double[][] inputs = inputs2D.Transpose().Convert2DtoJagged(); + double[] Y_ols = ycur; + if (dcur != null) + { + solverID += "(d subtracted)"; + Y_ols = vec.Subtract(ycur, dcur); + } + regResults = vec.RegressRegularized(Y_ols, inputs, yIndicesToIgnore.ToArray(), inputIndicesToRegularize); + timeConstant_s = 0; + if (regResults.Param != null) + { + linearProcessGains = Vec.SubArray(regResults.Param, 0, regResults.Param.Length - 2); + } + } + GainSchedParameters parameters = new GainSchedParameters(); + + parameters.Fitting = new FittingInfo(); + parameters.Fitting.SolverID = solverID; + if (dataSet.Times != null) + { + if (dataSet.Times.Count() > 0) + { + parameters.Fitting.StartTime = dataSet.Times.First(); + parameters.Fitting.EndTime = dataSet.Times.Last(); + } + } + // Vec.Regress can return very large values if y is noisy and u is stationary. + // in these cases varCovarMatrix is null + + const double maxAbsValueRegression = 10000; + + parameters.Fitting.NFittingTotalDataPoints = regResults.NfittingTotalDataPoints; + parameters.Fitting.NFittingBadDataPoints = regResults.NfittingBadDataPoints; + + if (regResults.Param == null || !regResults.AbleToIdentify) + { + parameters.Fitting.WasAbleToIdentify = false; + parameters.AddWarning(GainScheddentWarnings.RegressionProblemFailedToYieldSolution); + return parameters; + } + else if (regResults.Param.Contains(Double.NaN) || linearProcessGains.Contains(Double.NaN)) + { + parameters.Fitting.WasAbleToIdentify = false; + parameters.AddWarning(GainScheddentWarnings.RegressionProblemNaNSolution); + return parameters; + } + else if (Math.Abs(regResults.Param[1]) > maxAbsValueRegression) + { + parameters.Fitting.WasAbleToIdentify = false; + parameters.AddWarning(GainScheddentWarnings.NotPossibleToIdentify); + return parameters; + } + else // able to identify + { + parameters.Fitting.WasAbleToIdentify = true; + parameters.TimeDelay_s = timeDelay_samples * dataSet.GetTimeBase(); + parameters.TimeConstant_s = null; // TODO: SignificantDigits.Format(timeConstant_s, nDigits); + parameters.LinearGains = null; // TODO: SignificantDigits.Format(linearProcessGains, nDigits); + parameters.U0 = u0; + parameters.UNorm = uNorm; + + (double? recalcBias, double[] y_sim_recalc) = + SimulateAndReEstimateBias(dataSet, parameters); + dataSet.Y_sim = y_sim_recalc; + if (recalcBias.HasValue) + { + parameters.Bias = SignificantDigits.Format(recalcBias.Value, nDigits); + } + else + { + parameters.AddWarning(GainScheddentWarnings.ReEstimateBiasFailed); + parameters.Bias = SignificantDigits.Format(regResults.Param.Last(), nDigits); + } + /* + if (useDynamicModel) + { + parameters.Fitting.CalcCommonFitMetricsFromDiffData(regResults.Rsq, regResults.ObjectiveFunctionValue, + dataSet); + } + else + { + parameters.Fitting.CalcCommonFitMetricsFromDataset(regResults.Rsq, regResults.ObjectiveFunctionValue, dataSet); + }*/ + // parameters.Fitting.CalcCommonFitMetricsFromDataset(dataSet, yIndicesToIgnore); // TODO: fitting for GainSched + + // add inn uncertainty + if (useDynamicModel) + CalculateDynamicUncertainty(regResults, dataSet.GetTimeBase(), ref parameters); + else + CalculateStaticUncertainty(regResults, ref parameters); + + // round uncertainty to certain number of digits + parameters.LinearGainUnc = SignificantDigits.Format(parameters.LinearGainUnc, nDigits); + if (parameters.BiasUnc.HasValue) + parameters.BiasUnc = SignificantDigits.Format(parameters.BiasUnc.Value, nDigits); + // if (parameters.TimeConstantUnc_s.HasValue) + // parameters.TimeConstantUnc_s = SignificantDigits.Format(parameters.TimeConstantUnc_s.Value, nDigits); // TODO: uncertainty in time constants + return parameters; + } + } + + + + + /// + /// Provided that regResults is the result of fitting a statix equation x[k] = B*U} + /// + /// regression results, where first paramter is the "a" forgetting term + /// + /// + private void CalculateStaticUncertainty(RegressionResults regResults, ref GainSchedParameters parameters) + { + if (regResults.VarCovarMatrix == null) + return; + double a = regResults.Param[0]; + double varA = regResults.VarCovarMatrix[0][0]; + double sqrtN = Math.Sqrt(regResults.NfittingTotalDataPoints - regResults.NfittingBadDataPoints); + ///////////////////////////////////////////////////// + /// linear gain unceratinty + var LinearGainUnc = new List(); + for (int inputIdx = 0; inputIdx < parameters.GetNumInputs(); inputIdx++) + { + double b = regResults.Param[inputIdx]; + double varB = regResults.VarCovarMatrix[inputIdx][inputIdx]; + // standard error is the population standard deviation divided by square root of the number of samples N + double standardError_processGain = varB / sqrtN; + // the 95% conf intern is 1.96 times the standard error for a normal distribution + LinearGainUnc.Add(standardError_processGain * 1.96);//95% uncertainty + } + parameters.LinearGainUnc = LinearGainUnc.ToArray(); + ///////////////////////////////////////////////// + parameters.TimeConstantUnc_s = null;//95 prc unceratinty + // bias uncertainty + int biasInd = regResults.Param.Length - 1; + //parameters.BiasUnc = regResults.VarCovarMatrix[biasInd][biasInd]; + parameters.BiasUnc = regResults.VarCovarMatrix[biasInd][biasInd] / sqrtN * 1.96; + } + + + // references: + // http://dept.stat.lsa.umich.edu/~kshedden/Courses/Stat406_2004/Notes/variance.pdf + //https://stats.stackexchange.com/questions/41896/varx-is-known-how-to-calculate-var1-x + /// + /// Provided that regResults is the result of fitting a dynamic equation x[k] = a*x[k-1]+B*U} + /// + /// regression results, where first paramter is the "a" forgetting term + /// + /// + private void CalculateDynamicUncertainty(RegressionResults regResults, double timeBase_s, ref GainSchedParameters parameters) + { + if (regResults.VarCovarMatrix == null) + return; + + double a = regResults.Param[0]; + double varA = regResults.VarCovarMatrix[0][0]; + double sqrtN = Math.Sqrt(regResults.NfittingTotalDataPoints - regResults.NfittingBadDataPoints); + + ///////////////////////////////////////////////////// + /// linear gain unceratinty + + var LinearGainUnc = new List(); + for (int inputIdx = 0; inputIdx< parameters.GetNumInputs(); inputIdx++) + { + double b = regResults.Param[1+inputIdx]; + double varB = regResults.VarCovarMatrix[inputIdx+1][inputIdx+1]; + double covAB1 = regResults.VarCovarMatrix[0][inputIdx+1]; + // first approach: + // process gain uncertainty + // process gain dy/du: b /(1- a) + // idea to take first order taylor( where mu_x = mean value of x): + //var(g(x)) approx (dg(mu_x)/dx)^2 * varx + // var(x1+x2) = var(x1) + var(x2) +2*cov(x1,x2) + //var(g(x1,x2)) (approx=) (dg/dx1)^2 *var(x1) + (dg/dx2)^2 *var(x2) +dg/dx1dx2 *cov(a,b1) + // for + // g(a,b) = b/(1-a) + // var(g(a,b)) (approx=) (dg/da)^2 *var(a) + (dg/db)^2 *var(b) +dg/dadb *cov(a,b) + double dg_da = b * -a* Math.Pow(1 - a, -2);//chain rule + double dg_db = 1 / (1 - a); + double covTerm = a* Math.Pow(1 - a, -2); + double varbdivby1minusA = + (Math.Pow(dg_da, 2) * varA + Math.Pow(dg_db, 2) * varB + covTerm * covAB1); + // second approach : + // variance of multipled vairables : var(XY) = var(x)*var(y) +var(x)*(E(Y))^2 + varY*E(X)^2 + // variance of var (b*(1/(1-a))) = var(b)*var(1/(1-a)) + var(b)*E(1/(1-a))^2 +var(1/(1-a))^2 * b + // var(1/(1-a)) - >first order linear tayolor approximation. + // + // https://stats.stackexchange.com/questions/41896/varx-is-known-how-to-calculate-var1-x + // if you use first order taylor expanison, + // Var[g(X)]≈g′(μ)^2Var(X) + // var(1/x) = 1/(mu^4)*var(x), where mu is the E + // double var1divBya = Math.Pow(1-a, -4) * varA; + // double varbdivby1minusA = varB * var1divBya + varB * Math.Pow(1 / (1 - a), 2) + Math.Pow(var1divBya, 2) * b; + + // common to both appraoches: + // standard error is the population standard deviation divided by square root of the number of samples N + double standardError_processGain = varbdivby1minusA / sqrtN; + + // the 95% conf intern is 1.96 times the standard error for a normal distribution + LinearGainUnc.Add(standardError_processGain * 1.96);//95% uncertainty + } + parameters.LinearGainUnc = LinearGainUnc.ToArray(); + + ///////////////////////////////////////////////// + // time constant uncertainty + // Tc = TimeBase_s * (1/a - 1)^1 + // var(1/(1/a - 1)) - > first order linear tayolor approximation. + // // Var[g(X)]≈dg/dmu(μ)^2 * Var(X) + + // derivate(chain rule) : d[(1/a - 1)^-1]/da == (1/a-1)^-2*(a^-2) + + // remmember that Var(cX) = c^2*Var(X)! + // thus since Tc = TimeBase_s * (1/a - 1)^-1 + //var(Tc = TimeBase_s * (1/a - 1)) = TimeBase_s^2 * var(1/a - 1) + + // uncertainty = var(Tc*(1/a-1)^-1)*sqrt(N)*1.96 + double varTc = (Math.Pow(timeBase_s, 2) * Math.Pow(1 / a - 1, -2) * Math.Pow(a, -2) * varA); + double standardErrorTc = varTc/ sqrtN; + // parameters.TimeConstantUnc_s = standardErrorTc *1.96;//95 prc unceratinty // TODO: make this for array type too + //parameters.TimeConstantUnc_s = varTc; + + // bias uncertainty + int biasInd = regResults.Param.Length-1; + //parameters.BiasUnc = regResults.VarCovarMatrix[biasInd][biasInd]; + parameters.BiasUnc = regResults.VarCovarMatrix[biasInd][biasInd] /sqrtN * 1.96; + } + + // + // bias is not always accurate for dynamic model identification + // as it is as "difference equation" that matches the changes in the + // + static public (double?, double[]) SimulateAndReEstimateBias(GainSchedDataSet dataSet, GainSchedParameters parameters) + { + GainSchedDataSet internalData = new GainSchedDataSet(dataSet); + parameters.Bias = 0; + double nanValue = internalData.BadDataID; + var model = new GainSchedModel(parameters); + var simulator = new GainSchedSimulator(model); + var y_sim = simulator.Simulate(ref internalData); + var yMeas_exceptIgnoredValues = internalData.Y_meas; + var ySim_exceptIgnoredValues = y_sim; + if (dataSet.IndicesToIgnore != null) + { + for (int ind = 0; ind < dataSet.IndicesToIgnore.Count(); ind++) + { + int indToIgnore = dataSet.IndicesToIgnore.ElementAt(ind); + yMeas_exceptIgnoredValues[indToIgnore] = Double.NaN;//nan values are ignored by Vec.Means + if (ySim_exceptIgnoredValues != null) + { + ySim_exceptIgnoredValues[indToIgnore] = Double.NaN;//nan values are ignored by Vec.Means + } + } + } + double[] diff = (new Vec(nanValue)).Subtract(yMeas_exceptIgnoredValues, ySim_exceptIgnoredValues); + double? bias = (new Vec(nanValue)).Mean(diff); + double[] y_sim_ret = null; + if (bias.HasValue && y_sim != null) + { + y_sim_ret = (new Vec(nanValue)).Add(y_sim, bias.Value); + } + return (bias, y_sim_ret); + } + + /// + /// Freezed one input to a given pre-determined value, but re-identifies other static paramters. + /// This is useful if doing a "global search" where varying a single gain. + /// + /// + /// the index of the value to freeze + /// the linear gain to freeze the at + /// identified model, to check if identification suceeded, check + /// .modelParameters.Fitting.WasAbleToIdentify + public GainSchedModel IdentifyLinearAndStaticWhileKeepingLinearGainFixed(GainSchedDataSet dataSet, int inputIdxToFix, + double inputProcessGainValueToFix, double u0_fixedInput, double uNorm_fixedInput) + { + var internalDataset = new GainSchedDataSet(dataSet); + var vec = new Vec(dataSet.BadDataID); + var D_fixedInput = vec.Multiply(vec.Multiply(vec.Subtract(dataSet.U.GetColumn(inputIdxToFix), u0_fixedInput), uNorm_fixedInput), + inputProcessGainValueToFix); + if (dataSet.D != null) + { + internalDataset.D = vec.Add(internalDataset.D, D_fixedInput); + } + else + { + internalDataset.D = D_fixedInput; + } + + + + // remove the input that is frozen from the dataset given to the "identify" algorithm + double[,] newU = new double[internalDataset.U.GetNRows(), internalDataset.U.GetNColumns()-1]; + int writeColIdx = 0; + for (int colIdx = 0; colIdx < internalDataset.U.GetNColumns(); colIdx++) + { + if (colIdx == inputIdxToFix) + continue; + newU.WriteColumn(writeColIdx, dataSet.U.GetColumn(colIdx)); + writeColIdx++; + } + internalDataset.U = newU; + + var idGainSchedModel = IdentifyLinearAndStatic(ref internalDataset, new FittingSpecs()); + if (!idGainSchedModel.modelParameters.Fitting.WasAbleToIdentify) + return idGainSchedModel; + //trick now is to add back the paramters that are fixed to the returned model: + var idLinGains = idGainSchedModel.modelParameters.LinearGains; + var idU = idGainSchedModel.modelParameters.U0; + var idUnorm = idGainSchedModel.modelParameters.UNorm; + + var newLinGainsList = new List(); + var newU0List = new List(); + var newUNormList = new List(); + + var curIdInput = 0; + // for (int curInputIdx = 0; curInputIdx < idGainSchedModel.modelParameters.LinearGains.Length + 1; curInputIdx++) // TODO: .Length does not work because of array + // { + // if (curInputIdx == inputIdxToFix) + // { + // newLinGainsList.Add(inputProcessGainValueToFix); + // newU0List.Add(u0_fixedInput); + // newUNormList.Add(uNorm_fixedInput); + // } + // else + // { + // // newLinGainsList.Add(idLinGains.ElementAt(curIdInput)); // TODO: removed this + // newU0List.Add(idU.ElementAt(curIdInput)); + // newUNormList.Add(idUnorm.ElementAt(curIdInput)); + // curIdInput++; + // } + // } + + var newParams = new GainSchedParameters(); + // newParams.LinearGains = newLinGainsList.ToArray(); // TODO: removed this + newParams.LinearGainUnc = Vec.Fill(double.NaN, newLinGainsList.Count); // + int counter = 0; + for (int idx = 0; idx < newLinGainsList.Count; idx++) + { + if (idx == inputIdxToFix) + { + newParams.LinearGainUnc[idx] = double.NaN; + } + else + { + newParams.LinearGainUnc[idx] = idGainSchedModel.modelParameters.LinearGainUnc[counter]; + counter++; + } + + } + newParams.U0 = newU0List.ToArray(); + newParams.UNorm = newUNormList.ToArray(); + newParams.Bias = idGainSchedModel.modelParameters.Bias; + newParams.Fitting = idGainSchedModel.modelParameters.Fitting; + newParams.Fitting.SolverID = "Linear,static while fixing index:" + inputIdxToFix; + var retGainSchedModel = new GainSchedModel(newParams); + retGainSchedModel.SetFittedDataSet(idGainSchedModel.GetFittedDataSet()); + return retGainSchedModel; + } + } +} diff --git a/Dynamic/Identification/GainSchedTimeDelayIdentifier.cs b/Dynamic/Identification/GainSchedTimeDelayIdentifier.cs new file mode 100644 index 00000000..51362a71 --- /dev/null +++ b/Dynamic/Identification/GainSchedTimeDelayIdentifier.cs @@ -0,0 +1,245 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using TimeSeriesAnalysis; +using TimeSeriesAnalysis.Utility; + +namespace TimeSeriesAnalysis.Dynamic +{ + + + + /// + /// Brute-force(trial and error) estimation of time delay + /// + /// This method is intended to be generic, so that it can be applied on different kinds of process models, and thus it uses + /// interfaces and dependency injection. + /// + /// + /// The idea of the method is to re-identify the model first with no time-delay, and then increasing the time-delay step-by-step. + /// This process should continue so long as the model improves (as measured by uncertainty mangitude, objective function value or Rsquared). + /// + /// + /// Thus, this method is a component in reducing the problem of determining continous paramters along with the integer time delay + /// (a mixed-integer problem) to a sequential optimization approach where the integer and continous parts of the problem are solved + /// sequentially. + /// + /// + /// + class GainSchedTimeDelayIdentifier + { + private double TimeBase_s; + + private const int minTimeDelayIts = 5; + private int maxExpectedTimeDelay_samples; + + private List modelRuns; + + + public GainSchedTimeDelayIdentifier(double TimeBase_s, double maxExpectedTc_s) + { + this.TimeBase_s = TimeBase_s; + modelRuns = new List(); + + this.maxExpectedTimeDelay_samples = Math.Max((int)Math.Floor(maxExpectedTc_s / TimeBase_s), minTimeDelayIts); + + } + + public void AddRun(ModelParametersBaseClass modelParameters) + { + modelRuns.Add(modelParameters); + } + + public bool DetermineIfContinueIncreasingTimeDelay(int timeDelayIdx) + { + if (timeDelayIdx < minTimeDelayIts) + return true; + + // nb! note that time window decreases with one timestep for each increase in time delay of one timestep. + // thus there is a chance of the object function decreasing without the model fit improving. + // especially if the only information in the dataset is at the start of the dataset. + var objFunVals = GetObjFunDiffValList(); + bool isObjFunDecreasing = objFunVals.ElementAt(objFunVals.Length - 2) < objFunVals.ElementAt(objFunVals.Length-1); + + var r2Vals = GetR2DiffList(); + bool isR2Decreasing = r2Vals.ElementAt(r2Vals.Length - 2) < r2Vals.ElementAt(r2Vals.Length - 1); ; + + bool continueIncreasingTimeDelayEst;// = true; + + if (isObjFunDecreasing || isR2Decreasing) + { + if (timeDelayIdx < maxExpectedTimeDelay_samples) + { + continueIncreasingTimeDelayEst = true; + } + else + { + continueIncreasingTimeDelayEst = false; + } + } + else + continueIncreasingTimeDelayEst = false; + + + return continueIncreasingTimeDelayEst; + } + + /// + /// Get R2 of all runs so far in an array, + /// + /// array or R-squared values, with Nan if any runs have failed + private double[] GetR2DiffList() + { + List objR2List = new List(); + for (int i = 0; i < modelRuns.Count; i++) + { + if (modelRuns[i] == null) + { + objR2List.Add(Double.NaN); + } + else + { + if (modelRuns[i].Fitting.WasAbleToIdentify) + { + objR2List.Add(modelRuns[i].Fitting.RsqDiff); + } + else + { + objR2List.Add(Double.NaN); + } + } + + } + return objR2List.ToArray(); + } + + /// + /// Get objective functions valueof all runs so far in an array + /// + /// array of objective function values, nan for failed runs + private double[] GetObjFunDiffValList() + { + List objFunValList = new List(); + for (int i = 0; i < modelRuns.Count; i++) + { + if (modelRuns[i] == null) + { + objFunValList.Add(Double.NaN); + } + else + { + if (modelRuns[i].Fitting.WasAbleToIdentify) + { + objFunValList.Add(modelRuns[i].Fitting.ObjFunValDiff); + } + else + { + objFunValList.Add(Double.NaN); + } + } + } + return objFunValList.ToArray(); + } + + + + /// + /// Chooses between all the stored model runs,the model which seems the best. + /// + /// Index of best time delay model + public int ChooseBestTimeDelay(out List warnings) + { + int bestTimeDelayIdx; + warnings = new List(); + + // one issue is that if time delay is increasing, the ymod becomes shorter, + // so this needs to be objective function either normalized or excluding the first few points. + + // + // R-squared analysis + // + int R2BestTimeDelayIdx; + { + var objR2List = GetR2DiffList(); + Vec.Sort(objR2List.ToArray(), VectorSortType.Descending, out int[] sortedIndices); + R2BestTimeDelayIdx = sortedIndices[0]; + // the solution space should be "concave", so there should not be several local minimia + // as you compare R2list for different time delays-that has happended but indicates something + // is wrong. + if (sortedIndices.Count() > 1) + { + + int R2distanceToRunnerUp = Math.Abs(sortedIndices[1] - sortedIndices[0]); + if (R2distanceToRunnerUp > 1) + { + warnings.Add(ProcessTimeDelayIdentWarnings.NonConvexRsquaredSolutionSpace); + } + if (objR2List.Contains(Double.NaN)) + { + warnings.Add(ProcessTimeDelayIdentWarnings.SomeModelRunsFailedToFindSolution); + } + double R2valueDiffToRunnerUp = objR2List[sortedIndices[0]] - objR2List[sortedIndices[1]]; + if (R2valueDiffToRunnerUp < 0.1) + { + warnings.Add(ProcessTimeDelayIdentWarnings.NoUniqueRsquaredMinima); + } + } + } + // + // objective function value analysis + // + { + var objObjFunList = GetObjFunDiffValList(); + Vec.Sort(objObjFunList.ToArray(), VectorSortType.Ascending, out int[] objFunSortedIndices); + // int objFunBestTimeDelayIdx = objFunSortedIndices[0]; + // the solution space should be "concave", so there should not be several local minimia + // as you compare R2list for different time delays-that has happended but indicates something + // is wrong. + if (objFunSortedIndices.Count() > 1) + { + int objFunDistanceToRunnerUp = Math.Abs(objFunSortedIndices[1] - objFunSortedIndices[0]); + if (objFunDistanceToRunnerUp > 1) + { + warnings.Add(ProcessTimeDelayIdentWarnings.NonConvexObjectiveFunctionSolutionSpace); + } + double ObjFunvalueDiffToRunnerUp = Math.Abs(objObjFunList[objFunSortedIndices[0]] + - objObjFunList[objFunSortedIndices[1]]); + if (ObjFunvalueDiffToRunnerUp <= 0.0001) + { + warnings.Add(ProcessTimeDelayIdentWarnings.NoUniqueObjectiveFunctionMinima); + } + } + } + + // + // parameter uncertatinty value analysis + // + // TODO: consider re-introducing ranking by uncertainty in a generic way. + + /* + // v1: use uncertainty to choose time delay + if (!ProcessGainEstUnc_prc.ToArray().Contains(Double.NaN) && + !(BiasEstUnc.ToArray().Contains(Double.NaN))) + { + Vec.Min(ProcessGainEstUnc_prc.ToArray(), out int processGainUncBestTimeDelayIdx); + Vec.Min(TimeConstantEstUnc_prc.ToArray(), out int processTimeConstantUncBestTimeDelayIdx); + Vec.Min(BiasEstUnc.ToArray(), out int biasUncBestTimeDelayIdx); + bestTimeDelayIdx = Math.Min(biasUncBestTimeDelayIdx, processGainUncBestTimeDelayIdx); + } + else*/// fallback: use just lowest objective function + { + bestTimeDelayIdx = R2BestTimeDelayIdx; + } + return bestTimeDelayIdx; + } + + public ModelParametersBaseClass GetRun(int runIndex) + { + return modelRuns.ElementAt(runIndex); + } + + } +} diff --git a/Dynamic/Identification/GainScheddentWarings.cs b/Dynamic/Identification/GainScheddentWarings.cs new file mode 100644 index 00000000..ecaf072c --- /dev/null +++ b/Dynamic/Identification/GainScheddentWarings.cs @@ -0,0 +1,96 @@ +namespace TimeSeriesAnalysis.Dynamic +{ + /// + /// Enum of recognized warning or error states during identification of process model + /// + public enum GainScheddentWarnings + { + /// + /// No errors or warnings + /// + Nothing = 0, + + /// + /// The dataset is time span is very short compared to the maximal time-constant given as input to the algorithm + /// + DataSetVeryShortComparedtoTMax = 1, + + /// + /// + /// + RegressionProblemFailedToYieldSolution = 2, + + /// + /// The time delay which gave the lowest objective function is the biggest allowed time delay + /// - consider increasing this limit or if something is wrong + /// + TimeDelayAtMaximumConstraint = 3, + + /// + /// When considering different time delays internally, you expect the "best" to have both + /// the lowest objective functino _and_ the lowest paramter uncertainty + /// - but for some reason this is not the case + /// + TimeDelayInternalInconsistencyBetweenObjFunAndUncertainty = 4, + + /// + /// Time constant is not identifiable from dataset + /// + TimeConstantNotIdentifiable = 5, + + /// + /// Time constant estimates vary significantly across dataset, indicating that something is wrong + /// + TimeConstantEstimateNotConsistent = 6, // validation did not match estimation + + /// + /// It was not possible to identify the + /// + NotPossibleToIdentify = 7, + + /// + /// Estimation returned an enourmous time constant, this is an indication of something is wrong + /// + TimeConstantEstimateTooBig = 8, + + /// + /// Re-estimating bias method returned null, so the bias from the intial estimation is used, be careful the bias + /// estimate may be off! + /// + ReEstimateBiasFailed = 9, + + /// + /// If disturbance is nonzero, then re-estimation of bias is turned off + /// + + ReEstimateBiasDisabledDueToNonzeroDisturbance = 10, + + + /// + /// One or more inputs were constant and not identifiable, this can affect other parameters as well, and consider removing the input + /// or using a different dataset + /// + ConstantInputU = 11, + + /// + /// Correlated inputs + /// + CorrelatedInputsU = 12, + + /// + /// Some of the parameters returned were NaN, this can happen in the input and output vectors are all constants, such as all-zero + /// + /// + RegressionProblemNaNSolution = 14, + + /// + /// If this is a closed loop system where setpoints of PID-controller change, and the "global search" at step1 of the ClosedLoopIdentifier failed to find a local minima when + /// trying different gains. This warning likely means that the linear gain can be totally off the mark and may be too low. + /// + /// + ClosedLoopEst_GlobalSearchFailedToFindLocalMinima = 15 + + + + } +} diff --git a/Dynamic/SimulatableModels/GainSchedModel.cs b/Dynamic/SimulatableModels/GainSchedModel.cs index 2f511e3a..c3f324ba 100644 --- a/Dynamic/SimulatableModels/GainSchedModel.cs +++ b/Dynamic/SimulatableModels/GainSchedModel.cs @@ -32,7 +32,7 @@ public class GainSchedModel : ModelBaseClass, ISimulatableModel private bool isFirstIteration; private double[] lastGoodValuesOfInputs; - private UnitDataSet FittedDataSet=null; + private GainSchedDataSet FittedDataSet=null; private List TimeDelayEstWarnings { get; } @@ -54,7 +54,7 @@ public GainSchedModel(GainSchedParameters modelParameters, string ID="not_named" /// /// /// a unique string that identifies this model in larger process models - public GainSchedModel(GainSchedParameters modelParameters, UnitDataSet dataSet,string ID = "not_named") + public GainSchedModel(GainSchedParameters modelParameters, GainSchedDataSet dataSet,string ID = "not_named") { processModelType = ModelType.SubProcess; this.ID = ID; @@ -135,12 +135,12 @@ private void InitSim(GainSchedParameters modelParameters) } } - public void SetFittedDataSet(UnitDataSet dataset) + public void SetFittedDataSet(GainSchedDataSet dataset) { FittedDataSet = dataset; } - public UnitDataSet GetFittedDataSet() + public GainSchedDataSet GetFittedDataSet() { return FittedDataSet; } diff --git a/Dynamic/SimulatableModels/GainSchedParameters.cs b/Dynamic/SimulatableModels/GainSchedParameters.cs index 447ae9ab..812dc328 100644 --- a/Dynamic/SimulatableModels/GainSchedParameters.cs +++ b/Dynamic/SimulatableModels/GainSchedParameters.cs @@ -94,7 +94,7 @@ public class GainSchedParameters : ModelParametersBaseClass public double? BiasUnc { get; set; } = null; - private List errorsAndWarningMessages; + private List errorsAndWarningMessages; internal List TimeDelayEstimationWarnings; /// @@ -103,7 +103,7 @@ public class GainSchedParameters : ModelParametersBaseClass public GainSchedParameters() { Fitting = null; - errorsAndWarningMessages = new List(); + errorsAndWarningMessages = new List(); } @@ -228,7 +228,7 @@ public double GetTotalCombinedProcessGainUncertainty(int inputIdx) /// Adds a identifiation warning to the object /// /// - public void AddWarning(UnitdentWarnings warning) + public void AddWarning(GainScheddentWarnings warning) { if (!errorsAndWarningMessages.Contains(warning)) errorsAndWarningMessages.Add(warning); @@ -238,7 +238,7 @@ public void AddWarning(UnitdentWarnings warning) /// Get the list of all warnings given during identification of the model /// /// - public List GetWarningList() + public List GetWarningList() { return errorsAndWarningMessages; } diff --git a/Dynamic/UnitSimulator/ModelType.cs b/Dynamic/UnitSimulator/ModelType.cs index a620bc73..aa56404d 100644 --- a/Dynamic/UnitSimulator/ModelType.cs +++ b/Dynamic/UnitSimulator/ModelType.cs @@ -45,7 +45,7 @@ public enum ModelType Divide = 5, /// - /// Divide block + /// GainSched block /// GainSchedModel = 6 } diff --git a/TimeSeriesAnalysis.Tests/Tests/GainSchedSimulatorTests.cs b/TimeSeriesAnalysis.Tests/Tests/GainSchedSimulatorTests.cs new file mode 100644 index 00000000..b1536a9e --- /dev/null +++ b/TimeSeriesAnalysis.Tests/Tests/GainSchedSimulatorTests.cs @@ -0,0 +1,142 @@ +using System; +using System.Buffers.Text; +using System.Collections.Generic; +using System.Configuration; +using System.Linq; +using System.Reflection; +using System.Reflection.Metadata; +using System.Text; +using System.Threading.Tasks; +using Accord.Statistics.Filters; +using Accord.Statistics.Kernels; +using NUnit.Framework; + +using TimeSeriesAnalysis; +using TimeSeriesAnalysis.Dynamic; +using TimeSeriesAnalysis.Utility; + + + +namespace TimeSeriesAnalysis.Test.GainSchedSim +{ + [TestFixture] + class InitializationTests + { + //Test that the gain scheduler initializes correctly with the expected default values. + //Test that the gain scheduler correctly accepts and sets initial configuration parameters. + + int timeBase_s = 1; + int N = 500; + + int Ysetpoint = 50; + + GainSchedParameters gainSchedParameters1; + GainSchedModel GainSchedModel1; + + [SetUp] + public void SetUp() + { + Shared.GetParserObj().EnableDebugOutput(); + + gainSchedParameters1 = new GainSchedParameters + { + TimeConstant_s = new double[] { 10, 0 }, + TimeConstantThresholds = new double[] { 2 }, + LinearGains = new List { new double[] { 5 }, new double[] { 10 } }, + LinearGainThresholds = new double[] { 2.5 }, + TimeDelay_s = 0, + Bias = 0 + }; + + GainSchedModel1 = new GainSchedModel(gainSchedParameters1, "Gain"); + } + + + [TestCase] + public void Sim_NoDisturbance_InitalizesSteady() + { + GainSchedDataSet GSData = new GainSchedDataSet("test"); + GSData.Y_setpoint = TimeSeriesCreator.Constant(Ysetpoint, N); + GSData.Times = TimeSeriesCreator.CreateDateStampArray(new DateTime(2000, 1, 1), timeBase_s, N); + + var sim = new GainSchedSimulator(GainSchedModel1); + var inputData = new TimeSeriesDataSet(); + + // Arrange + + // Act + + // Assert + + } + + // ... + + } + + [TestFixture] + class InputHandlingTests + { + //Test that the gain scheduler correctly handles valid input ranges for each input variable. + //Test that the gain scheduler properly reports or handles out-of-range inputs or invalid data types. + + // ... + + } + + [TestFixture] + class ComputationTests + { + //Test for correct gain computation for known input sets. + //Test gain computation at the boundaries of input ranges to ensure linear or nonlinear transitions are handled as expected. + //Test how the gain computation handles sudden changes in inputs(e.g., step changes). + + // ... + + } + + [TestFixture] + class OutputTests + { + //Test that the output gains are within the expected range. + //Test for expected output given specific input conditions (e.g., nominal, edge, and corner cases). + + // ... + + } + + [TestFixture] + class LogicTests + { + //Test that the scheduling logic switches between different sets of gains correctly based on the predefined rules or conditions. + //Test the scheduler's behavior when conditions for multiple gain sets are met simultaneously (if applicable). + + // ... + + } + + [TestFixture] + class RobustnessAndErrorHandlingTests + { + //Test the system's response to erroneous inputs or failure modes. + //Test how the system recovers from errors or handles continuous out-of-range inputs. + + // ... + + } + + [TestFixture] + class PerformanceTests + { + //Test that the gain scheduler meets performance requirements, such as computation time, especially under rapidly changing input conditions. + + // ... + + } + + [TestFixture] + class IdentifyTests + { + //Test that ??? + } +} diff --git a/TimeSeriesAnalysis.Tests/Tests/PlantSimulatorMISOTests.cs b/TimeSeriesAnalysis.Tests/Tests/PlantSimulatorMISOTests.cs index d94b4d36..166d7907 100644 --- a/TimeSeriesAnalysis.Tests/Tests/PlantSimulatorMISOTests.cs +++ b/TimeSeriesAnalysis.Tests/Tests/PlantSimulatorMISOTests.cs @@ -376,14 +376,14 @@ public void GainSched_Single_RunsAndConverges(int ver, double step1Out, double s // Assert.IsTrue(Math.Abs(simY.Last() - (1 * 55 + 0.5 * 45 + 5)) < 0.01); - Shared.EnablePlots(); - Plot.FromList(new List { - simY1, - inputData.GetValues(gainSched.GetID(),SignalType.External_U,0), - }, - new List { "y1=y_sim" + ver.ToString(), "y3=u1" }, - timeBase_s, "GainSched_Single"); - Shared.DisablePlots(); + //Shared.EnablePlots(); + //Plot.FromList(new List { + // simY1, + // inputData.GetValues(gainSched.GetID(),SignalType.External_U,0), + // }, + // new List { "y1=y_sim" + ver.ToString(), "y3=u1" }, + // timeBase_s, "GainSched_Single"); + //Shared.DisablePlots(); } @@ -424,15 +424,15 @@ public void GainSched_Multiple_RunsAndConverges(int ver, double step3Out, double // Assert.IsTrue(Math.Abs(simY.Last() - (1 * 55 + 0.5 * 45 + 5)) < 0.01); - Shared.EnablePlots(); - Plot.FromList(new List { - simY1, - inputData.GetValues(gainSched.GetID(),SignalType.External_U,0), - inputData.GetValues(gainSched.GetID(),SignalType.External_U,1), - inputData.GetValues(gainSched.GetID(),SignalType.External_U,2)}, - new List { "y1=y_sim" + ver.ToString(), "y3=u1", "y3=u2", "y3=u3"}, - timeBase_s, "GainSched_Multiple"); - Shared.DisablePlots(); + //Shared.EnablePlots(); + //Plot.FromList(new List { + // simY1, + // inputData.GetValues(gainSched.GetID(),SignalType.External_U,0), + // inputData.GetValues(gainSched.GetID(),SignalType.External_U,1), + // inputData.GetValues(gainSched.GetID(),SignalType.External_U,2)}, + // new List { "y1=y_sim" + ver.ToString(), "y3=u1", "y3=u2", "y3=u3"}, + // timeBase_s, "GainSched_Multiple"); + //Shared.DisablePlots(); }