Temporary Encog Normalization Extensions Library

.gitignore

@@ -14,28 +14,32 @@ encog-core-cs/obj/
 encog-core-test/bin/
 encog-core-test/obj/
 
-encog-core-cs.5.1.ReSharper.user
-EncogCmd/VSdoc/EncogCmd.hhc
-EncogCmd/VSdoc/encog-core-cs_dyn_help.xml
-EncogCmd/VSdoc/msdn2/CFW.gif
-EncogCmd/VSdoc/msdn2/collall.gif
-SOMColors/SOMColors.csproj.vs10x
-ConsoleExamples/ConsoleExamples.csproj.vs10x
-OCR/OCR.csproj.vs10x
-encog-core-cs/encog-core-cs.csproj.vs10x
-EncogCmd/VSdoc/unregister_encog-core-cs.bat
-EncogCmd/VSdoc/HelixoftHelpReg.exe
-EncogCmd/VSdoc/FINAL_CHM_DOC/encog-core-cs.chm
-EncogCmd/VSdoc/encog-core-cs.chm
-encog-core-cs.5.1.ReSharper.user
-encog-core-cs.5.1.ReSharper.user
-encog-core-cs.5.1.ReSharper.user
-ConsoleExamples/ConsoleExamples.vsdoc
-encog-core-cs.5.1.ReSharper.user
-encog-core-cs.5.1.ReSharper.user
-encog-core-cs.5.1.ReSharper.user
-QuickConsoleTests/Program.cs
-encog-core-cs.sln.vsdoc
-EncogCmd/EncogCmd.csproj.vs10x
-encog-core-cs/encog-core-cs.vsdoc
-encog-core-cs.5.1.ReSharper.user.orig
+encog-core-cs.5.1.ReSharper.user
+EncogCmd/VSdoc/EncogCmd.hhc
+EncogCmd/VSdoc/encog-core-cs_dyn_help.xml
+EncogCmd/VSdoc/msdn2/CFW.gif
+EncogCmd/VSdoc/msdn2/collall.gif
+SOMColors/SOMColors.csproj.vs10x
+ConsoleExamples/ConsoleExamples.csproj.vs10x
+OCR/OCR.csproj.vs10x
+encog-core-cs/encog-core-cs.csproj.vs10x
+EncogCmd/VSdoc/unregister_encog-core-cs.bat
+EncogCmd/VSdoc/HelixoftHelpReg.exe
+EncogCmd/VSdoc/FINAL_CHM_DOC/encog-core-cs.chm
+EncogCmd/VSdoc/encog-core-cs.chm
+encog-core-cs.5.1.ReSharper.user
+encog-core-cs.5.1.ReSharper.user
+encog-core-cs.5.1.ReSharper.user
+ConsoleExamples/ConsoleExamples.vsdoc
+encog-core-cs.5.1.ReSharper.user
+encog-core-cs.5.1.ReSharper.user
+encog-core-cs.5.1.ReSharper.user
+QuickConsoleTests/Program.cs
+encog-core-cs.sln.vsdoc
+EncogCmd/EncogCmd.csproj.vs10x
+encog-core-cs/encog-core-cs.vsdoc
+encog-core-cs.5.1.ReSharper.user.orig
+encog-extensions-test/bin
+encog-extensions-test/obj
+EncogExtensions/bin
+EncogExtensions/obj

EncogExtensions/Normalization/AnalystNormalizeDataSet.cs

@@ -0,0 +1,129 @@
+using Encog.App.Analyst;
+using Encog.App.Analyst.Missing;
+using Encog.App.Analyst.Script.Normalize;
+using Encog.Util.Arrayutil;
+using System;
+using System.Data;
+
+namespace EncogExtensions.Normalization
+{
+    public class AnalystNormalizeDataSet
+    {
+        private EncogAnalyst _analyst;
+
+        public AnalystNormalizeDataSet(EncogAnalyst analyst)
+        {
+            this._analyst = analyst;
+        }
+
+        public double[,] Normalize(DataSet dataSet)
+        {
+            int outputLength = _analyst.DetermineTotalColumns();
+            double[,] result = new double[dataSet.Tables[0].Rows.Count, outputLength];
+
+            for (var irow = 0; irow < dataSet.Tables[0].Rows.Count; irow++)
+            {
+                var normalizedRowFields = ExtractFields(_analyst, dataSet.Tables[0].Rows[irow], outputLength, false);
+                if (normalizedRowFields != null)
+                {
+                    for (var iNormColumn = 0; iNormColumn < outputLength; iNormColumn++)
+                    {
+                        result[irow, iNormColumn] = normalizedRowFields[iNormColumn];
+                    }
+                }
+            }
+
+            return result;
+        }
+
+        private double[] ExtractFields(EncogAnalyst analyst, DataRow rowData, int outputLength, bool v)
+        {
+            var output = new double[outputLength];
+            int outputIndex = 0;
+
+            foreach (AnalystField stat in analyst.Script.Normalize.NormalizedFields)
+            {
+                stat.Init();
+                if (stat.Action == NormalizationAction.Ignore)
+                {
+                    continue;
+                }
+
+                var index = Array.FindIndex(analyst.Script.Fields, x => x.Name == stat.Name);
+                var str = rowData[index].ToString();
+                // is this an unknown value?
+                if (str.Equals("?") || str.Length == 0)
+                {
+                    IHandleMissingValues handler = analyst.Script.Normalize.MissingValues;
+                    double[] d = handler.HandleMissing(analyst, stat);
+
+                    // should we skip the entire row
+                    if (d == null)
+                    {
+                        return null;
+                    }
+
+                    // copy the returned values in place of the missing values
+                    for (int i = 0; i < d.Length; i++)
+                    {
+                        output[outputIndex++] = d[i];
+                    }
+                }
+                else
+                {
+                    // known value
+
+                    if (stat.Action == NormalizationAction.Normalize)
+                    {
+                        double d = double.Parse(str.Trim());
+                        d = stat.Normalize(d);
+                        output[outputIndex++] = d;
+                    }
+                    else if (stat.Action == NormalizationAction.PassThrough)
+                    {
+                        double d = double.Parse(str);
+                        output[outputIndex++] = d;
+                    }
+                    else
+                    {
+                        double[] d = stat.Encode(str.Trim());
+
+                        foreach (double element in d)
+                        {
+                            output[outputIndex++] = element;
+                        }
+                    }
+                }
+            }
+
+            return output;
+        }
+
+        /*
+                private double[] ExtractFields(EncogAnalyst _analyst, DataSet dataSet, bool skipOutput)
+                {
+                    var output = new List<double>();
+                    int outputIndex = 0;
+
+                    foreach (AnalystField stat in _analyst.Script.Normalize.NormalizedFields)
+                    {
+                        stat.Init();
+                        if (stat.Action == NormalizationAction.Ignore)
+                        {
+                            continue;
+                        }
+
+                        if (stat.Output && skipOutput)
+                        {
+                            continue;
+                        }
+
+                        if(stat.Action == NormalizationAction.Normalize)
+                        {
+                            output[outputIndex] =
+                        }
+                    }
+                }
+          */
+    }
+}

EncogExtensions/Normalization/AnalystWizardExtensions.cs

@@ -0,0 +1,41 @@
+using Encog.App.Analyst;
+using Encog.App.Analyst.Wizard;
+using System;
+using System.Collections.Generic;
+using System.Data;
+
+namespace EncogExtensions.Normalization
+{
+    public static class AnalystWizardExtensions
+    {
+
+
+        public static void Wizard(this AnalystWizard wizard, DataSet data)
+        {
+            EncogAnalyst analyst = wizard.GetPrivateField<EncogAnalyst>("_analyst");
+            int lagWindowSize = wizard.GetPrivateField<int>("_lagWindowSize");
+            int leadWindowSize = wizard.GetPrivateField<int>("_leadWindowSize");
+
+            wizard.SetPrivateField("_timeSeries", (lagWindowSize > 0 || leadWindowSize > 0));
+
+            wizard.CallPrivateMethod("DetermineClassification");
+            wizard.CallPrivateMethod("GenerateSettings");
+
+
+            analyst.Analyze(data);
+
+
+            wizard.CallPrivateMethod("GenerateNormalizedFields");
+            wizard.CallPrivateMethod("GenerateSegregate");
+            wizard.CallPrivateMethod("GenerateGenerate");
+            wizard.CallPrivateMethod("GenerateTasks");
+
+            if (wizard.GetPrivateField<bool>("_timeSeries") 
+                && (lagWindowSize > 0) 
+                && (leadWindowSize > 0))
+            {
+                wizard.CallPrivateMethod("ExpandTimeSlices");
+            }
+        }
+    }
+}

EncogExtensions/Normalization/DatasetExtensions.cs

@@ -0,0 +1,78 @@
+using Encog.ML.Data.Market;
+using Encog.ML.Data.Market.Loader;
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Linq;
+
+namespace EncogExtensions.Normalization
+{
+    public static class DatasetExtensions
+    {
+        public static DataSet Convert(this DataSet dataSet, List<LoadedMarketData> data, string dataSetName = "dataset")
+        {
+            if (data.Count < 1)
+            {
+                throw new ArgumentOutOfRangeException("Loaded Market Data passed to DataSet.Convert() method appears to be empty (Contains 0 Rows).");
+            }
+
+            var resultDataSet = new DataSet("dataset");
+            DataTable table = new DataTable("Market Data Table");
+
+            var dataRowCount = data.Count();
+            var initialDataColumns = new List<DataColumn>();
+
+            AddInitialColumn(initialDataColumns, "StockSymbol", typeof(String));
+            AddInitialColumn(initialDataColumns, "Day", typeof(int));
+            AddInitialColumn(initialDataColumns, "Month", typeof(int));
+            AddInitialColumn(initialDataColumns, "Year", typeof(int));
+            CopyInitialColumnsToTable(table, initialDataColumns);
+
+            var dataColumnInitialIndex = initialDataColumns.Count;
+
+            foreach (KeyValuePair<MarketDataType, double> column in data[0].Data)
+            {
+                DataColumn dataColumn = new DataColumn(column.Key.ToString());
+                dataColumn.DataType = column.Value.GetType();
+                table.Columns.Add(dataColumn);
+                dataColumnInitialIndex++;
+            }
+
+            for (var dataRowIndex = 0; dataRowIndex < dataRowCount; dataRowIndex++)
+            {
+                var row = table.NewRow();
+                row[0] = data[dataRowIndex].Ticker.Symbol; // stock symbol
+                row[1] = data[dataRowIndex].When.Day;
+                row[2] = data[dataRowIndex].When.Month;
+                row[3] = data[dataRowIndex].When.Year;
+
+                var dataColumnIndex = initialDataColumns.Count;
+
+                foreach (KeyValuePair<MarketDataType, double> entry in data[dataRowIndex].Data)
+                {
+                    row[dataColumnIndex] = entry.Value;
+                    dataColumnIndex++;
+                }
+
+                table.Rows.Add(row);
+            }
+
+            resultDataSet.Tables.Add(table);
+
+            return resultDataSet;
+        }
+
+        private static void CopyInitialColumnsToTable(DataTable table, List<DataColumn> initialDataColumns)
+        {
+            table.Columns.AddRange(initialDataColumns.ToArray());
+        }
+
+        private static void AddInitialColumn(List<DataColumn> initialDataColumns, string name, Type dataType)
+        {
+            DataColumn InitialColumn = null;
+            InitialColumn = new DataColumn(name);
+            InitialColumn.DataType = dataType;
+            initialDataColumns.Add(InitialColumn);
+        }
+    }
+}

EncogExtensions/Normalization/EncogAnalystExtensions.cs

@@ -0,0 +1,20 @@
+using Encog.App.Analyst;
+using Encog.App.Analyst.Analyze;
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace EncogExtensions.Normalization
+{
+    public static class EncogAnalystExtensions
+    {
+        public static void Analyze(this EncogAnalyst analyst, DataSet data)
+        {
+            var a = new PerformAnalysis(analyst.Script);
+            a.Process(analyst, data);// Kiran:2
+        }
+    }
+}