From ae41127ba28380129b312688b1d0ff782862f4d0 Mon Sep 17 00:00:00 2001 From: Adrian Kierzkowski Date: Wed, 14 Jul 2021 16:18:34 +0200 Subject: [PATCH] WIP. Messing with Serializer (fixes, features). --- EA.mqh | 16 +----- Serializer.enum.h | 1 - Serializer.mqh | 13 ++--- SerializerCsv.mqh | 57 ++++++++++---------- SerializerSql.mqh => SerializerSqlite.mqh | 49 +++++++++--------- Test.mqh | 63 +++++++++++++---------- tests/SerializerTest.mq5 | 2 +- 7 files changed, 93 insertions(+), 108 deletions(-) rename SerializerSql.mqh => SerializerSqlite.mqh (70%) diff --git a/EA.mqh b/EA.mqh index 82b4c1489..728aa6b1e 100644 --- a/EA.mqh +++ b/EA.mqh @@ -43,7 +43,7 @@ #include "SerializerConverter.mqh" #include "SerializerCsv.mqh" #include "SerializerJson.mqh" -#include "SerializerSql.mqh" +#include "SerializerSqlite.mqh" #include "Strategy.mqh" #include "SummaryReport.mqh" #include "Task.mqh" @@ -368,24 +368,12 @@ class EA { int _serializer_flags = SERIALIZER_FLAG_SKIP_HIDDEN | SERIALIZER_FLAG_INCLUDE_DYNAMIC; SerializerConverter _stub_chart = Serializer::MakeStubObject>(_serializer_flags); - - ChartEntry entry1; - BarOHLC ohlc1(1.2f, 1.21f, 1.19f, 1.20f, D '2025-01-01 10:00:00'); - entry1.bar = BarEntry(ohlc1); - - ChartEntry entry2; - BarOHLC ohlc2(1.23f, 1.20f, 1.15f, 1.23f, D '2025-01-01 10:00:10'); - entry2.bar = BarEntry(ohlc2); - - data_chart.Add(entry1, D '2025-01-01 10:00:00'); - data_chart.Add(entry2, D '2025-01-01 10:00:10'); - SerializerConverter csv = SerializerConverter::FromObject(data_chart, _serializer_flags); if ((_methods & EA_DATA_EXPORT_CSV) != 0) { csv.ToFile(_key_chart + ".csv", _serializer_flags, &_stub_chart); } else if ((_methods & EA_DATA_EXPORT_DB) != 0) { - SerializerSql::ConvertToFile(csv, _key_chart + ".sql", _serializer_flags, &_stub_chart); + SerializerSqlite::ConvertToFile(csv, _key_chart + ".sqlite", "chart", _serializer_flags, &_stub_chart); } } if ((_methods & EA_DATA_EXPORT_JSON) != 0) { diff --git a/Serializer.enum.h b/Serializer.enum.h index 4581d3900..6fc13444f 100644 --- a/Serializer.enum.h +++ b/Serializer.enum.h @@ -56,7 +56,6 @@ enum ENUM_SERIALIZER_FLAGS { /* Enumeration for serializer field flags. */ enum ENUM_SERIALIZER_FIELD_FLAGS { - SERIALIZER_FIELD_FLAG_UNSPECIFIED = 0, _SERIALIZER_FIELD_FLAGS_START = 1 << 16, SERIALIZER_FIELD_FLAG_HIDDEN = 1 << 16, SERIALIZER_FIELD_FLAG_DYNAMIC = 1 << 17, diff --git a/Serializer.mqh b/Serializer.mqh index 40ae862e0..710bab842 100644 --- a/Serializer.mqh +++ b/Serializer.mqh @@ -187,7 +187,7 @@ class Serializer { * Serializes or unserializes object. */ template - void PassObject(T& self, string name, V& value, unsigned int flags = SERIALIZER_FIELD_FLAG_UNSPECIFIED) { + void PassObject(T& self, string name, V& value, unsigned int flags = SERIALIZER_FIELD_FLAG_DEFAULT) { PassStruct(self, name, value, flags); } @@ -195,7 +195,7 @@ class Serializer { * Serializes or unserializes object that acts as a value. */ template - void PassValueObject(T& self, string name, V& value, unsigned int flags = SERIALIZER_FIELD_FLAG_UNSPECIFIED) { + void PassValueObject(T& self, string name, V& value, unsigned int flags = SERIALIZER_FIELD_FLAG_DEFAULT) { if (_mode == Serialize) { value.Serialize(this); fp_precision = SERIALIZER_DEFAULT_FP_PRECISION; @@ -210,11 +210,6 @@ class Serializer { } bool IsFieldVisible(int serializer_flags, int field_flags) { - if (field_flags == SERIALIZER_FIELD_FLAG_UNSPECIFIED) { - // Fields with unspecified flags are treated as visible. - return true; - } - // Is field visible? Such field cannot be exluded in anyway. if ((field_flags & SERIALIZER_FIELD_FLAG_VISIBLE) == SERIALIZER_FIELD_FLAG_VISIBLE) { return true; @@ -268,7 +263,7 @@ class Serializer { * Serializes or unserializes structure. */ template - void PassStruct(T& self, string name, V& value, unsigned int flags = SERIALIZER_FIELD_FLAG_UNSPECIFIED) { + void PassStruct(T& self, string name, V& value, unsigned int flags = SERIALIZER_FIELD_FLAG_DEFAULT) { if (_mode == Serialize) { if (!IsFieldVisible(_flags, flags)) return; } @@ -384,7 +379,7 @@ class Serializer { * Serializes or unserializes simple value. */ template - SerializerNode* Pass(T& self, string name, V& value, unsigned int flags = SERIALIZER_FIELD_FLAG_UNSPECIFIED) { + SerializerNode* Pass(T& self, string name, V& value, unsigned int flags = SERIALIZER_FIELD_FLAG_DEFAULT) { SerializerNode* child = NULL; bool _skip_push = (_flags & SERIALIZER_FLAG_SKIP_PUSH) == SERIALIZER_FLAG_SKIP_PUSH; diff --git a/SerializerCsv.mqh b/SerializerCsv.mqh index c39e9408f..ee9aa56f8 100644 --- a/SerializerCsv.mqh +++ b/SerializerCsv.mqh @@ -48,7 +48,7 @@ enum ENUM_SERIALIZER_CSV_FLAGS { class SerializerCsv { public: - static string Stringify(SerializerNode* _root, unsigned int serializer_flags = 0, void* serializer_aux_arg = NULL, + static string Stringify(SerializerNode* _root, unsigned int serializer_flags, void* serializer_aux_arg = NULL, MiniMatrix2d* _matrix_out = NULL, MiniMatrix2d* _column_types_out = NULL) { SerializerConverter* _stub = (SerializerConverter*)serializer_aux_arg; @@ -80,6 +80,7 @@ class SerializerCsv { } MiniMatrix2d _cells; + MiniMatrix2d _column_titles; MiniMatrix2d _column_types; if (_matrix_out == NULL) { @@ -93,6 +94,16 @@ class SerializerCsv { _matrix_out.Resize(_num_columns, _num_rows); _column_types_out.Resize(_num_columns, 1); + if (_include_titles) { + _column_titles.Resize(_num_columns, 1); + int _titles_current_column = 0; + SerializerCsv::ExtractColumns(_stub.Node(), &_column_titles, _column_types_out, serializer_flags, + _titles_current_column); + for (int x = 0; x < _matrix_out.SizeX(); ++x) { + _matrix_out.Set(x, 0, _column_titles.Get(x, 0)); + } + } + #ifdef __debug__ Print("Stub: ", _stub.Node().ToString()); Print("Data: ", _root.ToString()); @@ -142,6 +153,22 @@ class SerializerCsv { return "\"" + _result + "\""; } + /** + * Extracts column names and types from the stub, so even if there is not data, we'll still have information about + * columns. + */ + static void ExtractColumns(SerializerNode* _stub, MiniMatrix2d* _titles, + MiniMatrix2d* _column_types, int _flags, int& _column) { + for (unsigned int _stub_entry_idx = 0; _stub_entry_idx < _stub.NumChildren(); ++_stub_entry_idx) { + SerializerNode* _child = _stub.GetChild(_stub_entry_idx); + if (_child.IsContainer()) { + ExtractColumns(_child, _titles, _column_types, _flags, _column); + } else if (_child.HasKey()) { + _titles.Set(_column++, 0, _child.Key()); + } + } + } + /** * */ @@ -239,34 +266,6 @@ class SerializerCsv { _column_types.Set(_column, 0, _data.GetValueParam().GetType()); } - if (_include_titles && StringLen(_cells.Get(_column, _row - 1)) == 0) { - if (_include_titles_tree) { - // Creating fully qualified title. - string _fqt = ""; - - bool _include_key = bool(_flags & SERIALIZER_CSV_INCLUDE_KEY); - - for (SerializerNode* node = _data; node != NULL; node = node.GetParent()) { - if (_include_key && (node.GetParent() == NULL || node.GetParent().GetParent() == NULL)) { - // Key of the root element is already included in the first column. - break; - } - string key = node.HasKey() ? node.Key() : IntegerToString(node.Index()); - if (key != "") { - if (_fqt == "") { - _fqt = key; - } else { - _fqt = key + "." + _fqt; - } - } - } - _cells.Set(_column, 0, EscapeString(_fqt)); - } else { - string title = _data.HasKey() ? _data.Key() : ""; - _cells.Set(_column, 0, EscapeString(title)); - } - } - _cells.Set(_column, _row, ParamToString(_data.GetValueParam())); } diff --git a/SerializerSql.mqh b/SerializerSqlite.mqh similarity index 70% rename from SerializerSql.mqh rename to SerializerSqlite.mqh index 3afb54f6e..5e73571af 100644 --- a/SerializerSql.mqh +++ b/SerializerSqlite.mqh @@ -29,7 +29,7 @@ #include "SerializerConverter.mqh" #include "SerializerCsv.mqh" -class SerializerSql { +class SerializerSqlite { public: static ENUM_DATATYPE CsvParamTypeToSqlType(SerializerNodeParamType _param_type) { switch (_param_type) { @@ -42,46 +42,43 @@ class SerializerSql { case SerializerNodeParamString: return TYPE_STRING; } - Print("Error: CsvParamTypeToSqlType: wrong _param_type parameter!"); + Print("Error: CsvParamTypeToSqlType: wrong _param_type parameter! Got ", _param_type, "."); DebugBreak(); return (ENUM_DATATYPE)-1; } - static string Convert(SerializerConverter& source, unsigned int _stringify_flags = 0, void* _stub = NULL) { + static bool ConvertToFile(SerializerConverter& source, string _path, string _table, unsigned int _stringify_flags = 0, + void* _stub = NULL) { // We must have titles tree as MiniMatrix2d _matrix_out; MiniMatrix2d _column_types; string _csv = SerializerCsv::Stringify(source.root_node, _stringify_flags | SERIALIZER_CSV_INCLUDE_TITLES, _stub, &_matrix_out, &_column_types); - Database _db("test.sqlite"); - DatabaseTableSchema _schema; + Database _db(_path); int i; - for (i = 0; i < _matrix_out.SizeX(); ++i) { - DatabaseTableColumnEntry _column; - _column.name = _matrix_out.Get(i, 0); - _column.type = CsvParamTypeToSqlType(_column_types.Get(i, 0)); - _column.flags = 0; - _column.char_size = 0; - _schema.AddColumn(_column); - } - - _db.CreateTableIfNotExist("bla", _schema); + if (!_db.TableExists(_table)) { + DatabaseTableSchema _schema; + for (i = 0; i < _matrix_out.SizeX(); ++i) { + DatabaseTableColumnEntry _column; + _column.name = _matrix_out.Get(i, 0); + _column.type = CsvParamTypeToSqlType(_column_types.Get(i, 0)); + _column.flags = 0; + _column.char_size = 0; + _schema.AddColumn(_column); + } - _db.ImportData("bla", _matrix_out); + if (!_db.CreateTable(_table, _schema)) { + return false; + } + } - Print("Conversion from: \n", _csv); - return "INSERT INTO ..."; - } + if (!_db.ImportData(_table, _matrix_out)) { + return false; + } - /** - * Converts object into CSV and then SQL. Thus way we don't duplicate CSV serializer's code. - */ - static bool ConvertToFile(SerializerConverter& source, string _path, unsigned int _stringify_flags = 0, - void* _stub = NULL) { - string _data = Convert(source, _stringify_flags, _stub); - return File::SaveFile(_path, _data); + return true; } }; diff --git a/Test.mqh b/Test.mqh index 5caae14d4..fd280471d 100644 --- a/Test.mqh +++ b/Test.mqh @@ -26,44 +26,51 @@ */ // Define an assert macros. -#define assertTrueOrFail(cond, msg) \ - if (!(cond)) { \ - Alert(msg + " - Assert fail on " + #cond + " in " + __FILE__ + ":" + (string) __LINE__); \ - return (INIT_FAILED); \ +#define assertTrueOrFail(cond, msg) \ + if (!(cond)) { \ + Alert(msg + " - Assert fail on " + #cond + " in " + __FILE__ + ":" + (string)__LINE__); \ + return (INIT_FAILED); \ } -#define assertFalseOrFail(cond, msg) \ - if ((cond)) { \ - Alert(msg + " - Assert fail on " + #cond + " in " + __FILE__ + ":" + (string) __LINE__); \ - return (INIT_FAILED); \ +#define assertEqualOrFail(current, expected, msg) \ + if ((current) != (expected)) { \ + Alert(msg + " - Assert fail. Expected ", expected, ", but got ", current, \ + " in " + __FILE__ + ":" + (string)__LINE__); \ + return (INIT_FAILED); \ } -#define assertTrueOrReturn(cond, msg, ret) \ - if (!(cond)) { \ - Alert(msg + " - Assert fail on " + #cond + " in " + __FILE__ + ":" + (string) __LINE__); \ - return (ret); \ +#define assertFalseOrFail(cond, msg) \ + if ((cond)) { \ + Alert(msg + " - Assert fail on " + #cond + " in " + __FILE__ + ":" + (string)__LINE__); \ + return (INIT_FAILED); \ } -#define assertTrueOrReturnFalse(cond, msg) \ - if (!(cond)) { \ - Alert(msg + " - Assert fail on " + #cond + " in " + __FILE__ + ":" + (string) __LINE__); \ - return (false); \ +#define assertTrueOrReturn(cond, msg, ret) \ + if (!(cond)) { \ + Alert(msg + " - Assert fail on " + #cond + " in " + __FILE__ + ":" + (string)__LINE__); \ + return (ret); \ } -#define assertFalseOrReturn(cond, msg, ret) \ - if ((cond)) { \ - Alert(msg + " - Assert fail on " + #cond + " in " + __FILE__ + ":" + (string) __LINE__); \ - return (ret); \ +#define assertTrueOrReturnFalse(cond, msg) \ + if (!(cond)) { \ + Alert(msg + " - Assert fail on " + #cond + " in " + __FILE__ + ":" + (string)__LINE__); \ + return (false); \ } -#define assertTrueOrExit(cond, msg) \ - if (!(cond)) { \ - Alert(msg + " - Assert fail on " + #cond + " in " + __FILE__ + ":" + (string) __LINE__); \ - ExpertRemove(); \ +#define assertFalseOrReturn(cond, msg, ret) \ + if ((cond)) { \ + Alert(msg + " - Assert fail on " + #cond + " in " + __FILE__ + ":" + (string)__LINE__); \ + return (ret); \ } -#define assertFalseOrExit(cond, msg) \ - if ((cond)) { \ - Alert(msg + " - Assert fail on " + #cond + " in " + __FILE__ + ":" + (string) __LINE__); \ - ExpertRemove(); \ +#define assertTrueOrExit(cond, msg) \ + if (!(cond)) { \ + Alert(msg + " - Assert fail on " + #cond + " in " + __FILE__ + ":" + (string)__LINE__); \ + ExpertRemove(); \ + } + +#define assertFalseOrExit(cond, msg) \ + if ((cond)) { \ + Alert(msg + " - Assert fail on " + #cond + " in " + __FILE__ + ":" + (string)__LINE__); \ + ExpertRemove(); \ } diff --git a/tests/SerializerTest.mq5 b/tests/SerializerTest.mq5 index 72c250afa..6777f3deb 100644 --- a/tests/SerializerTest.mq5 +++ b/tests/SerializerTest.mq5 @@ -340,7 +340,7 @@ int OnInit() { assertTrueOrFail(subentry_none_json == "{\"x\":1,\"y\":2,\"dynamic\":3,\"feature\":4}", "Serializer flags not obeyed!"); - assertTrueOrFail(subentry_dynamic_json == "{\"dynamic\":3}", "Serializer flags not obeyed!"); + assertEqualOrFail(subentry_dynamic_json, "{\"dynamic\":3}", "Serializer flags not obeyed!"); assertTrueOrFail(subentry_feature_json == "{\"feature\":4}", "Serializer flags not obeyed!"); assertTrueOrFail(subentry_dynamic_feature_json == "{\"dynamic\":3,\"feature\":4}", "Serializer flags not obeyed!");