WIP. Added Candle regeneration from historic ticks if candle was wipe…

…d from cache. Not yet works as we need to fix GetBarTime() in situation where there is no candle at shift 0.

Buffer/BufferCandle.h

@@ -27,8 +27,8 @@
 // Includes.
 #include "../BufferStruct.mqh"
 #include "../Candle.struct.h"
-#include "../SerializerConverter.mqh"
-#include "../SerializerJson.mqh"
+#include "../Serializer/SerializerConverter.h"
+#include "../Serializer/SerializerJson.h"
 
 /**
  * Class to store struct data.

Indicator/IndicatorCandle.h

@@ -40,6 +40,7 @@
 #include "../Storage/ValueStorage.time.h"
 #include "../Storage/ValueStorage.volume.h"
 #include "Indicator.h"
+#include "IndicatorData.h"
 #include "TickBarCounter.h"
 
 // Indicator modes.
@@ -79,7 +80,6 @@ class IndicatorCandle : public Indicator<TS> {
   void Init() {
     // Along with indexing by shift, we can also index via timestamp!
     flags |= INDI_FLAG_INDEXABLE_BY_TIMESTAMP;
-    icdata.AddFlags(DICT_FLAG_FILL_HOLES_UNSORTED);
     icdata.SetOverflowListener(BufferStructOverflowListener, 10);
     Set<int>(STRUCT_ENUM(IndicatorDataParams, IDATA_PARAM_MAX_MODES), FINAL_INDI_CANDLE_MODE_ENTRY);
   }
@@ -109,6 +109,11 @@ class IndicatorCandle : public Indicator<TS> {
 
   /* Getters */
 
+  /**
+   * Returns buffer where candles are temporarily stored.
+   */
+  BufferCandle<TV>* GetCandlesBuffer() { return &icdata; }
+
   /**
    * Gets open price for a given, optional shift.
    */
@@ -165,28 +170,33 @@ class IndicatorCandle : public Indicator<TS> {
     return THIS_PTR;
   }
 
+  /**
+   * Removes candle from the buffer. Used mainly for testing purposes.
+   */
+  void InvalidateCandle(datetime _bar_time = 0) override {
+    if (_bar_time == 0) {
+      _bar_time = GetBarTime();
+    }
+
+    icdata.Unset(_bar_time);
+  }
+
   /**
    * Gets OHLC price values.
    */
   BarOHLC GetOHLC(int _shift = 0) override {
-    datetime _bar_time = GetBarTime(_shift);
-    BarOHLC _ohlc;
-
-    if ((long)_bar_time != 0) {
-      CandleOCTOHLC<TV> candle = icdata.GetByKey((long)_bar_time);
-      _ohlc.open = (float)candle.open;
-      _ohlc.high = (float)candle.high;
-      _ohlc.low = (float)candle.low;
-      _ohlc.close = (float)candle.close;
-      _ohlc.time = _bar_time;
-    }
+    IndicatorDataEntry _entry = GetEntry(_shift);
+    BarOHLC _bar(0, 0, 0, _entry.timestamp);
 
-#ifdef __debug_verbose__
-    Print("Fetching OHLC #", _shift, " from ", TimeToString(_ohlc.time, TIME_DATE | TIME_MINUTES | TIME_SECONDS));
-    Print("^- ", _ohlc.open, ", ", _ohlc.high, ", ", _ohlc.low, ", ", _ohlc.close);
-#endif
+    if (!_entry.IsValid()) {
+      return _bar;
+    }
 
-    return _ohlc;
+    _bar.open = _entry.GetValue<double>(INDI_CANDLE_MODE_PRICE_OPEN);
+    _bar.high = _entry.GetValue<double>(INDI_CANDLE_MODE_PRICE_HIGH);
+    _bar.low = _entry.GetValue<double>(INDI_CANDLE_MODE_PRICE_LOW);
+    _bar.close = _entry.GetValue<double>(INDI_CANDLE_MODE_PRICE_CLOSE);
+    return _bar;
   }
 
   /**
@@ -231,17 +241,26 @@ class IndicatorCandle : public Indicator<TS> {
     ResetLastError();
     int _ishift = _index >= 0 ? (int)_index : iparams.GetShift();
     long _candle_time = GetBarTime(_ishift);
-    CandleOCTOHLC<TV> _candle;
-    _candle = icdata.GetByKey(_candle_time);
+    long _candle_end_time = GetBarTime(_ishift - 1);
+
+    CandleOCTOHLC<TV> _candle = icdata.GetByKey(_candle_time);
 
     if (!_candle.IsValid()) {
-      // No candle found.
-      DebugBreak();
-      Print(GetFullName(), ": Missing candle at shift ", _index, " (",
-            TimeToString(_candle_time, TIME_DATE | TIME_MINUTES | TIME_SECONDS), "). Lowest timestamp in history is ",
-            icdata.GetMin());
+      // No candle found. Regenerating it.
+      GetTick() PTR_DEREF FetchHistory(_candle_time * 1000, _candle_end_time * 1000 - 1);
+      // At this point candle should be regenerated (or not) via
+      // OnDataSourceEntry() called from IndicatorTick.
+      _candle = icdata.GetByKey(_candle_time);
+
+      if (!_candle.IsValid()) {
+        // Candle wasn't regenerated. Maybe there is no history for that bar?
+        IndicatorDataEntry _entry = CandleToEntry(_candle_time, _candle);
+        _entry.AddFlags(INDI_ENTRY_FLAG_INSUFFICIENT_DATA);
+        return _entry;
+      }
     }
 
+    // At this point candle is filled with proper values.
     return CandleToEntry(_candle_time, _candle);
   }
 
@@ -440,4 +459,4 @@ class IndicatorCandle : public Indicator<TS> {
   /* Virtual methods */
 };
 
-#endif  // INDICATOR_CANDLE_H
+#endif

Indicator/IndicatorData.h

@@ -1432,6 +1432,16 @@ class IndicatorData : public IndicatorBase {
    */
   virtual long GetTickVolume(int _shift = 0) { return GetCandle() PTR_DEREF GetTickVolume(_shift); }
 
+  /**
+   * Removes candle from the buffer. Used mainly for testing purposes.
+   */
+  virtual void InvalidateCandle(datetime _bar_time = 0) { GetCandle() PTR_DEREF InvalidateCandle(_bar_time); }
+
+  /**
+   * Fetches historic ticks for a given range and emits these ticks. Used to regenerate candles.
+   */
+  virtual void FetchHistory(long _range_from, long _range_to) {}
+
   /**
    * Returns value storage of given kind.
    */

Indicator/IndicatorRenko.struct.h

@@ -31,7 +31,7 @@
 #endif
 
 // Includes.
-#include "../Indicator.struct.h"
+#include "Indicator.struct.h"
 
 /* Structure for IndicatorRenko class parameters. */
 struct IndicatorRenkoParams : IndicatorParams {

Indicator/IndicatorTick.h

@@ -68,7 +68,6 @@ class IndicatorTick : public Indicator<TS> {
     // We can only index via timestamp.
     flags |= INDI_FLAG_INDEXABLE_BY_TIMESTAMP;
 
-    itdata.AddFlags(DICT_FLAG_FILL_HOLES_UNSORTED);
     itdata.SetOverflowListener(BufferStructOverflowListener, 10);
     // Ask and Bid price.
     Set<int>(STRUCT_ENUM(IndicatorDataParams, IDATA_PARAM_MAX_MODES), 2);

Indicator/tests/IndicatorCandle.test.mq5

@@ -25,13 +25,47 @@
  */
 
 // Includes.
+#include "../../Platform.h"
 #include "../../Test.mqh"
 #include "../IndicatorCandle.h"
 
+Ref<IndicatorData> indi_candle;
+
 /**
  * Implements OnInit().
  */
 int OnInit() {
-  // @todo
-  return (INIT_SUCCEEDED);
+  Platform::Init();
+  Platform::Add(indi_candle = Platform::FetchDefaultCandleIndicator());
+  return _LastError == ERR_NO_ERROR ? INIT_SUCCEEDED : INIT_FAILED;
 }
+
+void OnTick() {
+  Platform::Tick();
+  if (Platform::IsNewHour()) {
+    // If a new hour occur, we check for a candle OHLCs, then we invalidate the
+    // candle and try to regenerate it by checking again the OHLCs.
+    BarOHLC _ohlc1 = indi_candle REF_DEREF GetOHLC();
+
+    // Now we invalidate current candle (candle will be removed from the IndicatorCandle's cache).
+    indi_candle REF_DEREF InvalidateCandle();
+
+    // Retrieving candle again.
+    BarOHLC _ohlc2 = indi_candle REF_DEREF GetOHLC();
+    assertEqualOrExit(
+        _ohlc2.time, _ohlc1.time,
+        "Difference between consecutive OHLC values after invalidating and then regenerating the candle!");
+    assertEqualOrExit(
+        _ohlc2.open, _ohlc1.open,
+        "Difference between consecutive OHLC values after invalidating and then regenerating the candle!");
+    assertEqualOrExit(
+        _ohlc2.high, _ohlc1.high,
+        "Difference between consecutive OHLC values after invalidating and then regenerating the candle!");
+    assertEqualOrExit(
+        _ohlc2.low, _ohlc1.low,
+        "Difference between consecutive OHLC values after invalidating and then regenerating the candle!");
+    assertEqualOrExit(
+        _ohlc2.close, _ohlc1.close,
+        "Difference between consecutive OHLC values after invalidating and then regenerating the candle!");
+  }
+}

Indicators/Tick/Indi_TickMt.mqh

@@ -107,6 +107,40 @@ class Indi_TickMt : public IndicatorTick<Indi_TickMtParams, double> {
     _fetch_history_on_first_tick = true;
   }
 
+  /**
+   * Fetches historic ticks for a given range and emits these ticks. Used to regenerate candles.
+   */
+  void FetchHistory(long _range_from, long _range_to) override {
+    // Number of retries for CopyTicksRange().
+    int _tries = 10;
+
+    static MqlTick _tmp_ticks[];
+    ArrayResize(_tmp_ticks, 0);
+
+    while (_tries > 0) {
+      int _num_copied = CopyTicksRange(GetSymbol(), _tmp_ticks, COPY_TICKS_INFO, _range_from, _range_to);
+
+      if (_num_copied == -1) {
+        ResetLastError();
+        Sleep(1000);
+        --_tries;
+      } else {
+        for (int i = 0; i < _num_copied; ++i) {
+          TickAB<double> _tick(_tmp_ticks[i].ask, _tmp_ticks[i].bid);
+#ifdef __debug_verbose__
+          Print("Emitting historic tick at ", TimeToString(_tmp_ticks[i].time, TIME_DATE | TIME_MINUTES | TIME_SECONDS),
+                ": ", _tmp_ticks[i].ask, ", ", _tmp_ticks[i].bid);
+#endif
+          EmitEntry(TickToEntry(_tmp_ticks[i].time, _tick));
+        }
+        break;
+      }
+    }
+  }
+
+  /**
+   * Fetches historic ticks for last two weeks and emits those ticks.
+   */
   void FetchHistory() {
     if (INDICATOR_TICK_REAL_FETCH_HISTORY == 0) {
       // No history requested.

Test.mqh

@@ -46,6 +46,13 @@
     return (ret);                                                             \
   }
 
+#define assertEqualOrExit(current, expected, msg)                             \
+  if ((current) != (expected)) {                                              \
+    Alert(msg + " - Assert fail. Expected ", expected, ", but got ", current, \
+          " in " + __FILE__ + ":" + (string)__LINE__);                        \
+    ExpertRemove();                                                           \
+  }
+
 #define assertFalseOrFail(cond, msg)                                                        \
   if ((cond)) {                                                                             \
     Alert(msg + " - Assert fail on " + #cond + " in " + __FILE__ + ":" + (string)__LINE__); \

Tick/TickManager.h

@@ -40,10 +40,7 @@ class TickManager : public BufferStruct<MqlTick> {
   /**
    * Init code (called on constructor).
    */
-  void Init() {
-    AddFlags(DICT_FLAG_FILL_HOLES_UNSORTED);
-    SetOverflowListener(TickManagerOverflowListener, 10);
-  }
+  void Init() { SetOverflowListener(BufferStructOverflowListener, 10); }
 
  public:
   /**
@@ -73,22 +70,4 @@ class TickManager : public BufferStruct<MqlTick> {
   // template <typename T>
   // void Set(STRUCT_ENUM(TickManagerParams, ENUM_TSM_PARAMS_PROP) _prop, T _value)
   // { params.Set<T>(_prop, _value); }
-
-  /* Other methods */
-
-  /**
-   * Function should return true if resize can be made, or false to overwrite current slot.
-   */
-  static bool TickManagerOverflowListener(ENUM_DICT_OVERFLOW_REASON _reason, int _size, int _num_conflicts) {
-    switch (_reason) {
-      case DICT_OVERFLOW_REASON_FULL:
-        // We allow resize if dictionary size is less than 86400 slots.
-        return _size < 86400;
-      case DICT_OVERFLOW_REASON_TOO_MANY_CONFLICTS:
-      default:
-        // When there is too many conflicts, we just reject doing resize, so first conflicting slot will be reused.
-        break;
-    }
-    return false;
-  }
 };