From baa49424a9800639ff5136172e9fd0f7baad8169 Mon Sep 17 00:00:00 2001 From: Adrian Kierzkowski Date: Wed, 24 Aug 2022 14:17:51 +0200 Subject: [PATCH] WIP. Fixes Dict's logic inside InsertInto(). Also changed the way Overflow Listener works (enum items). Got rid of unnecessary overflow listeners in some classes. --- Buffer/BufferCandle.h | 22 ++---- Buffer/BufferTick.h | 21 +----- BufferStruct.mqh | 14 +++- Candle.struct.h | 17 +++++ Dict.enum.h | 6 +- Dict.mqh | 136 +++++++++++++++++++---------------- DictBase.mqh | 12 ---- DictObject.mqh | 138 ++++++++++++++++++++---------------- DictStruct.mqh | 132 ++++++++++++++++++---------------- Indicator/IndicatorCandle.h | 18 +---- Indicator/IndicatorRenko.h | 18 ++++- Indicator/IndicatorTick.h | 20 +----- tests/DictTest.mq5 | 15 ++-- 13 files changed, 287 insertions(+), 282 deletions(-) diff --git a/Buffer/BufferCandle.h b/Buffer/BufferCandle.h index 1bba5ba45..9ff42dc75 100644 --- a/Buffer/BufferCandle.h +++ b/Buffer/BufferCandle.h @@ -27,6 +27,8 @@ // Includes. #include "../BufferStruct.mqh" #include "../Candle.struct.h" +#include "../SerializerConverter.mqh" +#include "../SerializerJson.mqh" /** * Class to store struct data. @@ -41,7 +43,7 @@ class BufferCandle : public BufferStruct> { * * Called on constructor. */ - void Init() { SetOverflowListener(BufferCandleOverflowListener, 10); } + void Init() { SetOverflowListener(BufferStructOverflowListener, 10); } public: /* Constructors */ @@ -55,24 +57,10 @@ class BufferCandle : public BufferStruct> { Init(); } - /* Callback methods */ - /** - * Function should return true if resize can be made, or false to overwrite current slot. + * Returns JSON representation of the buffer. */ - static bool BufferCandleOverflowListener(ENUM_DICT_OVERFLOW_REASON _reason, int _size, int _num_conflicts) { - static int cache_limit = 86400; - switch (_reason) { - case DICT_OVERFLOW_REASON_FULL: - // We allow resize if dictionary size is less than 86400 slots. - return _size < cache_limit; - case DICT_OVERFLOW_REASON_TOO_MANY_CONFLICTS: - default: - // When there is too many conflicts, we just reject doing resize, so first conflicting slot will be reused. - break; - } - return false; - } + string ToJSON() { return SerializerConverter::FromObject(THIS_REF).ToString(); } }; #endif // BUFFER_CANDLE_H diff --git a/Buffer/BufferTick.h b/Buffer/BufferTick.h index 4eed02d69..fdfc5d901 100644 --- a/Buffer/BufferTick.h +++ b/Buffer/BufferTick.h @@ -109,7 +109,7 @@ class BufferTick : public BufferStruct> { _vs_spread = NULL; _vs_volume = NULL; _vs_tick_volume = NULL; - SetOverflowListener(BufferTickOverflowListener, 10); + SetOverflowListener(BufferStructOverflowListener, 10); } public: @@ -210,25 +210,6 @@ class BufferTick : public BufferStruct> { // Convert to OHLC in upper method return NULL; } - - /* Callback methods */ - - /** - * Function should return true if resize can be made, or false to overwrite current slot. - */ - static bool BufferTickOverflowListener(ENUM_DICT_OVERFLOW_REASON _reason, int _size, int _num_conflicts) { - static int cache_limit = 86400; - switch (_reason) { - case DICT_OVERFLOW_REASON_FULL: - // We allow resize if dictionary size is less than 86400 slots. - return _size < cache_limit; - case DICT_OVERFLOW_REASON_TOO_MANY_CONFLICTS: - default: - // When there is too many conflicts, we just reject doing resize, so first conflicting slot will be reused. - break; - } - return false; - } }; #endif // BUFFER_TICK_H diff --git a/BufferStruct.mqh b/BufferStruct.mqh index c02eba371..bd16b9e54 100644 --- a/BufferStruct.mqh +++ b/BufferStruct.mqh @@ -35,8 +35,18 @@ * @see DictBase */ bool BufferStructOverflowListener(ENUM_DICT_OVERFLOW_REASON _reason, int _size, int _num_conflicts) { - // We allow resize if dictionary size is less than 10000 slots. - return _size < 10000; + static int cache_limit = 86400; + switch (_reason) { + case DICT_LISTENER_FULL_CAN_RESIZE: + case DICT_LISTENER_NOT_PERFORMANT_CAN_RESIZE: + // We allow resize if dictionary size is less than 86400 slots. + return _size < cache_limit; + case DICT_LISTENER_CONFLICTS_CAN_OVERWRITE: + // We start to overwrite slots when we can't make dict bigger and there is at least 10 consecutive conflicts while + // inserting new value. + return _size >= cache_limit && _num_conflicts >= 10; + } + return true; } /** diff --git a/Candle.struct.h b/Candle.struct.h index f7034a1f3..0f9bfa3af 100644 --- a/Candle.struct.h +++ b/Candle.struct.h @@ -262,6 +262,9 @@ struct CandleOCTOHLC : CandleOHLC { // Returns timestamp of close price. long GetCloseTimestamp() { return close_timestamp; } + // Serializers. + SerializerNodeType Serialize(Serializer &s); + // Returns text representation of candle. string ToString() { return StringFormat("%.5f %.5f %.5f %.5f [%s] @ %s - %s", open, high, low, close, @@ -309,3 +312,17 @@ SerializerNodeType CandleTOHLC::Serialize(Serializer &s) { s.Pass(THIS_REF, "close", close, SERIALIZER_FIELD_FLAG_DYNAMIC); return SerializerNodeObject; } + +/* Method to serialize CandleEntry structure. */ +template +SerializerNodeType CandleOCTOHLC::Serialize(Serializer &s) { + s.Pass(THIS_REF, "is_complete", is_complete, SERIALIZER_FIELD_FLAG_DYNAMIC); + s.Pass(THIS_REF, "open_timestamp", open_timestamp, SERIALIZER_FIELD_FLAG_DYNAMIC); + s.Pass(THIS_REF, "close_timestamp", close_timestamp, SERIALIZER_FIELD_FLAG_DYNAMIC); + s.Pass(THIS_REF, "open", open, SERIALIZER_FIELD_FLAG_DYNAMIC); + s.Pass(THIS_REF, "high", high, SERIALIZER_FIELD_FLAG_DYNAMIC); + s.Pass(THIS_REF, "low", low, SERIALIZER_FIELD_FLAG_DYNAMIC); + s.Pass(THIS_REF, "close", close, SERIALIZER_FIELD_FLAG_DYNAMIC); + s.Pass(THIS_REF, "volume", volume, SERIALIZER_FIELD_FLAG_DYNAMIC); + return SerializerNodeObject; +} diff --git a/Dict.enum.h b/Dict.enum.h index 080a50271..a56671b41 100644 --- a/Dict.enum.h +++ b/Dict.enum.h @@ -42,8 +42,10 @@ enum DictMode { DictModeUnknown, DictModeDict, DictModeList }; * Reason of call to overflow listener. */ enum ENUM_DICT_OVERFLOW_REASON { - DICT_OVERFLOW_REASON_FULL, - DICT_OVERFLOW_REASON_TOO_MANY_CONFLICTS, + DICT_LISTENER_FULL_CAN_RESIZE, // Dict is full. Can we grow up the dict? + DICT_LISTENER_NOT_PERFORMANT_CAN_RESIZE, // Dict is not performant (too many average number of conflicts). Can we + // grow up the dict? + DICT_LISTENER_CONFLICTS_CAN_OVERWRITE // Conflict(s) when inserting new slot. Can we overwrite random used slot? }; /** diff --git a/Dict.mqh b/Dict.mqh index 743cc92d1..5e4e317c5 100644 --- a/Dict.mqh +++ b/Dict.mqh @@ -206,94 +206,108 @@ class Dict : public DictBase { * Inserts value into given array of DictSlots. */ bool InsertInto(DictSlotsRef& dictSlotsRef, const K key, V value, bool allow_resize) { - if (_mode == DictModeUnknown) - _mode = DictModeDict; - else if (_mode != DictModeDict) { + if (THIS_ATTR _mode == DictModeUnknown) + THIS_ATTR _mode = DictModeDict; + else if (THIS_ATTR _mode != DictModeDict) { Alert("Warning: Dict already operates as a list, not a dictionary!"); return false; } unsigned int position; - DictSlot* keySlot = GetSlotByKey(dictSlotsRef, key, position); + DictSlot* _slot = THIS_ATTR GetSlotByKey(dictSlotsRef, key, position); - if (keySlot == NULL && !IsGrowUpAllowed()) { - // Resize is prohibited, so we will just overwrite some slot. - allow_resize = false; + // If we have a slot then we can overwrite it. + if (_slot != NULL) { + WriteSlot(_slot, key, value, DICT_SLOT_HAS_KEY | DICT_SLOT_IS_USED | DICT_SLOT_WAS_USED); + // We're done, we don't have to increment number of slots used. + return true; } - if (allow_resize) { - // Will resize dict if there were performance problems before or there is no slots. - if (IsGrowUpAllowed() && !dictSlotsRef.IsPerformant()) { + // If we don't have a slot then we should consider growing up number of slots or overwrite some existing slot. + + bool _is_performant = dictSlotsRef.IsPerformant(); // Whether there is no performance problems. + bool _is_full = + dictSlotsRef._num_used == ArraySize(dictSlotsRef.DictSlots); // Whether we don't have empty slots to use. + + if ((_is_full || !_is_performant) && allow_resize) { + // We have to resize the dict as it is either full or have perfomance problems due to massive number of conflicts + // when inserting new values. + if (overflow_listener == NULL) { + // There is no overflow listener so we can freely grow up the dict. if (!GrowUp()) { + // Can't resize the dict. Error happened. return false; } - // We now have new positions of slots, so we have to take the corrent slot again. - keySlot = GetSlotByKey(dictSlotsRef, key, position); - } - - if (keySlot == NULL && dictSlotsRef._num_used == ArraySize(dictSlotsRef.DictSlots)) { - // No DictSlotsRef.DictSlots available. - if (overflow_listener != NULL) { - if (!overflow_listener(DICT_OVERFLOW_REASON_FULL, dictSlotsRef._num_used, 0)) { - // Overwriting slot pointed exactly by key's position in the hash table (we don't check for possible - // conflicts). - keySlot = &dictSlotsRef.DictSlots[Hash(key) % ArraySize(dictSlotsRef.DictSlots)]; + } else { + // Overflow listener will decide if we can grow up the dict. + if (overflow_listener(_is_full ? DICT_LISTENER_FULL_CAN_RESIZE : DICT_LISTENER_NOT_PERFORMANT_CAN_RESIZE, + dictSlotsRef._num_used, 0)) { + // We can freely grow up the dict. + if (!GrowUp()) { + // Can't resize the dict. Error happened. + return false; } } - - if (keySlot == NULL) { - // We need to expand array of DictSlotsRef.DictSlots (by 25% by default). - if (!GrowUp()) return false; - } } } - if (keySlot == NULL) { - position = Hash(key) % ArraySize(dictSlotsRef.DictSlots); - - unsigned int _starting_position = position; - int _num_conflicts = 0; - bool _overwrite_slot = false; - - // Searching for empty DictSlot or used one with the matching key. It skips used, hashless DictSlots. - while (dictSlotsRef.DictSlots[position].IsUsed() && - (!dictSlotsRef.DictSlots[position].HasKey() || dictSlotsRef.DictSlots[position].key != key)) { - if (overflow_listener_max_conflicts != 0 && ++_num_conflicts == overflow_listener_max_conflicts) { - if (overflow_listener != NULL) { - if (!overflow_listener(DICT_OVERFLOW_REASON_TOO_MANY_CONFLICTS, dictSlotsRef._num_used, _num_conflicts)) { - // Overflow listener returned false so we won't search for further empty slot. - _overwrite_slot = true; - break; - } - } else { - // Even if there is no overflow listener function, we stop searching for further empty slot as maximum - // number of conflicts has been reached. - _overwrite_slot = true; - break; - } - } + // At this point we have at least one free slot and we won't be doing any dict's grow up in the loop where we search + // for an empty slot. + + // Position we will start from in order to search free slot. + position = THIS_ATTR Hash(key) % ArraySize(dictSlotsRef.DictSlots); - // Position may overflow, so we will start from the beginning. - position = (position + 1) % ArraySize(dictSlotsRef.DictSlots); + // Saving position for further, possible overwrite. + unsigned int _starting_position = position; + + // How many times we had to skip slot as it was already occupied. + unsigned int _num_conflicts = 0; + + // Searching for empty DictSlot or used one with the matching key. It skips used, hashless DictSlots. + while (dictSlotsRef.DictSlots[position].IsUsed() && + (!dictSlotsRef.DictSlots[position].HasKey() || dictSlotsRef.DictSlots[position].key != key)) { + ++_num_conflicts; + + if (overflow_listener == NULL) { + // There is no overflow listener, so we can't overwrite a slot. We will be looping until we find empty slot. + continue; } - if (_overwrite_slot) { - // Overwriting starting position for faster further lookup. + // We had to skip slot as it is already occupied. Now we are checking if + // there is too many conflicts/skips and thus we can overwrite slot in + // the starting position. + if (overflow_listener(DICT_LISTENER_CONFLICTS_CAN_OVERWRITE, dictSlotsRef._num_used, _num_conflicts)) { + // Looks like dict is working as buffer and we can overwrite slot in the starting position. position = _starting_position; - } else if (!dictSlotsRef.DictSlots[position].IsUsed()) { - // If slot isn't already used then we increment number of used slots. - ++dictSlotsRef._num_used; + break; } - dictSlotsRef.AddConflicts(_num_conflicts); + // Position may overflow, so we will start from the beginning. + position = (position + 1) % ArraySize(dictSlotsRef.DictSlots); } - dictSlotsRef.DictSlots[position].key = key; - dictSlotsRef.DictSlots[position].value = value; - dictSlotsRef.DictSlots[position].SetFlags(DICT_SLOT_HAS_KEY | DICT_SLOT_IS_USED | DICT_SLOT_WAS_USED); + // Acknowledging slots array about number of conflicts as it calculates average number of conflicts per insert. + dictSlotsRef.AddConflicts(_num_conflicts); + + // Incrementing number of slots used only if we're writing into empty slot. + if (!dictSlotsRef.DictSlots[position].IsUsed()) { + ++dictSlotsRef._num_used; + } + + // Writing slot in the position of empty slot or, when overwriting, in starting position. + WriteSlot(dictSlotsRef.DictSlots[position], key, value, DICT_SLOT_HAS_KEY | DICT_SLOT_IS_USED | DICT_SLOT_WAS_USED); return true; } + /*** + * Writes slot with given key, value and flags. + */ + void WriteSlot(DictSlot& _slot, const K _key, V _value, unsigned char _slot_flags) { + _slot.key = _key; + _slot.value = _value; + _slot.SetFlags(_slot_flags); + } + /** * Inserts hashless value into given array of DictSlots. */ diff --git a/DictBase.mqh b/DictBase.mqh index e7f0b8a75..718d78359 100644 --- a/DictBase.mqh +++ b/DictBase.mqh @@ -236,18 +236,6 @@ class DictBase { // No key found. } - /** - * Checks whether overflow listener allows dict to grow up. - */ - bool IsGrowUpAllowed() { - if (overflow_listener == NULL) { - return true; - } - - // Checking if overflow listener allows resize from current to higher number of slots. - return overflow_listener(DICT_OVERFLOW_REASON_FULL, Size(), 0); - } - /** * Moves last slot to given one to fill the hole after removing the value. */ diff --git a/DictObject.mqh b/DictObject.mqh index 6f7519766..2ed26024a 100644 --- a/DictObject.mqh +++ b/DictObject.mqh @@ -210,96 +210,108 @@ class DictObject : public DictBase { * Inserts value into given array of DictSlots. */ bool InsertInto(DictSlotsRef& dictSlotsRef, const K key, V& value, bool allow_resize) { - if (this PTR_DEREF _mode == DictModeUnknown) - this PTR_DEREF _mode = DictModeDict; - else if (this PTR_DEREF _mode != DictModeDict) { + if (THIS_ATTR _mode == DictModeUnknown) + THIS_ATTR _mode = DictModeDict; + else if (THIS_ATTR _mode != DictModeDict) { Alert("Warning: Dict already operates as a list, not a dictionary!"); return false; } unsigned int position; - DictSlot* keySlot = this PTR_DEREF GetSlotByKey(dictSlotsRef, key, position); + DictSlot* _slot = THIS_ATTR GetSlotByKey(dictSlotsRef, key, position); - if (keySlot == NULL && !this PTR_DEREF IsGrowUpAllowed()) { - // Resize is prohibited, so we will just overwrite some slot. - allow_resize = false; + // If we have a slot then we can overwrite it. + if (_slot != NULL) { + WriteSlot(_slot, key, value, DICT_SLOT_HAS_KEY | DICT_SLOT_IS_USED | DICT_SLOT_WAS_USED); + // We're done, we don't have to increment number of slots used. + return true; } - if (allow_resize) { - // Will resize dict if there were performance problems before or there is no slots. - if (this PTR_DEREF IsGrowUpAllowed() && !dictSlotsRef.IsPerformant()) { + // If we don't have a slot then we should consider growing up number of slots or overwrite some existing slot. + + bool _is_performant = dictSlotsRef.IsPerformant(); // Whether there is no performance problems. + bool _is_full = + dictSlotsRef._num_used == ArraySize(dictSlotsRef.DictSlots); // Whether we don't have empty slots to use. + + if ((_is_full || !_is_performant) && allow_resize) { + // We have to resize the dict as it is either full or have perfomance problems due to massive number of conflicts + // when inserting new values. + if (overflow_listener == NULL) { + // There is no overflow listener so we can freely grow up the dict. if (!GrowUp()) { + // Can't resize the dict. Error happened. return false; } - // We now have new positions of slots, so we have to take the corrent slot again. - keySlot = this PTR_DEREF GetSlotByKey(dictSlotsRef, key, position); - } - - if (keySlot == NULL && dictSlotsRef._num_used == ArraySize(dictSlotsRef.DictSlots)) { - // No DictSlotsRef.DictSlots available. - if (this PTR_DEREF overflow_listener != NULL) { - if (!this PTR_DEREF overflow_listener(DICT_OVERFLOW_REASON_FULL, dictSlotsRef._num_used, 0)) { - // Overwriting slot pointed exactly by key's position in the hash table (we don't check for possible - // conflicts). - keySlot = &dictSlotsRef.DictSlots[this PTR_DEREF Hash(key) % ArraySize(dictSlotsRef.DictSlots)]; + } else { + // Overflow listener will decide if we can grow up the dict. + if (overflow_listener(_is_full ? DICT_LISTENER_FULL_CAN_RESIZE : DICT_LISTENER_NOT_PERFORMANT_CAN_RESIZE, + dictSlotsRef._num_used, 0)) { + // We can freely grow up the dict. + if (!GrowUp()) { + // Can't resize the dict. Error happened. + return false; } } - - if (keySlot == NULL) { - // We need to expand array of DictSlotsRef.DictSlots. - if (!GrowUp()) return false; - } } } - if (keySlot == NULL) { - position = this PTR_DEREF Hash(key) % ArraySize(dictSlotsRef.DictSlots); - - unsigned int _starting_position = position; - unsigned int _num_conflicts = 0; - bool _overwrite_slot = false; - - // Searching for empty DictSlot or used one with the matching key. It skips used, hashless DictSlots. - while (dictSlotsRef.DictSlots[position].IsUsed() && - (!dictSlotsRef.DictSlots[position].HasKey() || dictSlotsRef.DictSlots[position].key != key)) { - if (this PTR_DEREF overflow_listener_max_conflicts != 0 && - ++_num_conflicts == this PTR_DEREF overflow_listener_max_conflicts) { - if (this PTR_DEREF overflow_listener != NULL) { - if (!this PTR_DEREF overflow_listener(DICT_OVERFLOW_REASON_TOO_MANY_CONFLICTS, dictSlotsRef._num_used, - _num_conflicts)) { - // Overflow listener returned false so we won't search for further empty slot PTR_DEREF - _overwrite_slot = true; - break; - } - } else { - // Even if there is no overflow listener function, we stop searching for further empty slot as maximum - // number of conflicts has been reached. - _overwrite_slot = true; - break; - } - } + // At this point we have at least one free slot and we won't be doing any dict's grow up in the loop where we search + // for an empty slot. + + // Position we will start from in order to search free slot. + position = THIS_ATTR Hash(key) % ArraySize(dictSlotsRef.DictSlots); - // Position may overflow, so we will start from the beginning. - position = (position + 1) % ArraySize(dictSlotsRef.DictSlots); + // Saving position for further, possible overwrite. + unsigned int _starting_position = position; + + // How many times we had to skip slot as it was already occupied. + unsigned int _num_conflicts = 0; + + // Searching for empty DictSlot or used one with the matching key. It skips used, hashless DictSlots. + while (dictSlotsRef.DictSlots[position].IsUsed() && + (!dictSlotsRef.DictSlots[position].HasKey() || dictSlotsRef.DictSlots[position].key != key)) { + ++_num_conflicts; + + if (overflow_listener == NULL) { + // There is no overflow listener, so we can't overwrite a slot. We will be looping until we find empty slot. + continue; } - if (_overwrite_slot) { - // Overwriting starting position for faster further lookup. + // We had to skip slot as it is already occupied. Now we are checking if + // there is too many conflicts/skips and thus we can overwrite slot in + // the starting position. + if (overflow_listener(DICT_LISTENER_CONFLICTS_CAN_OVERWRITE, dictSlotsRef._num_used, _num_conflicts)) { + // Looks like dict is working as buffer and we can overwrite slot in the starting position. position = _starting_position; - } else if (!dictSlotsRef.DictSlots[position].IsUsed()) { - // If slot isn't already used then we increment number of used slots. - ++dictSlotsRef._num_used; + break; } - dictSlotsRef.AddConflicts(_num_conflicts); + // Position may overflow, so we will start from the beginning. + position = (position + 1) % ArraySize(dictSlotsRef.DictSlots); } - dictSlotsRef.DictSlots[position].key = key; - dictSlotsRef.DictSlots[position].value = value; - dictSlotsRef.DictSlots[position].SetFlags(DICT_SLOT_HAS_KEY | DICT_SLOT_IS_USED | DICT_SLOT_WAS_USED); + // Acknowledging slots array about number of conflicts as it calculates average number of conflicts per insert. + dictSlotsRef.AddConflicts(_num_conflicts); + + // Incrementing number of slots used only if we're writing into empty slot. + if (!dictSlotsRef.DictSlots[position].IsUsed()) { + ++dictSlotsRef._num_used; + } + + // Writing slot in the position of empty slot or, when overwriting, in starting position. + WriteSlot(dictSlotsRef.DictSlots[position], key, value, DICT_SLOT_HAS_KEY | DICT_SLOT_IS_USED | DICT_SLOT_WAS_USED); return true; } + /*** + * Writes slot with given key, value and flags. + */ + void WriteSlot(DictSlot& _slot, const K _key, V& _value, unsigned char _slot_flags) { + _slot.key = _key; + _slot.value = _value; + _slot.SetFlags(_slot_flags); + } + /** * Inserts hashless value into given array of DictSlots. */ diff --git a/DictStruct.mqh b/DictStruct.mqh index 6fc1425cd..e57ce86a6 100644 --- a/DictStruct.mqh +++ b/DictStruct.mqh @@ -290,88 +290,100 @@ class DictStruct : public DictBase { } unsigned int position; - DictSlot* keySlot = THIS_ATTR GetSlotByKey(dictSlotsRef, key, position); + DictSlot* _slot = THIS_ATTR GetSlotByKey(dictSlotsRef, key, position); - if (keySlot == NULL && !THIS_ATTR IsGrowUpAllowed()) { - // Resize is prohibited, so we will just overwrite some slot. - allow_resize = false; + // If we have a slot then we can overwrite it. + if (_slot != NULL) { + WriteSlot(_slot, key, value, DICT_SLOT_HAS_KEY | DICT_SLOT_IS_USED | DICT_SLOT_WAS_USED); + // We're done, we don't have to increment number of slots used. + return true; } - if (allow_resize) { - // Will resize dict if there were performance problems before or there is no slots. - if (THIS_ATTR IsGrowUpAllowed() && !dictSlotsRef.IsPerformant()) { + // If we don't have a slot then we should consider growing up number of slots or overwrite some existing slot. + + bool _is_performant = dictSlotsRef.IsPerformant(); // Whether there is no performance problems. + bool _is_full = + dictSlotsRef._num_used == ArraySize(dictSlotsRef.DictSlots); // Whether we don't have empty slots to use. + + if ((_is_full || !_is_performant) && allow_resize) { + // We have to resize the dict as it is either full or have perfomance problems due to massive number of conflicts + // when inserting new values. + if (overflow_listener == NULL) { + // There is no overflow listener so we can freely grow up the dict. if (!GrowUp()) { + // Can't resize the dict. Error happened. return false; } - // We now have new positions of slots, so we have to take the corrent slot again. - keySlot = THIS_ATTR GetSlotByKey(dictSlotsRef, key, position); - } - - if (keySlot == NULL && dictSlotsRef._num_used == ArraySize(dictSlotsRef.DictSlots)) { - // No DictSlotsRef.DictSlots available. - if (THIS_ATTR overflow_listener != NULL) { - if (!THIS_ATTR overflow_listener(DICT_OVERFLOW_REASON_FULL, dictSlotsRef._num_used, 0)) { - // Overwriting slot pointed exactly by key's position in the hash table (we don't check for possible - // conflicts). - keySlot = &dictSlotsRef.DictSlots[THIS_ATTR Hash(key) % ArraySize(dictSlotsRef.DictSlots)]; + } else { + // Overflow listener will decide if we can grow up the dict. + if (overflow_listener(_is_full ? DICT_LISTENER_FULL_CAN_RESIZE : DICT_LISTENER_NOT_PERFORMANT_CAN_RESIZE, + dictSlotsRef._num_used, 0)) { + // We can freely grow up the dict. + if (!GrowUp()) { + // Can't resize the dict. Error happened. + return false; } } - - if (keySlot == NULL) { - // We need to expand array of DictSlotsRef.DictSlots. - if (!GrowUp()) return false; - } } } - if (keySlot == NULL) { - position = THIS_ATTR Hash(key) % ArraySize(dictSlotsRef.DictSlots); - - unsigned int _starting_position = position; - unsigned int _num_conflicts = 0; - bool _overwrite_slot = false; - - // Searching for empty DictSlot or used one with the matching key. It skips used, hashless DictSlots. - while (dictSlotsRef.DictSlots[position].IsUsed() && - (!dictSlotsRef.DictSlots[position].HasKey() || dictSlotsRef.DictSlots[position].key != key)) { - if (THIS_ATTR overflow_listener_max_conflicts != 0 && - ++_num_conflicts == THIS_ATTR overflow_listener_max_conflicts) { - if (THIS_ATTR overflow_listener != NULL) { - if (!THIS_ATTR overflow_listener(DICT_OVERFLOW_REASON_TOO_MANY_CONFLICTS, dictSlotsRef._num_used, - _num_conflicts)) { - // Overflow listener returned false so we won't search for further empty slot. - _overwrite_slot = true; - break; - } - } else { - // Even if there is no overflow listener function, we stop searching for further empty slot as maximum - // number of conflicts has been reached. - _overwrite_slot = true; - break; - } - } + // At this point we have at least one free slot and we won't be doing any dict's grow up in the loop where we search + // for an empty slot. + + // Position we will start from in order to search free slot. + position = THIS_ATTR Hash(key) % ArraySize(dictSlotsRef.DictSlots); - // Position may overflow, so we will start from the beginning. - position = (position + 1) % ArraySize(dictSlotsRef.DictSlots); + // Saving position for further, possible overwrite. + unsigned int _starting_position = position; + + // How many times we had to skip slot as it was already occupied. + unsigned int _num_conflicts = 0; + + // Searching for empty DictSlot or used one with the matching key. It skips used, hashless DictSlots. + while (dictSlotsRef.DictSlots[position].IsUsed() && + (!dictSlotsRef.DictSlots[position].HasKey() || dictSlotsRef.DictSlots[position].key != key)) { + ++_num_conflicts; + + if (overflow_listener == NULL) { + // There is no overflow listener, so we can't overwrite a slot. We will be looping until we find empty slot. + continue; } - if (_overwrite_slot) { - // Overwriting starting position for faster further lookup. + // We had to skip slot as it is already occupied. Now we are checking if + // there is too many conflicts/skips and thus we can overwrite slot in + // the starting position. + if (overflow_listener(DICT_LISTENER_CONFLICTS_CAN_OVERWRITE, dictSlotsRef._num_used, _num_conflicts)) { + // Looks like dict is working as buffer and we can overwrite slot in the starting position. position = _starting_position; - } else if (!dictSlotsRef.DictSlots[position].IsUsed()) { - // If slot isn't already used then we increment number of used slots. - ++dictSlotsRef._num_used; + break; } - dictSlotsRef.AddConflicts(_num_conflicts); + // Position may overflow, so we will start from the beginning. + position = (position + 1) % ArraySize(dictSlotsRef.DictSlots); } - dictSlotsRef.DictSlots[position].key = key; - dictSlotsRef.DictSlots[position].value = value; - dictSlotsRef.DictSlots[position].SetFlags(DICT_SLOT_HAS_KEY | DICT_SLOT_IS_USED | DICT_SLOT_WAS_USED); + // Acknowledging slots array about number of conflicts as it calculates average number of conflicts per insert. + dictSlotsRef.AddConflicts(_num_conflicts); + + // Incrementing number of slots used only if we're writing into empty slot. + if (!dictSlotsRef.DictSlots[position].IsUsed()) { + ++dictSlotsRef._num_used; + } + + // Writing slot in the position of empty slot or, when overwriting, in starting position. + WriteSlot(dictSlotsRef.DictSlots[position], key, value, DICT_SLOT_HAS_KEY | DICT_SLOT_IS_USED | DICT_SLOT_WAS_USED); return true; } + /*** + * Writes slot with given key, value and flags. + */ + void WriteSlot(DictSlot& _slot, const K _key, V& _value, unsigned char _slot_flags) { + _slot.key = _key; + _slot.value = _value; + _slot.SetFlags(_slot_flags); + } + /** * Inserts hashless value into given array of DictSlots. */ diff --git a/Indicator/IndicatorCandle.h b/Indicator/IndicatorCandle.h index 8221afe82..475155aba 100644 --- a/Indicator/IndicatorCandle.h +++ b/Indicator/IndicatorCandle.h @@ -80,7 +80,7 @@ class IndicatorCandle : public Indicator { // Along with indexing by shift, we can also index via timestamp! flags |= INDI_FLAG_INDEXABLE_BY_TIMESTAMP; icdata.AddFlags(DICT_FLAG_FILL_HOLES_UNSORTED); - icdata.SetOverflowListener(IndicatorCandleOverflowListener, 10); + icdata.SetOverflowListener(BufferStructOverflowListener, 10); Set(STRUCT_ENUM(IndicatorDataParams, IDATA_PARAM_MAX_MODES), FINAL_INDI_CANDLE_MODE_ENTRY); } @@ -288,22 +288,6 @@ class IndicatorCandle : public Indicator { return value_storages[_mode].Ptr(); } - /** - * Function should return true if resize can be made, or false to overwrite current slot. - */ - static bool IndicatorCandleOverflowListener(ENUM_DICT_OVERFLOW_REASON _reason, int _size, int _num_conflicts) { - switch (_reason) { - case DICT_OVERFLOW_REASON_FULL: - // We allow resize if dictionary size is less than 86400 slots. - return _size < 86400; - case DICT_OVERFLOW_REASON_TOO_MANY_CONFLICTS: - default: - // When there is too many conflicts, we just reject doing resize, so first conflicting slot will be reused. - break; - } - return false; - } - /** * Sends historic entries to listening indicators. May be overriden. */ diff --git a/Indicator/IndicatorRenko.h b/Indicator/IndicatorRenko.h index a226124be..3c24b6f4b 100644 --- a/Indicator/IndicatorRenko.h +++ b/Indicator/IndicatorRenko.h @@ -228,14 +228,26 @@ class IndicatorRenko : public IndicatorCandle { // Creating new candle. icdata.Add(_candle, entry.timestamp); - Print("Added candle: ", _candle.ToString()); + Print("Added candle: ", _candle.ToString(), " now there is ", icdata.Size(), " candles in the buffer."); last_incomplete_candle_ts = entry.timestamp; } - Print("Last Incomplete Time: ", TimeToString(last_incomplete_candle_ts, TIME_DATE | TIME_MINUTES | TIME_SECONDS)); + static int iteration = 0; + + ++iteration; + + Print("Iteration: ", iteration); + + if (iteration > 1793) { + // Print(icdata.ToJSON()); + } + + Print("Last Incomplete Time: ", TimeToString(last_incomplete_candle_ts, TIME_DATE | TIME_MINUTES | TIME_SECONDS), + " (", last_incomplete_candle_ts, ")"); Print("Last Incomplete Candle: ", icdata.GetByKey(last_incomplete_candle_ts).ToString()); - Print("Last Completed Time: ", TimeToString(last_completed_candle_ts, TIME_DATE | TIME_MINUTES | TIME_SECONDS)); + Print("Last Completed Time: ", TimeToString(last_completed_candle_ts, TIME_DATE | TIME_MINUTES | TIME_SECONDS), + " (", last_completed_candle_ts, ")"); Print("Last Completed Candle: ", icdata.GetByKey(last_completed_candle_ts).ToString()); // Updating tick & bar indices. Bar time is time of the last completed candle. diff --git a/Indicator/IndicatorTick.h b/Indicator/IndicatorTick.h index 5f78a1882..d055958eb 100644 --- a/Indicator/IndicatorTick.h +++ b/Indicator/IndicatorTick.h @@ -69,7 +69,7 @@ class IndicatorTick : public Indicator { flags |= INDI_FLAG_INDEXABLE_BY_TIMESTAMP; itdata.AddFlags(DICT_FLAG_FILL_HOLES_UNSORTED); - itdata.SetOverflowListener(IndicatorTickOverflowListener, 10); + itdata.SetOverflowListener(BufferStructOverflowListener, 10); // Ask and Bid price. Set(STRUCT_ENUM(IndicatorDataParams, IDATA_PARAM_MAX_MODES), 2); } @@ -326,24 +326,6 @@ class IndicatorTick : public Indicator { _tick.ask = _entry[1]; return _tick; } - - /* Callback methods */ - - /** - * Function should return true if resize can be made, or false to overwrite current slot. - */ - static bool IndicatorTickOverflowListener(ENUM_DICT_OVERFLOW_REASON _reason, int _size, int _num_conflicts) { - switch (_reason) { - case DICT_OVERFLOW_REASON_FULL: - // We allow resize if dictionary size is less than 86400 slots. - return _size < 86400; - case DICT_OVERFLOW_REASON_TOO_MANY_CONFLICTS: - default: - // When there is too many conflicts, we just reject doing resize, so first conflicting slot will be reused. - break; - } - return false; - } }; #endif diff --git a/tests/DictTest.mq5 b/tests/DictTest.mq5 index 619dcc90f..35abc6b0d 100644 --- a/tests/DictTest.mq5 +++ b/tests/DictTest.mq5 @@ -55,15 +55,18 @@ class DictTestClass { // Function should return true if resize can be made, or false to overwrite current slot. bool Dict14_OverflowListener(ENUM_DICT_OVERFLOW_REASON _reason, int _size, int _num_conflicts) { + static int cache_limit = 10; switch (_reason) { - case DICT_OVERFLOW_REASON_FULL: + case DICT_LISTENER_FULL_CAN_RESIZE: + case DICT_LISTENER_NOT_PERFORMANT_CAN_RESIZE: // We allow resize if dictionary size is less than 10 slots. - return _size < 10; - case DICT_OVERFLOW_REASON_TOO_MANY_CONFLICTS: - default: - // When there is too many conflicts, we just reject doing resize, so first conflicting slot will be reused. - return false; + return _size < cache_limit; + case DICT_LISTENER_CONFLICTS_CAN_OVERWRITE: + // We start to overwrite slots when we can't make dict bigger and there is at least 10 consecutive conflicts while + // inserting new value. + return _size >= cache_limit && _num_conflicts >= 10; } + return true; } /**