From 1e939255ad727aab35c5a6d837ad76bdb96f4063 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 15 Mar 2024 18:33:04 +0200 Subject: [PATCH 1/5] feat: `volatile` for prevent spamming flags --- QuantConnect.IEX/IEXDataProvider.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/QuantConnect.IEX/IEXDataProvider.cs b/QuantConnect.IEX/IEXDataProvider.cs index 21e1a70..3c9c865 100644 --- a/QuantConnect.IEX/IEXDataProvider.cs +++ b/QuantConnect.IEX/IEXDataProvider.cs @@ -56,22 +56,22 @@ public class IEXDataProvider : SynchronizingHistoryProvider, IDataQueueHandler /// /// Flag indicating whether a warning about unsupported data types in user history should be suppressed to prevent spam. /// - private static bool _invalidHistoryDataTypeWarningFired; + private volatile bool _invalidHistoryDataTypeWarningFired; /// /// Indicates whether the warning for invalid has been fired. /// - private bool _invalidSecurityTypeWarningFired; + private volatile bool _invalidSecurityTypeWarningFired; /// /// Indicates whether a warning for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC. /// - private bool _invalidStartTimeWarningFired; + private volatile bool _invalidStartTimeWarningFired; /// /// Indicates whether a warning for an invalid has been fired, where the resolution is neither daily nor minute-based. /// - private bool _invalidResolutionWarningFired; + private volatile bool _invalidResolutionWarningFired; /// /// Represents two clients: one for the trade channel and another for the top-of-book channel. @@ -517,9 +517,9 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) { if (!_invalidHistoryDataTypeWarningFired) { + _invalidHistoryDataTypeWarningFired = true; Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Not supported data type - {request.DataType.Name}. " + "Currently available support only for historical of type - TradeBar"); - _invalidHistoryDataTypeWarningFired = true; } continue; } @@ -528,8 +528,8 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) { if (!_invalidSecurityTypeWarningFired) { - Log.Trace($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Unsupported SecurityType '{request.Symbol.SecurityType}' for symbol '{request.Symbol}'"); _invalidSecurityTypeWarningFired = true; + Log.Trace($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Unsupported SecurityType '{request.Symbol.SecurityType}' for symbol '{request.Symbol}'"); } continue; } @@ -538,8 +538,8 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) { if (!_invalidStartTimeWarningFired) { - Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Error - The start date in the history request must come before the end date. No historical data will be returned."); _invalidStartTimeWarningFired = true; + Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Error - The start date in the history request must come before the end date. No historical data will be returned."); } continue; } @@ -548,8 +548,8 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) { if (!_invalidResolutionWarningFired) { - Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: History calls for IEX only support daily & minute resolution."); _invalidResolutionWarningFired = true; + Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: History calls for IEX only support daily & minute resolution."); } continue; } From 1d949b38664860fd3996db9dede0e0ba668cb3ff Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 15 Mar 2024 19:12:42 +0200 Subject: [PATCH 2/5] test:remove: extra param in test data feat: add overloading of CreateHistoryRequest --- QuantConnect.IEX.Tests/IEXDataHistoryTests.cs | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/QuantConnect.IEX.Tests/IEXDataHistoryTests.cs b/QuantConnect.IEX.Tests/IEXDataHistoryTests.cs index ee28f06..2b08307 100644 --- a/QuantConnect.IEX.Tests/IEXDataHistoryTests.cs +++ b/QuantConnect.IEX.Tests/IEXDataHistoryTests.cs @@ -14,6 +14,7 @@ */ using System; +using NodaTime; using System.Linq; using NUnit.Framework; using QuantConnect.Data; @@ -65,7 +66,7 @@ internal static IEnumerable ValidDataTestParameters .SetDescription("Valid parameters - Minute resolution, 5 days period.") .SetCategory("Valid"); - yield return new TestCaseData(Symbols.SPY, Resolution.Minute, TickType.Trade, TimeSpan.FromDays(45), true) + yield return new TestCaseData(Symbols.SPY, Resolution.Minute, TickType.Trade, TimeSpan.FromDays(45)) .SetDescription("Valid parameters - Beyond 45 days, Minute resolution.") .SetCategory("Valid"); } @@ -235,25 +236,42 @@ private Slice[] GetHistory(Symbol symbol, Resolution resolution, TickType tickTy internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period) { - var utcNow = DateTime.UtcNow; + var end = new DateTime(2024, 3, 15, 16, 0, 0); - var dataType = LeanData.GetDataType(resolution, tickType); + if (resolution == Resolution.Daily) + { + end = end.Date.AddDays(1); + } + + return CreateHistoryRequest(symbol, resolution, tickType, end.Subtract(period), end); + } + + internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, DateTime startDateTime, DateTime endDateTime, + SecurityExchangeHours exchangeHours = null, DateTimeZone dataTimeZone = null) + { + if (exchangeHours == null) + { + exchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork); + } - var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); - var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); + if (dataTimeZone == null) + { + dataTimeZone = TimeZones.NewYork; + } + var dataType = LeanData.GetDataType(resolution, tickType); return new HistoryRequest( - startTimeUtc: utcNow.Add(-period), - endTimeUtc: utcNow, + startTimeUtc: startDateTime, + endTimeUtc: endDateTime, dataType: dataType, symbol: symbol, resolution: resolution, exchangeHours: exchangeHours, dataTimeZone: dataTimeZone, - fillForwardResolution: resolution, + fillForwardResolution: null, includeExtendedMarketHours: true, isCustomData: false, - DataNormalizationMode.Raw, + dataNormalizationMode: DataNormalizationMode.Adjusted, tickType: tickType ); } From c255169aab1088981a63607018efe5463d8aa53c Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 15 Mar 2024 19:18:10 +0200 Subject: [PATCH 3/5] refactor: DataDownloader provider to general structure --- QuantConnect.IEX/IEXDataDownloader.cs | 35 +++++------ QuantConnect.IEX/IEXDataProvider.cs | 84 ++++++++++++++------------- 2 files changed, 60 insertions(+), 59 deletions(-) diff --git a/QuantConnect.IEX/IEXDataDownloader.cs b/QuantConnect.IEX/IEXDataDownloader.cs index 826165b..f1905b9 100644 --- a/QuantConnect.IEX/IEXDataDownloader.cs +++ b/QuantConnect.IEX/IEXDataDownloader.cs @@ -14,8 +14,8 @@ */ using QuantConnect.Data; +using QuantConnect.Util; using QuantConnect.Securities; -using QuantConnect.Data.Market; namespace QuantConnect.Lean.DataSource.IEX { @@ -44,30 +44,25 @@ public void Dispose() public IEnumerable? Get(DataDownloaderGetParameters dataDownloaderGetParameters) { var symbol = dataDownloaderGetParameters.Symbol; - var resolution = dataDownloaderGetParameters.Resolution; - var startUtc = dataDownloaderGetParameters.StartUtc; - var endUtc = dataDownloaderGetParameters.EndUtc; - var tickType = dataDownloaderGetParameters.TickType; var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); - var historyRequests = new[] { - new HistoryRequest(startUtc, - endUtc, - typeof(TradeBar), - symbol, - resolution, - exchangeHours, - dataTimeZone, - resolution, - true, - false, - DataNormalizationMode.Raw, - TickType.Trade) - }; + var historyRequests = new HistoryRequest( + dataDownloaderGetParameters.StartUtc, + dataDownloaderGetParameters.EndUtc, + LeanData.GetDataType(dataDownloaderGetParameters.Resolution, dataDownloaderGetParameters.TickType), + symbol, + dataDownloaderGetParameters.Resolution, + exchangeHours, + dataTimeZone, + dataDownloaderGetParameters.Resolution, + true, + false, + DataNormalizationMode.Raw, + dataDownloaderGetParameters.TickType); - return _handler.GetHistory(historyRequests, TimeZones.EasternStandard)?.Select(slice => (BaseData)slice[symbol]); + return _handler.ProcessHistoryRequests(historyRequests); } } } diff --git a/QuantConnect.IEX/IEXDataProvider.cs b/QuantConnect.IEX/IEXDataProvider.cs index 3c9c865..5cc9481 100644 --- a/QuantConnect.IEX/IEXDataProvider.cs +++ b/QuantConnect.IEX/IEXDataProvider.cs @@ -512,49 +512,13 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) var subscriptions = new List(); foreach (var request in requests) { - // IEX does return historical TradeBar - give one time warning if inconsistent data type was requested - if (request.TickType != TickType.Trade) - { - if (!_invalidHistoryDataTypeWarningFired) - { - _invalidHistoryDataTypeWarningFired = true; - Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Not supported data type - {request.DataType.Name}. " + - "Currently available support only for historical of type - TradeBar"); - } - continue; - } - - if (!CanSubscribe(request.Symbol)) - { - if (!_invalidSecurityTypeWarningFired) - { - _invalidSecurityTypeWarningFired = true; - Log.Trace($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Unsupported SecurityType '{request.Symbol.SecurityType}' for symbol '{request.Symbol}'"); - } - continue; - } - - if (request.StartTimeUtc >= request.EndTimeUtc) - { - if (!_invalidStartTimeWarningFired) - { - _invalidStartTimeWarningFired = true; - Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Error - The start date in the history request must come before the end date. No historical data will be returned."); - } - continue; - } + var history = ProcessHistoryRequests(request); - if (request.Resolution != Resolution.Daily && request.Resolution != Resolution.Minute) + if (history == null) { - if (!_invalidResolutionWarningFired) - { - _invalidResolutionWarningFired = true; - Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: History calls for IEX only support daily & minute resolution."); - } continue; } - var history = ProcessHistoryRequests(request); var subscription = CreateSubscription(request, history); subscriptions.Add(subscription); } @@ -570,8 +534,50 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) /// /// Populate request data /// - private IEnumerable? ProcessHistoryRequests(Data.HistoryRequest request) + public IEnumerable? ProcessHistoryRequests(Data.HistoryRequest request) { + // IEX does return historical TradeBar - give one time warning if inconsistent data type was requested + if (request.TickType != TickType.Trade) + { + if (!_invalidHistoryDataTypeWarningFired) + { + _invalidHistoryDataTypeWarningFired = true; + Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Not supported data type - {request.DataType.Name}. " + + "Currently available support only for historical of type - TradeBar"); + } + return null; + } + + if (!CanSubscribe(request.Symbol)) + { + if (!_invalidSecurityTypeWarningFired) + { + _invalidSecurityTypeWarningFired = true; + Log.Trace($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Unsupported SecurityType '{request.Symbol.SecurityType}' for symbol '{request.Symbol}'"); + } + return null; + } + + if (request.StartTimeUtc >= request.EndTimeUtc) + { + if (!_invalidStartTimeWarningFired) + { + _invalidStartTimeWarningFired = true; + Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Error - The start date in the history request must come before the end date. No historical data will be returned."); + } + return null; + } + + if (request.Resolution != Resolution.Daily && request.Resolution != Resolution.Minute) + { + if (!_invalidResolutionWarningFired) + { + _invalidResolutionWarningFired = true; + Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: History calls for IEX only support daily & minute resolution."); + } + return null; + } + var ticker = request.Symbol.ID.Symbol; var start = ConvertTickTimeBySymbol(request.Symbol, request.StartTimeUtc); var end = ConvertTickTimeBySymbol(request.Symbol, request.EndTimeUtc); From ef15c36d3dfa77d3383bb595b144adcc162f482d Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 15 Mar 2024 22:12:36 +0200 Subject: [PATCH 4/5] fix: convert DateTime to TZ in GetHistory feat: prevent spamming flag for limit of minute data refactor: reduce some part of code test:feat: get ancient historical data (renamed symbol test) --- QuantConnect.IEX.Tests/IEXDataHistoryTests.cs | 16 ++++ QuantConnect.IEX/IEXDataProvider.cs | 79 +++++++++---------- 2 files changed, 55 insertions(+), 40 deletions(-) diff --git a/QuantConnect.IEX.Tests/IEXDataHistoryTests.cs b/QuantConnect.IEX.Tests/IEXDataHistoryTests.cs index 2b08307..ec5f61f 100644 --- a/QuantConnect.IEX.Tests/IEXDataHistoryTests.cs +++ b/QuantConnect.IEX.Tests/IEXDataHistoryTests.cs @@ -160,6 +160,22 @@ public void IEXCloudGetHistoryWithValidDataTestParameters(Symbol symbol, Resolut Assert.That(slices, Is.Ordered.By("Time")); } + [Explicit("This tests require a iexcloud.io api key")] + [TestCase("GOOGL", Resolution.Daily, "2013/1/3", "2015/12/29", Description = "October 2, 2015. [GOOG -> GOOGL]")] + [TestCase("META", Resolution.Daily, "2020/1/3", "2023/12/29", Description = "October 28, 2021. [FB -> META]")] + public void GetAncientEquityHistoricalData(string ticker, Resolution resolution, DateTime startDate, DateTime endDate) + { + var symbol = Symbol.Create(ticker, SecurityType.Equity, Market.USA); + + var request = CreateHistoryRequest(symbol, resolution, TickType.Trade, startDate, endDate); + + var slices = iexDataProvider.GetHistory(new[] { request }, TimeZones.NewYork)?.ToList(); + + Assert.Greater(slices.Count, 1); + Assert.That(slices.First().Time.Date, Is.EqualTo(startDate)); + Assert.That(slices.Last().Time.Date, Is.LessThanOrEqualTo(endDate)); + } + internal static void AssertTradeBar(Symbol expectedSymbol, Resolution resolution, BaseData baseData, Symbol actualSymbol = null) { if (actualSymbol != null) diff --git a/QuantConnect.IEX/IEXDataProvider.cs b/QuantConnect.IEX/IEXDataProvider.cs index 5cc9481..deba374 100644 --- a/QuantConnect.IEX/IEXDataProvider.cs +++ b/QuantConnect.IEX/IEXDataProvider.cs @@ -73,6 +73,12 @@ public class IEXDataProvider : SynchronizingHistoryProvider, IDataQueueHandler /// private volatile bool _invalidResolutionWarningFired; + /// + /// Indicates whether a warning has been triggered for reaching the limit of resolution, + /// where the startDateTime is not greater than 2 years. + /// + private volatile bool _limitMinuteResolutionWarningFired; + /// /// Represents two clients: one for the trade channel and another for the top-of-book channel. /// @@ -578,21 +584,30 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) return null; } - var ticker = request.Symbol.ID.Symbol; - var start = ConvertTickTimeBySymbol(request.Symbol, request.StartTimeUtc); - var end = ConvertTickTimeBySymbol(request.Symbol, request.EndTimeUtc); + var ticker = request.Symbol.Value; + var startExchangeDateTime = ConvertTickTimeBySymbol(request.Symbol, request.StartTimeUtc); + var endExchangeDateTime = ConvertTickTimeBySymbol(request.Symbol, request.EndTimeUtc); - Log.Trace($"{nameof(IEXDataProvider)}.{nameof(ProcessHistoryRequests)}: {request.Symbol.SecurityType}.{ticker}, Resolution: {request.Resolution}, DateTime: [{start} - {end}]."); + if (request.Resolution == Resolution.Minute && startExchangeDateTime <= DateTime.Today.AddYears(-2)) + { + if (!_limitMinuteResolutionWarningFired) + { + _limitMinuteResolutionWarningFired = true; + Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: History calls with minute resolution for IEX available only not more 2 years."); + } + return null; + } + + Log.Trace($"{nameof(IEXDataProvider)}.{nameof(ProcessHistoryRequests)}: {request.Symbol.SecurityType}.{ticker}, Resolution: {request.Resolution}, DateTime: [{startExchangeDateTime} - {endExchangeDateTime}]."); - var span = end - start; var urls = new List(); switch (request.Resolution) { case Resolution.Minute: { - var begin = start; - while (begin < end) + var begin = startExchangeDateTime; + while (begin < endExchangeDateTime) { var url = $"{BaseUrl}/{ticker}/chart/date/{begin.ToStringInvariant("yyyyMMdd")}?token={_apiKey}"; urls.Add(url); @@ -603,35 +618,19 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) } case Resolution.Daily: { - string suffix; - if (span.Days < 30) - { - suffix = "1m"; - } - else if (span.Days < 3 * 30) - { - suffix = "3m"; - } - else if (span.Days < 6 * 30) - { - suffix = "6m"; - } - else if (span.Days < 12 * 30) - { - suffix = "1y"; - } - else if (span.Days < 24 * 30) - { - suffix = "2y"; - } - else if (span.Days < 60 * 30) - { - suffix = "5y"; - } - else + // To retrieve a specific start-to-end dateTime range, calculate the duration between the current time and the startExchangeDateTime. + var span = DateTime.Now - startExchangeDateTime; + + string suffix = span.Days switch { - suffix = "max"; // max is 15 years - } + < 30 => "1m", + < 3 * 30 => "3m", + < 6 * 30 => "6m", + < 12 * 30 => "1y", + < 24 * 30 => "2y", + < 60 * 30 => "5y", + _ => "max" // max is 15 years + }; var url = $"{BaseUrl}/{ticker}/chart/{suffix}?token={_apiKey}"; urls.Add(url); @@ -640,7 +639,9 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) } } - return GetHistoryRequestByUrls(urls, start, end, request.Resolution, request.DataNormalizationMode, request.Symbol); + Log.Debug($"{nameof(IEXDataProvider)}.{nameof(ProcessHistoryRequests)}: {string.Join("\n", urls)}"); + + return GetHistoryRequestByUrls(urls, startExchangeDateTime, endExchangeDateTime, request.Resolution, request.DataNormalizationMode, request.Symbol); } /// @@ -684,7 +685,7 @@ private IEnumerable GetHistoryRequestByUrls(List urls, DateTim period = TimeSpan.FromDays(1); } - if (date < startDateTime || date > endDateTime) + if (!(date >= startDateTime && date < endDateTime)) { continue; } @@ -716,9 +717,7 @@ private IEnumerable GetHistoryRequestByUrls(List urls, DateTim volume = item["volume"].Value(); } - var tradeBar = new TradeBar(date, symbol, open, high, low, close, volume, period); - - yield return tradeBar; + yield return new TradeBar(ConvertTickTimeBySymbol(symbol, date), symbol, open, high, low, close, volume, period); } } } From ae839a3d0a169331fa54ba6429fb1d0654a5720c Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 15 Mar 2024 23:52:39 +0200 Subject: [PATCH 5/5] feat: get relevant ticker for each historical request --- QuantConnect.IEX/IEXDataProvider.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/QuantConnect.IEX/IEXDataProvider.cs b/QuantConnect.IEX/IEXDataProvider.cs index deba374..c776ee6 100644 --- a/QuantConnect.IEX/IEXDataProvider.cs +++ b/QuantConnect.IEX/IEXDataProvider.cs @@ -584,7 +584,8 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) return null; } - var ticker = request.Symbol.Value; + // Always obtain the most relevant ticker symbol based on the current time. + var ticker = SecurityIdentifier.Ticker(request.Symbol, DateTime.UtcNow); var startExchangeDateTime = ConvertTickTimeBySymbol(request.Symbol, request.StartTimeUtc); var endExchangeDateTime = ConvertTickTimeBySymbol(request.Symbol, request.EndTimeUtc);