From 3c1baa513741f1af75ce4783ae68e1c2bc557dee Mon Sep 17 00:00:00 2001 From: Roman Yavnikov <45608740+Romazes@users.noreply.github.com> Date: Wed, 9 Oct 2024 03:03:33 +0300 Subject: [PATCH] Refacotr: use new Nasdaq url path (#13) * fix: nasdaq path in url source fix: parsing in Reader * refactor: NasdaqDataLink (naming, description) refactor: get date index in .csv row and use right position to parse date * remove: default value of _apiKey field * fix: workflow file * fix: workflow file * feat: parsed different row data and header of csv files test:feat: different csv file data * revert: previous version of code * refactor: parse dateTime format of csv data * feat: use NumberStyles.Any instead of specific ones * fix: free space in workflow file --- .github/workflows/build.yml | 31 ++++++++------ NasdaqDataLink.cs | 81 ++++++++++++++++++++++++++++-------- tests/NasdaqDataLinkTests.cs | 37 ++++++++++++++++ 3 files changed, 120 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ae5c948..0511678 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,17 +2,19 @@ name: Build & Test on: push: - branches: ['*'] + branches: ["*"] pull_request: branches: [master] jobs: build: runs-on: ubuntu-20.04 - container: - image: quantconnect/lean:foundation steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v2 + + - name: Free space + run: df -h && sudo rm -rf /usr/local/lib/android && sudo rm -rf /opt/ghc && rm -rf /opt/hostedtoolcache* && df -h - name: Checkout Lean Same Branch id: lean-same-branch @@ -33,11 +35,16 @@ jobs: - name: Move Lean run: mv Lean ../Lean - - name: BuildDataSource - run: dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - - - name: BuildTests - run: dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - - - name: Run Tests - run: dotnet test ./tests/bin/Release/net6.0/Tests.dll + - name: Run Image + uses: addnab/docker-run-action@v3 + with: + image: quantconnect/lean:foundation + options: -v /home/runner/work:/__w --workdir /__w/Lean.DataSource.NasdaqDataLink/Lean.DataSource.NasdaqDataLink + shell: bash + run: | + # Build NasdaqDataLink + dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + # Build Tests NasdaqDataLink + dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + # Run Tests NasdaqDataLink + dotnet test ./tests/bin/Release/net6.0/Tests.dll diff --git a/NasdaqDataLink.cs b/NasdaqDataLink.cs index 08fe396..3ac0281 100644 --- a/NasdaqDataLink.cs +++ b/NasdaqDataLink.cs @@ -14,16 +14,16 @@ * */ -using NodaTime; +using System; using ProtoBuf; +using NodaTime; +using System.Net; +using System.Linq; using QuantConnect.Data; -using QuantConnect.Configuration; -using System; using QuantConnect.Logging; -using System.Collections.Generic; using System.Globalization; -using System.Linq; -using System.Net; +using System.Collections.Generic; +using QuantConnect.Configuration; namespace QuantConnect.DataSource { @@ -35,6 +35,7 @@ public class NasdaqDataLink : DynamicData { private static string _authCode = "your_api_key"; private bool _isInitialized; + private readonly List _propertyNames = new List(); // The NasdaqDataLink will use one of these column names if they are available and another option is not provided @@ -66,7 +67,7 @@ static NasdaqDataLink() if (!string.IsNullOrEmpty(potentialNasdaqToken)) { SetAuthCode(potentialNasdaqToken); - } + } else { var potentialQuandlToken = Config.Get("quandl-auth-token"); @@ -113,8 +114,8 @@ public static bool IsAuthCodeSet /// STRING API Url for Nasdaq Data Link. public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) { - var source = $"https://data.nasdaq.com/api/v3/datasets/{config.Symbol.Value}.csv?order=asc&api_key={_authCode}"; - return new SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile); + var source = $"https://data.nasdaq.com/api/v3/datatables/{config.Symbol.Value}.csv?api_key={_authCode}"; + return new SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile) { Sort = true }; } /// @@ -128,33 +129,51 @@ public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) { // be sure to instantiate the correct type - var data = (NasdaqDataLink) Activator.CreateInstance(GetType()); + var data = (NasdaqDataLink)Activator.CreateInstance(GetType()); data.Symbol = config.Symbol; var csv = line.Split(','); if (!_isInitialized) { _isInitialized = true; - foreach (var propertyName in csv) + + for (int i = 0; i < csv.Length; i++) { + var propertyName = csv[i]; var property = propertyName.Trim().ToLowerInvariant(); data.SetProperty(property, 0m); _propertyNames.Add(property); } + + // Returns null at this point where we are only reading the properties names return null; } - data.Time = DateTime.ParseExact(csv[0], "yyyy-MM-dd", CultureInfo.InvariantCulture); - - for (var i = 1; i < csv.Length; i++) + for (var i = 0; i < csv.Length; i++) { - var value = csv[i].ToDecimal(); - data.SetProperty(_propertyNames[i], value); + if (string.IsNullOrEmpty(csv[i])) + { + continue; + } + + if (TryParseDateTimeFormat(_propertyNames[i], out var format) && DateTime.TryParseExact(csv[i], format, CultureInfo.InvariantCulture, DateTimeStyles.None, out var dateTime)) + { + data.Time = dateTime; + data.SetProperty(_propertyNames[i], dateTime); + } + else if (decimal.TryParse(csv[i], NumberStyles.Any, CultureInfo.InvariantCulture, out var value)) + { + data.SetProperty(_propertyNames[i], value); + } + else + { + data.SetProperty(_propertyNames[i], csv[i]); + } } var valueColumnName = _keywords.Intersect(_propertyNames).FirstOrDefault(); - + if (valueColumnName != null) { // If the dataset has any column matches the keywords, set .Value as the first common element with it/them @@ -242,5 +261,33 @@ private void SetValueColumnName(string valueColumnName) // Insert the value column name at the beginning of the keywords list _keywords.Insert(0, valueColumnName); } + + /// + /// Attempts to retrieve the date format string based on the specified property name. + /// + /// The name of the date-related property (e.g., "date", "year"). + /// The output format string corresponding to the property name, if found. + /// + /// true if a valid date format is found; otherwise, false. + /// + private static bool TryParseDateTimeFormat(string propertyName, out string format) + { + format = string.Empty; + switch (propertyName.ToLower()) + { + case "date": + format = "yyyy-MM-dd"; + break; + case "year": + format = "yyyy"; + break; + case "report_month": + format = "yyyy-MM"; + break; + default: + return false; + } + return true; + } } } diff --git a/tests/NasdaqDataLinkTests.cs b/tests/NasdaqDataLinkTests.cs index ebb89f3..53c35e3 100644 --- a/tests/NasdaqDataLinkTests.cs +++ b/tests/NasdaqDataLinkTests.cs @@ -97,6 +97,43 @@ public void ValueColumn() Assert.AreEqual(expected, data.Value); } + [TestCase("QDL/FON", "contract_code,type,date,market_participation,producer_merchant_processor_user_longs,producer_merchant_processor_user_shorts,swap_dealer_longs,swap_dealer_shorts,swap_dealer_spreads,money_manager_longs,money_manager_shorts,money_manager_spreads,other_reportable_longs,other_reportable_shorts,other_reportable_spreads,total_reportable_longs,total_reportable_shorts,non_reportable_longs,non_reportable_shorts", "967654,FO_OLD,2017-12-26,27122.0,9984.0,19225.0,1945.0,1405.0,1596.0,0.0,150.0,0.0,8316.0,597.0,1464.0,23305.0,24437.0,3817.0,2685.0", Description = "Commodity Futures Trading Commission Reports: Futures and Options Metrics: OI and NT")] + [TestCase("QDL/LFON", "contract_code,type,date,market_participation,non_commercial_longs,non_commercial_shorts,non_commercial_spreads,commercial_longs,commercial_shorts,total_reportable_longs,total_reportable_shorts,non_reportable_longs,non_reportable_shorts", "ZB9105,FO_L_OLD_OI,2018-11-27,100.0,68.6,68.6,31.4,0.0,0.0,100.0,100.0,0.0,0.0", Description = "Commodity Futures Trading Commission Reports: Legacy Futures and Options Metrics: OI and NT")] + [TestCase("QDL/FCR", "contract_code,type,date,largest_4_longs_gross,largest_4_shorts_gross,largest_8_longs_gross,largest_8_shorts_gross,largest_4_longs_net,largest_4_shorts_net,largest_8_longs_net,largest_8_shorts_net", "ZB9105,F_L_ALL_CR,2018-10-30,87.6,99.0,97.8,100.0,13.7,16.1,15.8,16.3", Description = "Commodity Futures Trading Commission Reports: Futures and Options Metrics: CR")] + [TestCase("QDL/BCHAIN", "code,date,value", "TRFUS,2020-08-28,1197064.8298", Description = "Bitcoin Data Insights")] + [TestCase("QDL/ODA", "indicator,date,value", "ZWE_PPPSH,2018-12-31,0.028", Description = "IMF Cross Country Macroeconomic Statistics")] + [TestCase("QDL/ODA", "indicator,date,value", "ZWE_PPPSH,1997-12-31,", Description = "IMF Cross Country Macroeconomic Statistics")] + [TestCase("QDL/JODI", "energy,code,country,date,value,notes", "OIL,TPSDKT,ZAF,2024-04-30,0.0000,3", Description = "JODI Oil World Database")] + [TestCase("QDL/JODI", "energy,code,country,date,value,notes", "OIL,TPSDKT,TTO,2005-03-31,,3", Description = "JODI Oil World Database")] + [TestCase("QDL/BITFINEX", "code,date,high,low,mid,last,bid,ask,volume", "ZRXUSD,2024-09-13,0.30349,0.28357,0.29886,0.29922,0.29865,0.29907,236649.194029", Description = "Bitfinex Crypto Coins Exchange Rate")] + [TestCase("QDL/BITFINEX", "code,date,high,low,mid,last,bid,ask,volume", "ZRXBTC,2022-02-20,1.493e-05,1.448e-05,1.487e-05,1.489e-05,1.485e-05,1.489e-05,6907.80795766", Description = "Bitfinex Crypto Coins Exchange Rate")] + [TestCase("QDL/OPEC", "date,value", "2024-01-12,80.18", Description = "Organization of the Petroleum Exporting Countries")] + [TestCase("QDL/LME", "item_code,country_code,date,opening_stock,delivered_in,delivered_out,closing_stock,open_tonnage,cancelled_tonnage", "ZIJ,UTO,2024-07-12,0.0,0.0,0.0,0.0,0.0,0.0", Description = "Metal Stocks Breakdown Report")] + [TestCase("QDL/LME", "item_code,country_code,date,opening_stock,delivered_in,delivered_out,closing_stock,open_tonnage,cancelled_tonnage", "ZII,UNE,2020-07-23,26075.0,0.0,0.0,26075.0,14425.0,11650.0", Description = "Metal Stocks Breakdown Report")] + [TestCase("ZILLOW/DATA", "indicator_id,region_id,date,value", "ZSFH,99999,2024-04-30,481777.608668988", Description = "Zillow Real Estate Data")] + [TestCase("ZILLOW/DATA", "indicator_id,region_id,date,value", "ZSFH,99993,2008-07-31,139908.0", Description = "Zillow Real Estate Data")] + [TestCase("WB/DATA", "series_id,country_code,country_name,year,value", "VC.PKP.TOTL.UN,XKX,Kosovo,2017,357.0", Description = "World Bank Data")] + [TestCase("WB/DATA", "series_id,country_code,country_name,year,value", "VC.IHR.PSRC.P5,ALB,Albania,1998,20.4196832752429", Description = "World Bank Data")] + [TestCase("WASDE/DATA", "code,report_month,region,commodity,item,year,period,value,min_value,max_value", "WHEAT_WORLD_19,2024-02,World Less China,Wheat,Production,2023/24 Proj.,Jan,648.32,,", Description = "World Agricultural Supply and Demand Estimates")] + [TestCase("WASDE/DATA", "code,report_month,region,commodity,item,year,period,value,min_value,max_value", "WHEAT_WORLD_19,2022-08,N. Africa 7/,Wheat,Production,2022/23 Proj.,Jul,17.15,,", Description = "World Agricultural Supply and Demand Estimates")] + [TestCase("WASDE/DATA", "code,report_month,region,commodity,item,year,period,value,min_value,max_value", "WHEAT_WORLD_19,2021-05,Brazil,Wheat,Beginning Stocks,2021/22 Proj.,May,0.64,,", Description = "World Agricultural Supply and Demand Estimates")] + public void CreateDifferentNasdaqDataSymbolWithVariousProperties(string nasdaqDataName, string csvHeader, string csvData) + { + var nasdaq = new NasdaqDataLink(); + + var symbol = Symbol.Create(nasdaqDataName, SecurityType.Base, "empty"); + + var config = new SubscriptionDataConfig(typeof(NasdaqDataLink), symbol, Resolution.Daily, TimeZones.Utc, TimeZones.Utc, true, true, false, true); + + var dateTimeUtcNow = DateTime.UtcNow; + + nasdaq.Reader(config, csvHeader, dateTimeUtcNow, false); + var data = nasdaq.Reader(config, csvData, dateTimeUtcNow, false); + + Assert.That(data.Time, Is.Not.EqualTo(default)); + Assert.GreaterOrEqual(data.Value, 0m); + } + [Test] public void PythonValueColumn() {