Refacotr: use new Nasdaq url path (#13)

* fix: nasdaq path in url source
fix: parsing in Reader

* refactor: NasdaqDataLink (naming, description)
refactor: get date index in .csv row and use right position to parse date

* remove: default value of _apiKey field

* fix: workflow file

* fix: workflow file

* feat: parsed different row data and header of csv files
test:feat: different csv file data

* revert: previous version of code

* refactor: parse dateTime format of csv data

* feat: use NumberStyles.Any instead of specific ones

* fix: free space in workflow file

.github/workflows/build.yml

@@ -2,17 +2,19 @@ name: Build & Test
 
 on:
   push:
-    branches: ['*']
+    branches: ["*"]
   pull_request:
     branches: [master]
 
 jobs:
   build:
     runs-on: ubuntu-20.04
-    container:
-        image: quantconnect/lean:foundation
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Free space
+        run: df -h && sudo rm -rf /usr/local/lib/android && sudo rm -rf /opt/ghc && rm -rf /opt/hostedtoolcache* && df -h
 
       - name: Checkout Lean Same Branch
         id: lean-same-branch
@@ -33,11 +35,16 @@ jobs:
       - name: Move Lean
         run: mv Lean ../Lean
 
-      - name: BuildDataSource
-        run: dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
-
-      - name: BuildTests
-        run: dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
-
-      - name: Run Tests
-        run: dotnet test ./tests/bin/Release/net6.0/Tests.dll
+      - name: Run Image
+        uses: addnab/docker-run-action@v3
+        with:
+          image: quantconnect/lean:foundation
+          options: -v /home/runner/work:/__w --workdir /__w/Lean.DataSource.NasdaqDataLink/Lean.DataSource.NasdaqDataLink
+          shell: bash
+          run: |
+            # Build NasdaqDataLink
+            dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
+            # Build Tests NasdaqDataLink
+            dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
+            # Run Tests NasdaqDataLink
+            dotnet test ./tests/bin/Release/net6.0/Tests.dll

NasdaqDataLink.cs

@@ -14,16 +14,16 @@
  *
 */
 
-using NodaTime;
+using System;
 using ProtoBuf;
+using NodaTime;
+using System.Net;
+using System.Linq;
 using QuantConnect.Data;
-using QuantConnect.Configuration;
-using System;
 using QuantConnect.Logging;
-using System.Collections.Generic;
 using System.Globalization;
-using System.Linq;
-using System.Net;
+using System.Collections.Generic;
+using QuantConnect.Configuration;
 
 namespace QuantConnect.DataSource
 {
@@ -35,6 +35,7 @@ public class NasdaqDataLink : DynamicData
     {
         private static string _authCode = "your_api_key";
         private bool _isInitialized;
+
         private readonly List<string> _propertyNames = new List<string>();
 
         // The NasdaqDataLink will use one of these column names if they are available and another option is not provided
@@ -66,7 +67,7 @@ static NasdaqDataLink()
             if (!string.IsNullOrEmpty(potentialNasdaqToken))
             {
                 SetAuthCode(potentialNasdaqToken);
-            } 
+            }
             else
             {
                 var potentialQuandlToken = Config.Get("quandl-auth-token");
@@ -113,8 +114,8 @@ public static bool IsAuthCodeSet
         /// <returns>STRING API Url for Nasdaq Data Link.</returns>
         public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
         {
-            var source = $"https://data.nasdaq.com/api/v3/datasets/{config.Symbol.Value}.csv?order=asc&api_key={_authCode}";
-            return new SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile);
+            var source = $"https://data.nasdaq.com/api/v3/datatables/{config.Symbol.Value}.csv?api_key={_authCode}";
+            return new SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile) { Sort = true };
         }
 
         /// <summary>
@@ -128,33 +129,51 @@ public override SubscriptionDataSource GetSource(SubscriptionDataConfig config,
         public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
         {
             // be sure to instantiate the correct type
-            var data = (NasdaqDataLink) Activator.CreateInstance(GetType());
+            var data = (NasdaqDataLink)Activator.CreateInstance(GetType());
             data.Symbol = config.Symbol;
             var csv = line.Split(',');
 
             if (!_isInitialized)
             {
                 _isInitialized = true;
-                foreach (var propertyName in csv)
+
+                for (int i = 0; i < csv.Length; i++)
                 {
+                    var propertyName = csv[i];
                     var property = propertyName.Trim().ToLowerInvariant();
                     data.SetProperty(property, 0m);
                     _propertyNames.Add(property);
                 }
+
+
                 // Returns null at this point where we are only reading the properties names
                 return null;
             }
 
-            data.Time = DateTime.ParseExact(csv[0], "yyyy-MM-dd", CultureInfo.InvariantCulture);
-
-            for (var i = 1; i < csv.Length; i++)
+            for (var i = 0; i < csv.Length; i++)
             {
-                var value = csv[i].ToDecimal();
-                data.SetProperty(_propertyNames[i], value);
+                if (string.IsNullOrEmpty(csv[i]))
+                {
+                    continue;
+                }
+
+                if (TryParseDateTimeFormat(_propertyNames[i], out var format) && DateTime.TryParseExact(csv[i], format, CultureInfo.InvariantCulture, DateTimeStyles.None, out var dateTime))
+                {
+                    data.Time = dateTime;
+                    data.SetProperty(_propertyNames[i], dateTime);
+                }
+                else if (decimal.TryParse(csv[i], NumberStyles.Any, CultureInfo.InvariantCulture, out var value))
+                {
+                    data.SetProperty(_propertyNames[i], value);
+                }
+                else
+                {
+                    data.SetProperty(_propertyNames[i], csv[i]);
+                }
             }
 
             var valueColumnName = _keywords.Intersect(_propertyNames).FirstOrDefault();
-            
+
             if (valueColumnName != null)
             {
                 // If the dataset has any column matches the keywords, set .Value as the first common element with it/them
@@ -242,5 +261,33 @@ private void SetValueColumnName(string valueColumnName)
             // Insert the value column name at the beginning of the keywords list
             _keywords.Insert(0, valueColumnName);
         }
+
+        /// <summary>
+        /// Attempts to retrieve the date format string based on the specified property name.
+        /// </summary>
+        /// <param name="propertyName">The name of the date-related property (e.g., "date", "year").</param>
+        /// <param name="format">The output format string corresponding to the property name, if found.</param>
+        /// <returns>
+        /// <c>true</c> if a valid date format is found; otherwise, <c>false</c>.
+        /// </returns>
+        private static bool TryParseDateTimeFormat(string propertyName, out string format)
+        {
+            format = string.Empty;
+            switch (propertyName.ToLower())
+            {
+                case "date":
+                    format = "yyyy-MM-dd";
+                    break;
+                case "year":
+                    format = "yyyy";
+                    break;
+                case "report_month":
+                    format = "yyyy-MM";
+                    break;
+                default:
+                    return false;
+            }
+            return true;
+        }
     }
 }

tests/NasdaqDataLinkTests.cs

@@ -97,6 +97,43 @@ public void ValueColumn()
             Assert.AreEqual(expected, data.Value);
         }
 
+        [TestCase("QDL/FON", "contract_code,type,date,market_participation,producer_merchant_processor_user_longs,producer_merchant_processor_user_shorts,swap_dealer_longs,swap_dealer_shorts,swap_dealer_spreads,money_manager_longs,money_manager_shorts,money_manager_spreads,other_reportable_longs,other_reportable_shorts,other_reportable_spreads,total_reportable_longs,total_reportable_shorts,non_reportable_longs,non_reportable_shorts", "967654,FO_OLD,2017-12-26,27122.0,9984.0,19225.0,1945.0,1405.0,1596.0,0.0,150.0,0.0,8316.0,597.0,1464.0,23305.0,24437.0,3817.0,2685.0", Description = "Commodity Futures Trading Commission Reports: Futures and Options Metrics: OI and NT")]
+        [TestCase("QDL/LFON", "contract_code,type,date,market_participation,non_commercial_longs,non_commercial_shorts,non_commercial_spreads,commercial_longs,commercial_shorts,total_reportable_longs,total_reportable_shorts,non_reportable_longs,non_reportable_shorts", "ZB9105,FO_L_OLD_OI,2018-11-27,100.0,68.6,68.6,31.4,0.0,0.0,100.0,100.0,0.0,0.0", Description = "Commodity Futures Trading Commission Reports: Legacy Futures and Options Metrics: OI and NT")]
+        [TestCase("QDL/FCR", "contract_code,type,date,largest_4_longs_gross,largest_4_shorts_gross,largest_8_longs_gross,largest_8_shorts_gross,largest_4_longs_net,largest_4_shorts_net,largest_8_longs_net,largest_8_shorts_net", "ZB9105,F_L_ALL_CR,2018-10-30,87.6,99.0,97.8,100.0,13.7,16.1,15.8,16.3", Description = "Commodity Futures Trading Commission Reports: Futures and Options Metrics: CR")]
+        [TestCase("QDL/BCHAIN", "code,date,value", "TRFUS,2020-08-28,1197064.8298", Description = "Bitcoin Data Insights")]
+        [TestCase("QDL/ODA", "indicator,date,value", "ZWE_PPPSH,2018-12-31,0.028", Description = "IMF Cross Country Macroeconomic Statistics")]
+        [TestCase("QDL/ODA", "indicator,date,value", "ZWE_PPPSH,1997-12-31,", Description = "IMF Cross Country Macroeconomic Statistics")]
+        [TestCase("QDL/JODI", "energy,code,country,date,value,notes", "OIL,TPSDKT,ZAF,2024-04-30,0.0000,3", Description = "JODI Oil World Database")]
+        [TestCase("QDL/JODI", "energy,code,country,date,value,notes", "OIL,TPSDKT,TTO,2005-03-31,,3", Description = "JODI Oil World Database")]
+        [TestCase("QDL/BITFINEX", "code,date,high,low,mid,last,bid,ask,volume", "ZRXUSD,2024-09-13,0.30349,0.28357,0.29886,0.29922,0.29865,0.29907,236649.194029", Description = "Bitfinex Crypto Coins Exchange Rate")]
+        [TestCase("QDL/BITFINEX", "code,date,high,low,mid,last,bid,ask,volume", "ZRXBTC,2022-02-20,1.493e-05,1.448e-05,1.487e-05,1.489e-05,1.485e-05,1.489e-05,6907.80795766", Description = "Bitfinex Crypto Coins Exchange Rate")]
+        [TestCase("QDL/OPEC", "date,value", "2024-01-12,80.18", Description = "Organization of the Petroleum Exporting Countries")]
+        [TestCase("QDL/LME", "item_code,country_code,date,opening_stock,delivered_in,delivered_out,closing_stock,open_tonnage,cancelled_tonnage", "ZIJ,UTO,2024-07-12,0.0,0.0,0.0,0.0,0.0,0.0", Description = "Metal Stocks Breakdown Report")]
+        [TestCase("QDL/LME", "item_code,country_code,date,opening_stock,delivered_in,delivered_out,closing_stock,open_tonnage,cancelled_tonnage", "ZII,UNE,2020-07-23,26075.0,0.0,0.0,26075.0,14425.0,11650.0", Description = "Metal Stocks Breakdown Report")]
+        [TestCase("ZILLOW/DATA", "indicator_id,region_id,date,value", "ZSFH,99999,2024-04-30,481777.608668988", Description = "Zillow Real Estate Data")]
+        [TestCase("ZILLOW/DATA", "indicator_id,region_id,date,value", "ZSFH,99993,2008-07-31,139908.0", Description = "Zillow Real Estate Data")]
+        [TestCase("WB/DATA", "series_id,country_code,country_name,year,value", "VC.PKP.TOTL.UN,XKX,Kosovo,2017,357.0", Description = "World Bank Data")]
+        [TestCase("WB/DATA", "series_id,country_code,country_name,year,value", "VC.IHR.PSRC.P5,ALB,Albania,1998,20.4196832752429", Description = "World Bank Data")]
+        [TestCase("WASDE/DATA", "code,report_month,region,commodity,item,year,period,value,min_value,max_value", "WHEAT_WORLD_19,2024-02,World Less China,Wheat,Production,2023/24 Proj.,Jan,648.32,,", Description = "World Agricultural Supply and Demand Estimates")]
+        [TestCase("WASDE/DATA", "code,report_month,region,commodity,item,year,period,value,min_value,max_value", "WHEAT_WORLD_19,2022-08,N. Africa 7/,Wheat,Production,2022/23 Proj.,Jul,17.15,,", Description = "World Agricultural Supply and Demand Estimates")]
+        [TestCase("WASDE/DATA", "code,report_month,region,commodity,item,year,period,value,min_value,max_value", "WHEAT_WORLD_19,2021-05,Brazil,Wheat,Beginning Stocks,2021/22 Proj.,May,0.64,,", Description = "World Agricultural Supply and Demand Estimates")]
+        public void CreateDifferentNasdaqDataSymbolWithVariousProperties(string nasdaqDataName, string csvHeader, string csvData)
+        {
+            var nasdaq = new NasdaqDataLink();
+
+            var symbol = Symbol.Create(nasdaqDataName, SecurityType.Base, "empty");
+
+            var config = new SubscriptionDataConfig(typeof(NasdaqDataLink), symbol, Resolution.Daily, TimeZones.Utc, TimeZones.Utc, true, true, false, true);
+
+            var dateTimeUtcNow = DateTime.UtcNow;
+
+            nasdaq.Reader(config, csvHeader, dateTimeUtcNow, false);
+            var data = nasdaq.Reader(config, csvData, dateTimeUtcNow, false);
+
+            Assert.That(data.Time, Is.Not.EqualTo(default));
+            Assert.GreaterOrEqual(data.Value, 0m);
+        }
+
         [Test]
         public void PythonValueColumn()
         {