Update host.json to add version

.gitignore

@@ -258,4 +258,7 @@ paket-files/
 
 # Python Tools for Visual Studio (PTVS)
 __pycache__/
-*.pyc
+*.pyc
+
+#VSCODE stuff
+.vscode

FunctionAppCSVToJSON/CSVToJSON.cs

@@ -11,45 +11,31 @@
 using System.Collections.Generic;
 using System.Linq;
 using System;
+using CsvHelper;
+using System.Text;
+using Microsoft.Extensions.Logging;
 
 namespace FunctionAppCSVToJSON
 {
 public static class CSVToJSON
 {
     [FunctionName("CSVToJSON")]
-    public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)]HttpRequest req, TraceWriter log)
+    public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)]HttpRequest req, ILogger log)
     {
-        log.Info("C# HTTP trigger function CSVToJSON processed a request.");
+        log.LogInformation("C# HTTP trigger function CSVToJSON processed a request.");
 
-        char[] fieldSeperator = new char[] { ',' };
-
-        string fileName = req.Query["fileName"];            
-        string rowsToSkipStr = req.Query["rowsToSkip"];
+        string fileName = req.Query["fileName"];
         string errorMessage = "";
 
         string requestBody = new StreamReader(req.Body).ReadToEnd();
         dynamic data = JsonConvert.DeserializeObject(requestBody);
-        int rowsToSkip = 0;
-        long lineSkipCounter = 0;
 
         fileName = fileName ?? data?.fileName;          
-        rowsToSkipStr = rowsToSkipStr ?? data?.rowsToSkip;
-
-        if (rowsToSkipStr == null)
-        {
-            errorMessage = "Please pass a rowsToSkip on the query string or in the request body";
-            log.Info("BadRequest: " + errorMessage);
-            return new BadRequestObjectResult(errorMessage);
-        }
-        else
-        {
-            Int32.TryParse(rowsToSkipStr, out rowsToSkip);                
-        }
 
         if (fileName == null)
         {
             errorMessage = "Please pass a fileName on the query string or in the request body";
-            log.Info("BadRequest: " + errorMessage);
+            log.LogInformation("BadRequest: " + errorMessage);
             return new BadRequestObjectResult(errorMessage);
         }            
 
@@ -58,81 +44,74 @@ public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get",
         if (csvData == null)
         {
             errorMessage = "Please pass the csv data in using the csv attribute in the request body";
-            log.Info("BadRequest: " + errorMessage);
+            log.LogInformation("BadRequest: " + errorMessage);
             return new BadRequestObjectResult(errorMessage);
         }
 
-        log.Info("csv data is present.");
-
-        string[] csvLines = ToLines(csvData);
+        log.LogInformation("csv data is present.");
 
-        log.Info(string.Format("There are {0} lines in the csv content {1}.", csvLines.Count(), fileName));
+
 
-        var headers = csvLines[0].Split(fieldSeperator).ToList<string>();
+
 
         JsonResult resultSet = new JsonResult(fileName);
 
+        byte[] byteArray = Encoding.UTF8.GetBytes(csvData);
+        MemoryStream csvStream = new MemoryStream(byteArray);
 
-        foreach (var line in csvLines.Skip(rowsToSkip))
-        {
-            //Check to see if a line is blank.
-            //This can happen on the last row if improperly terminated.
-            if (line != "" || line.Trim().Length > 0 )
-            {
-                var lineObject = new JObject();
-                var fields = line.Split(fieldSeperator);
-
-                for (int x = 0; x < headers.Count; x++)
-                {
-                    lineObject[headers[x]] = fields[x];
-                }
-
-                resultSet.Rows.Add(lineObject);
-            }
-            else
-            {
-                lineSkipCounter += 1;
-            }
+
+        log.LogInformation("translating CSV Data to a list");
+        var records = Convert(csvStream);
+
+        JArray jsonarray = JArray.FromObject(records);
+
+        foreach(var row in jsonarray.Children()){
+            resultSet.Rows.Add(row);
         }
 
-        log.Info(string.Format("There were {0} lines skipped, not including the header row.", lineSkipCounter));
+
+
+        log.LogInformation(string.Format("There are {0} lines in the csv records content {1}.", records.Count(), fileName));
+
+
+
 
         return (ActionResult)new OkObjectResult(resultSet);
     }
+        public static List<object> Convert(Stream blob)
+        {
+
+            var sReader = new StreamReader(blob);
+            var csv = new CsvReader(sReader);
+
+            //log bad data
+            csv.Configuration.BadDataFound = context =>
+                {
+
+                };
 
-    private static string[] ToLines(string dataIn)
-        {            
-            char[] EOLMarkerR = new char[] { '\r' };
-            char[] EOLMarkerN = new char[] { '\n' };
-            char[] EOLMarker = EOLMarkerR;
-
-            //check to see if the file has both \n and \r for end of line markers.
-            //common for files comming from Unix\Linux systems.
-            if (dataIn.IndexOf('\n') > 0 && dataIn.IndexOf('\r') > 0)
-            {
-                //if we find both just remove one of them.
-                dataIn = dataIn.Replace("\n", "");
-            }
-            //If the file only has \n then we will use that as the EOL marker to seperate the lines.
-            else if(dataIn.IndexOf('\n') > 0)
-            {
-                EOLMarker = EOLMarkerN; 
-            }
-
-            //How do we know the dynamic data will have Split capability?
-            return dataIn.Split(EOLMarker);
+
+            csv.Read();
+            csv.ReadHeader();
+
+            var csvRecords = csv.GetRecords<object>().ToList();
+
+            return (csvRecords);
         }
-    }
+        
 
     public class JsonResult
     {
         public JsonResult(string fileName)
         {
-            Rows = new List<object>();
+            Rows = new JArray();
             FileName = fileName;
         }
 
         public string FileName { get; set; }
-        public List<object> Rows { get; set; }
+        public JArray Rows { get; set; }
     }
 }
+
+
+}

FunctionAppCSVToJSON/FunctionAppCSVToJSON.csproj

@@ -1,10 +1,11 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
-    <AzureFunctionsVersion>v2</AzureFunctionsVersion>
+    <TargetFramework>netcoreapp2.2</TargetFramework>
+    <AzureFunctionsVersion></AzureFunctionsVersion>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Sdk.Functions" Version="1.0.8" />
+    <PackageReference Include="csvhelper" Version="12.0.1" />
+    <PackageReference Include="Microsoft.NET.Sdk.Functions" Version="1.0.28" />
   </ItemGroup>
   <ItemGroup>
     <None Update="host.json">
@@ -15,4 +16,4 @@
       <CopyToPublishDirectory>Never</CopyToPublishDirectory>
     </None>
   </ItemGroup>
-</Project>
+</Project>

FunctionAppCSVToJSON/host.json

@@ -1,4 +1,5 @@
 {
+  "version": "2.0",
   "logger": {
     "categoryFilter": {
       "defaultLevel": "Information",
@@ -9,4 +10,4 @@
       }
     }
   }
-}
+}

README.md

@@ -2,25 +2,22 @@
 
 Azure Function App [V2](https://docs.microsoft.com/en-us/azure/azure-functions/functions-versions) that converts CSV data to JSON
 
-I created this Azure Function as part of a larger solution to process incomming SFTP files. One possible soltion is to convert the CSV data to JSON and then perform Azure [SQL Bulk OPENROWSETS Bulk Inserts](https://blogs.msdn.microsoft.com/sqlserverstorageengine/2015/10/07/bulk-importing-json-files-into-sql-server/).
+I Forked this from [aaronralls](https://github.com/aaronralls/FunctionAppCSVToJSON) originally and changed the convert function to leverage [csvhelper](https://joshclose.github.io/CsvHelper/). This Azure Function as part of a larger solution to process incomming SFTP files.
 
 ## Getting Started
 
 These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.
 
 ### Sample Input
 
-The JSON the function accepts consists of the following fields.
-
-rowsToSkip - This indicates the number of rows to skip in the conversion process.
+The JSON the function accepts consists of the following body fields.
 
 fileName - This is the source file where the data came from. This is passed in for downstream processes that may need to know which file was processed.
 
 csv - This is the raw data from the source file that you want to be converted to JSON. This content may contain \r\n or \n end of line markers. The function will detect them an process the csv data accordingly.
 
 ```
 {
-  "rowsToSkip": 1,
   "fileName": "MyTestCSVFile.csv",
   "csv":"ID,Name,Score
 1,Aaron,99
@@ -63,10 +60,11 @@ rows - list of the CSV data in JSON format, with the field name from the header
 What things you need to install the software and how to install them
 
 ```
-Visual Studio 15.5.7 
+Visual Studio Code
 Postman v6.0.7
+Azure Subscription
+Azure funciton app service
 ```
-Download [Postman v6.0.7](https://www.getpostman.com/) 
 
 ### Installing
 
@@ -95,7 +93,7 @@ Explain how to run the automated tests for this system
 These variables used in the Postman tests.
 
 url - This is the URI of the Azure Function that you have published to or use for local testing. (ie: localhost:7071)
-This variable is in each [test collection](https://github.com/aaronralls/FunctionAppCSVToJSON/Postman%20Tests). Be sure to update them both.
+This variable is in each [test collection](https://github.com/houstonjb/FunctionAppCSVToJSON/Postman%20Tests). Be sure to update them both.
 
 ```
 "variable": [
@@ -121,7 +119,7 @@ variable": [
 ```
 
 functions-key - This is the Function Key that you can use to limit access to your Azure functions.
-This variable is only in the [Azure test collection](https://github.com/aaronralls/FunctionAppCSVToJSON/Postman%20Tests/FunctionAppCSVToJSON%20Azure.postman_collection.json).
+This variable is only in the [Azure test collection](https://github.com/houstonjb/FunctionAppCSVToJSON/Postman%20Tests/FunctionAppCSVToJSON%20Azure.postman_collection.json).
 
 ```
 "variable": [
@@ -138,17 +136,7 @@ This variable is only in the [Azure test collection](https://github.com/aaronral
 
 ### Break down into end to end tests
 
-There are two [Postman collections](https://github.com/aaronralls/FunctionAppCSVToJSON/tree/master/Postman%20Tests) that cover the testing of the CSVToJSON function. One for local testing and the other for Azure testing.
-
-#### Local Testing Collection
-
-FunctionAppCSVToJSON 
-
-```
-POST to CSVToJSON with Windows text file contents
 
-Negative Test: POST to CSVToJSON with Windows text file contents, no filename
-```
 
 #### Azure Testing Collection
 
@@ -164,25 +152,27 @@ CSVToJSON swagger
 
 ## Deployment
 
-Add additional notes about how to deploy this on a live system
+deploy to Azure function app using your favorite deployment methods.
 
 ## Built With
 
-
+.net core standard version 2.1 using Visual Studio Code and macOS. 
 
 ## Contributing
 
-Please read [CONTRIBUTING.md](https://gist.github.com/AaronRalls/b24679402957c63ec426) for details on our code of conduct, and the process for submitting pull requests to us.
+send it.
 
 ## Versioning
 
-We use [SemVer](http://semver.org/) for versioning. For the versions available, see the [tags on this repository](https://github.com/AaronRalls/FunctionAppCSVToJSON/tags). 
+We use [SemVer](http://semver.org/) for versioning. For the versions available, see the [releases on this repository](https://github.com/houstonjb/FunctionAppCSVToJSON/releases). 
 
 ## Authors
 
 * **Aaron Ralls** - *Initial work* - [Aaron Ralls](https://github.com/AaronRalls)
 
-See also the list of [contributors](https://github.com/AaronRalls/FunctionAppCSVToJSON/contributors) who participated in this project.
+* **houstonjb** - *Modified* - [houstonjb](https://github.com/houstonjb)
+
+See also the list of [contributors](https://github.com/houstonjb/FunctionAppCSVToJSON/contributors) who participated in this project.
 
 ## License
 
@@ -197,4 +187,3 @@ This project is licensed under the MIT License - see the [LICENSE.md](LICENSE) f
 ## Resources ##
 
 - [Azure Functions Documentation](https://docs.microsoft.com/en-us/azure/azure-functions/)
-- Contact me on twitter [@cajunAA](https://www.twitter.com/cajunAA)