diff --git a/.gitignore b/.gitignore index 3c4efe2..3d15eb0 100644 --- a/.gitignore +++ b/.gitignore @@ -258,4 +258,7 @@ paket-files/ # Python Tools for Visual Studio (PTVS) __pycache__/ -*.pyc \ No newline at end of file +*.pyc + +#VSCODE stuff +.vscode \ No newline at end of file diff --git a/FunctionAppCSVToJSON/CSVToJSON.cs b/FunctionAppCSVToJSON/CSVToJSON.cs index 51bbadf..cbe7b99 100644 --- a/FunctionAppCSVToJSON/CSVToJSON.cs +++ b/FunctionAppCSVToJSON/CSVToJSON.cs @@ -11,45 +11,31 @@ using System.Collections.Generic; using System.Linq; using System; +using CsvHelper; +using System.Text; +using Microsoft.Extensions.Logging; namespace FunctionAppCSVToJSON { public static class CSVToJSON { [FunctionName("CSVToJSON")] - public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)]HttpRequest req, TraceWriter log) + public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)]HttpRequest req, ILogger log) { - log.Info("C# HTTP trigger function CSVToJSON processed a request."); + log.LogInformation("C# HTTP trigger function CSVToJSON processed a request."); - char[] fieldSeperator = new char[] { ',' }; - - string fileName = req.Query["fileName"]; - string rowsToSkipStr = req.Query["rowsToSkip"]; + string fileName = req.Query["fileName"]; string errorMessage = ""; string requestBody = new StreamReader(req.Body).ReadToEnd(); dynamic data = JsonConvert.DeserializeObject(requestBody); - int rowsToSkip = 0; - long lineSkipCounter = 0; fileName = fileName ?? data?.fileName; - rowsToSkipStr = rowsToSkipStr ?? data?.rowsToSkip; - - if (rowsToSkipStr == null) - { - errorMessage = "Please pass a rowsToSkip on the query string or in the request body"; - log.Info("BadRequest: " + errorMessage); - return new BadRequestObjectResult(errorMessage); - } - else - { - Int32.TryParse(rowsToSkipStr, out rowsToSkip); - } if (fileName == null) { errorMessage = "Please pass a fileName on the query string or in the request body"; - log.Info("BadRequest: " + errorMessage); + log.LogInformation("BadRequest: " + errorMessage); return new BadRequestObjectResult(errorMessage); } @@ -58,81 +44,74 @@ public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", if (csvData == null) { errorMessage = "Please pass the csv data in using the csv attribute in the request body"; - log.Info("BadRequest: " + errorMessage); + log.LogInformation("BadRequest: " + errorMessage); return new BadRequestObjectResult(errorMessage); } - log.Info("csv data is present."); - - string[] csvLines = ToLines(csvData); + log.LogInformation("csv data is present."); - log.Info(string.Format("There are {0} lines in the csv content {1}.", csvLines.Count(), fileName)); + - var headers = csvLines[0].Split(fieldSeperator).ToList(); + JsonResult resultSet = new JsonResult(fileName); + byte[] byteArray = Encoding.UTF8.GetBytes(csvData); + MemoryStream csvStream = new MemoryStream(byteArray); - foreach (var line in csvLines.Skip(rowsToSkip)) - { - //Check to see if a line is blank. - //This can happen on the last row if improperly terminated. - if (line != "" || line.Trim().Length > 0 ) - { - var lineObject = new JObject(); - var fields = line.Split(fieldSeperator); - - for (int x = 0; x < headers.Count; x++) - { - lineObject[headers[x]] = fields[x]; - } - - resultSet.Rows.Add(lineObject); - } - else - { - lineSkipCounter += 1; - } + + log.LogInformation("translating CSV Data to a list"); + var records = Convert(csvStream); + + JArray jsonarray = JArray.FromObject(records); + + foreach(var row in jsonarray.Children()){ + resultSet.Rows.Add(row); } - log.Info(string.Format("There were {0} lines skipped, not including the header row.", lineSkipCounter)); + + + log.LogInformation(string.Format("There are {0} lines in the csv records content {1}.", records.Count(), fileName)); + + + return (ActionResult)new OkObjectResult(resultSet); } + public static List Convert(Stream blob) + { + + var sReader = new StreamReader(blob); + var csv = new CsvReader(sReader); + + //log bad data + csv.Configuration.BadDataFound = context => + { + + }; - private static string[] ToLines(string dataIn) - { - char[] EOLMarkerR = new char[] { '\r' }; - char[] EOLMarkerN = new char[] { '\n' }; - char[] EOLMarker = EOLMarkerR; - - //check to see if the file has both \n and \r for end of line markers. - //common for files comming from Unix\Linux systems. - if (dataIn.IndexOf('\n') > 0 && dataIn.IndexOf('\r') > 0) - { - //if we find both just remove one of them. - dataIn = dataIn.Replace("\n", ""); - } - //If the file only has \n then we will use that as the EOL marker to seperate the lines. - else if(dataIn.IndexOf('\n') > 0) - { - EOLMarker = EOLMarkerN; - } - - //How do we know the dynamic data will have Split capability? - return dataIn.Split(EOLMarker); + + csv.Read(); + csv.ReadHeader(); + + var csvRecords = csv.GetRecords().ToList(); + + return (csvRecords); } - } + public class JsonResult { public JsonResult(string fileName) { - Rows = new List(); + Rows = new JArray(); FileName = fileName; } public string FileName { get; set; } - public List Rows { get; set; } + public JArray Rows { get; set; } } } + + +} diff --git a/FunctionAppCSVToJSON/FunctionAppCSVToJSON.csproj b/FunctionAppCSVToJSON/FunctionAppCSVToJSON.csproj index f25dbef..50336a1 100644 --- a/FunctionAppCSVToJSON/FunctionAppCSVToJSON.csproj +++ b/FunctionAppCSVToJSON/FunctionAppCSVToJSON.csproj @@ -1,10 +1,11 @@ - netstandard2.0 - v2 + netcoreapp2.2 + - + + @@ -15,4 +16,4 @@ Never - \ No newline at end of file + diff --git a/FunctionAppCSVToJSON/host.json b/FunctionAppCSVToJSON/host.json index 1bb59ae..27084e4 100644 --- a/FunctionAppCSVToJSON/host.json +++ b/FunctionAppCSVToJSON/host.json @@ -1,4 +1,5 @@ { + "version": "2.0", "logger": { "categoryFilter": { "defaultLevel": "Information", @@ -9,4 +10,4 @@ } } } -} \ No newline at end of file +} diff --git a/README.md b/README.md index 20315c2..d171db6 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Azure Function App [V2](https://docs.microsoft.com/en-us/azure/azure-functions/functions-versions) that converts CSV data to JSON -I created this Azure Function as part of a larger solution to process incomming SFTP files. One possible soltion is to convert the CSV data to JSON and then perform Azure [SQL Bulk OPENROWSETS Bulk Inserts](https://blogs.msdn.microsoft.com/sqlserverstorageengine/2015/10/07/bulk-importing-json-files-into-sql-server/). +I Forked this from [aaronralls](https://github.com/aaronralls/FunctionAppCSVToJSON) originally and changed the convert function to leverage [csvhelper](https://joshclose.github.io/CsvHelper/). This Azure Function as part of a larger solution to process incomming SFTP files. ## Getting Started @@ -10,9 +10,7 @@ These instructions will get you a copy of the project up and running on your loc ### Sample Input -The JSON the function accepts consists of the following fields. - -rowsToSkip - This indicates the number of rows to skip in the conversion process. +The JSON the function accepts consists of the following body fields. fileName - This is the source file where the data came from. This is passed in for downstream processes that may need to know which file was processed. @@ -20,7 +18,6 @@ csv - This is the raw data from the source file that you want to be converted to ``` { - "rowsToSkip": 1, "fileName": "MyTestCSVFile.csv", "csv":"ID,Name,Score 1,Aaron,99 @@ -63,10 +60,11 @@ rows - list of the CSV data in JSON format, with the field name from the header What things you need to install the software and how to install them ``` -Visual Studio 15.5.7 +Visual Studio Code Postman v6.0.7 +Azure Subscription +Azure funciton app service ``` -Download [Postman v6.0.7](https://www.getpostman.com/) ### Installing @@ -95,7 +93,7 @@ Explain how to run the automated tests for this system These variables used in the Postman tests. url - This is the URI of the Azure Function that you have published to or use for local testing. (ie: localhost:7071) -This variable is in each [test collection](https://github.com/aaronralls/FunctionAppCSVToJSON/Postman%20Tests). Be sure to update them both. +This variable is in each [test collection](https://github.com/houstonjb/FunctionAppCSVToJSON/Postman%20Tests). Be sure to update them both. ``` "variable": [ @@ -121,7 +119,7 @@ variable": [ ``` functions-key - This is the Function Key that you can use to limit access to your Azure functions. -This variable is only in the [Azure test collection](https://github.com/aaronralls/FunctionAppCSVToJSON/Postman%20Tests/FunctionAppCSVToJSON%20Azure.postman_collection.json). +This variable is only in the [Azure test collection](https://github.com/houstonjb/FunctionAppCSVToJSON/Postman%20Tests/FunctionAppCSVToJSON%20Azure.postman_collection.json). ``` "variable": [ @@ -138,17 +136,7 @@ This variable is only in the [Azure test collection](https://github.com/aaronral ### Break down into end to end tests -There are two [Postman collections](https://github.com/aaronralls/FunctionAppCSVToJSON/tree/master/Postman%20Tests) that cover the testing of the CSVToJSON function. One for local testing and the other for Azure testing. - -#### Local Testing Collection - -FunctionAppCSVToJSON - -``` -POST to CSVToJSON with Windows text file contents -Negative Test: POST to CSVToJSON with Windows text file contents, no filename -``` #### Azure Testing Collection @@ -164,25 +152,27 @@ CSVToJSON swagger ## Deployment -Add additional notes about how to deploy this on a live system +deploy to Azure function app using your favorite deployment methods. ## Built With - +.net core standard version 2.1 using Visual Studio Code and macOS. ## Contributing -Please read [CONTRIBUTING.md](https://gist.github.com/AaronRalls/b24679402957c63ec426) for details on our code of conduct, and the process for submitting pull requests to us. +send it. ## Versioning -We use [SemVer](http://semver.org/) for versioning. For the versions available, see the [tags on this repository](https://github.com/AaronRalls/FunctionAppCSVToJSON/tags). +We use [SemVer](http://semver.org/) for versioning. For the versions available, see the [releases on this repository](https://github.com/houstonjb/FunctionAppCSVToJSON/releases). ## Authors * **Aaron Ralls** - *Initial work* - [Aaron Ralls](https://github.com/AaronRalls) -See also the list of [contributors](https://github.com/AaronRalls/FunctionAppCSVToJSON/contributors) who participated in this project. +* **houstonjb** - *Modified* - [houstonjb](https://github.com/houstonjb) + +See also the list of [contributors](https://github.com/houstonjb/FunctionAppCSVToJSON/contributors) who participated in this project. ## License @@ -197,4 +187,3 @@ This project is licensed under the MIT License - see the [LICENSE.md](LICENSE) f ## Resources ## - [Azure Functions Documentation](https://docs.microsoft.com/en-us/azure/azure-functions/) -- Contact me on twitter [@cajunAA](https://www.twitter.com/cajunAA)