Fixes #299 - upgraded OpenCSV to version 5.9

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Changed
 - Jersey to version 3.1.7 [#250](https://github.com/IN-CORE/incore-services/issues/250)
+- OpenCSV to version 5.9 [#299](https://github.com/IN-CORE/incore-services/issues/299)
 
 ## [1.26.1] - 2024-04-08
 

server/build.gradle

@@ -157,7 +157,7 @@ test {
             implementation("com.sun.activation:javax.activation:1.2.0")
             implementation("de.grundid.opendatalab:geojson-jackson:1.8")
             implementation("org.jamel.dbf:dbf-reader:0.2.0")
-            implementation("org.dom4j:dom4j:2.1.1")
+            implementation("org.dom4j:dom4j:2.1.3")
 
             //base jersey and jackson
             implementation("jakarta.ws.rs:jakarta.ws.rs-api:3.1.0")

server/data-service/build.gradle

@@ -13,7 +13,7 @@ repositories {
 dependencies {
     implementation("com.google.inject:guice:7.0.0")
 
-    implementation("com.opencsv:opencsv:4.6")
+    implementation("com.opencsv:opencsv:5.9")
 
     implementation("org.jsoup:jsoup:1.16.1")
     implementation("com.github.lookfirst:sardine:5.12")

server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetController.java

@@ -12,6 +12,7 @@
 
 package edu.illinois.ncsa.incore.service.data.controllers;
 
+import com.opencsv.exceptions.CsvValidationException;
 import edu.illinois.ncsa.incore.common.HazardConstants;
 import edu.illinois.ncsa.incore.common.auth.Authorizer;
 import edu.illinois.ncsa.incore.common.auth.IAuthorizer;
@@ -974,6 +975,9 @@ public Dataset uploadFiles(@Parameter(name = "Dataset Id from data service", req
                 } catch (IncoreHTTPException e) {
                     logger.error(e.getMessage());
                     throw new IncoreHTTPException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage());
+                } catch (CsvValidationException e) {
+                    logger.error(e.getMessage());
+                    throw new RuntimeException(e);
                 }
                 // clean up
                 FileUtils.deleteTmpDir(geoPkgFile);

server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/FileUtils.java

@@ -12,6 +12,7 @@
 import com.github.sardine.DavResource;
 import com.github.sardine.Sardine;
 import com.github.sardine.SardineFactory;
+import com.opencsv.exceptions.CsvValidationException;
 import edu.illinois.ncsa.incore.common.exceptions.IncoreHTTPException;
 import edu.illinois.ncsa.incore.service.data.dao.HttpDownloader;
 import edu.illinois.ncsa.incore.service.data.dao.IRepository;
@@ -694,7 +695,7 @@ public static File loadZipdataFromRepository(String inId) throws IOException {
      * @throws URISyntaxException
      * @throws IOException
      */
-    public static File joinShpTable(Dataset dataset, IRepository repository, boolean isRename) throws IncoreHTTPException, IOException {
+    public static File joinShpTable(Dataset dataset, IRepository repository, boolean isRename) throws IncoreHTTPException, IOException, CsvValidationException {
         List<FileDescriptor> csvFDs = dataset.getFileDescriptors();
         File csvFile = null;
         File shapeFile = null;

server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java

@@ -10,7 +10,12 @@
 
 package edu.illinois.ncsa.incore.service.data.utils;
 
+import com.opencsv.CSVParser;
+import com.opencsv.CSVParserBuilder;
 import com.opencsv.CSVReader;
+import com.opencsv.CSVReaderBuilder;
+import com.opencsv.exceptions.CsvException;
+import com.opencsv.exceptions.CsvValidationException;
 import edu.illinois.ncsa.incore.common.exceptions.IncoreHTTPException;
 import edu.illinois.ncsa.incore.common.utils.GeoUtils;
 import edu.illinois.ncsa.incore.service.data.models.Dataset;
@@ -263,7 +268,7 @@ public static SimpleFeatureCollection getSimpleFeatureCollectionFromGeopackage(F
      * @return
      * @throws IOException
      */
-    public static File joinTableShapefile(Dataset dataset, List<File> shpfiles, File csvFile, boolean isRename) throws IOException {
+    public static File joinTableShapefile(Dataset dataset, List<File> shpfiles, File csvFile, boolean isRename) throws IOException, CsvValidationException {
         String outFileName = FilenameUtils.getBaseName(csvFile.getName()) + "." + FileUtils.EXTENSION_SHP;
 
         // create temp dir and copy files to temp dir
@@ -354,7 +359,7 @@ public static SimpleFeatureSource createShapefileSource(String shapefilePath) th
     }
 
     public static SimpleFeatureCollection createCsvFeatureFromCsvType(String csvFilePath, SimpleFeatureType featureType)
-        throws IOException {
+        throws IOException, CsvValidationException {
         // create SimpleFeatureCollection from csv file
         List<SimpleFeature> csvFeatures = new ArrayList<>();
 
@@ -393,7 +398,7 @@ public static Object convertStringToType(String value, Class<?> outType) {
         }
     }
 
-    public static SimpleFeatureType createCsvFeatureType(String csvFilePath, String sourceName) throws IOException {
+    public static SimpleFeatureType createCsvFeatureType(String csvFilePath, String sourceName) throws IOException, CsvValidationException {
         // read CSV file to get column names and types
         try (CSVReader reader = new CSVReader(new FileReader(csvFilePath))) {
             String[] header = reader.readNext(); // assuming the first row is the header
@@ -520,7 +525,7 @@ public static void addAttributesWithoutDuplicate(SimpleFeatureTypeBuilder builde
         }
     }
 
-    public static File joinTableGeopackage(Dataset dataset, String gpkgFileName, File csvFile, boolean isRename) throws IOException {
+    public static File joinTableGeopackage(Dataset dataset, String gpkgFileName, File csvFile, boolean isRename) throws IOException, CsvException {
         // set geometry factory
         GeometryFactory geometryFactory = JTSFactoryFinder.getGeometryFactory();
         String outFileName = FilenameUtils.getBaseName(csvFile.getName()) + "." + FileUtils.EXTENSION_SHP;
@@ -927,8 +932,9 @@ public static File outToGpkgFile(File geoPkgPathFile, DefaultFeatureCollection c
      * @throws IOException
      */
     @SuppressWarnings("resource")
-    public static List<String[]> readCsvFile(File inCsv) throws IOException {
-        CSVReader reader = new CSVReader(new FileReader(inCsv), ',', '"', 1);
+    public static List<String[]> readCsvFile(File inCsv) throws IOException, CsvException {
+	CSVParser parser = new CSVParserBuilder().withSeparator(',').withQuoteChar('"').build();
+	CSVReader reader = new CSVReaderBuilder(new FileReader(inCsv)).withSkipLines(1).withCSVParser(parser).build();
         List<String[]> rows = reader.readAll();
         return rows;
     }

server/data-service/src/test/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetControllerTest.java

@@ -10,6 +10,10 @@
  *******************************************************************************/
 package edu.illinois.ncsa.incore.service.data.controllers;
 
+import com.opencsv.CSVParser;
+import com.opencsv.CSVParserBuilder;
+import com.opencsv.CSVReader;
+import com.opencsv.CSVReaderBuilder;
 import edu.illinois.ncsa.incore.common.HazardConstants;
 import edu.illinois.ncsa.incore.service.data.models.Dataset;
 import mocks.MockApplication;
@@ -29,7 +33,9 @@
 import jakarta.ws.rs.core.Response;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.StringReader;
 import java.net.URL;
+import java.util.List;
 
 import static org.junit.jupiter.api.Assertions.*;
 
@@ -185,4 +191,17 @@ public void testFindHazardDatasetTypeWithBadDataType() throws IOException {
         assertFalse(isHazardDataset);
     }
 
+    @Test
+    public void testReadCSV() throws Exception {
+        // Verifies the CSV reader is parsing correctly and ignoring the header
+        StringBuilder csv = new StringBuilder();
+        csv.append("\"guid\",\"LS_0\", \"LS_1\"").append("\n");
+        csv.append("\"some-guid-1\",\"0.0\"\"0.0\"").append("\n");
+        csv.append("\"some-guid-2\",\"0.0\",\"0.0\"");
+        CSVParser parser = new CSVParserBuilder().withSeparator(',').withQuoteChar('"').build();
+        CSVReader reader = new CSVReaderBuilder(new StringReader(csv.toString())).withSkipLines(1).withCSVParser(parser).build();
+        List<String[]> rows = reader.readAll();
+        assertEquals(2, rows.size());
+    }
+
 }

server/semantics-service/build.gradle

@@ -7,6 +7,6 @@ dependencies {
     implementation group: 'org.freemarker', name: 'freemarker', version: '2.3.32'
     implementation group: 'org.apache.jena', name: 'jena-core', version: '3.1.1'
     implementation project(':semantic-core')
-    implementation("com.opencsv:opencsv:4.6")
+    implementation("com.opencsv:opencsv:5.9")
 }
 

server/tools-common/build.gradle

@@ -2,6 +2,6 @@ apply plugin: 'java'
 
 dependencies {
     implementation 'commons-lang:commons-lang:2.6'
-    implementation 'org.dom4j:dom4j:2.0.0'
+    implementation 'org.dom4j:dom4j:2.1.3'
     implementation 'log4j:log4j:1.2.17.norce'
-}
+}