From 0a82f97cd812774a6e4b82b1b4592a813d61f4dc Mon Sep 17 00:00:00 2001 From: Chris Navarro Date: Mon, 8 Jul 2024 15:40:38 -0500 Subject: [PATCH 1/3] Fixes #299 - upgraded OpenCSV to version 5.9 --- CHANGELOG.md | 1 + server/data-service/build.gradle | 2 +- .../data/controllers/DatasetController.java | 4 ++++ .../incore/service/data/utils/FileUtils.java | 3 ++- .../service/data/utils/GeotoolsUtils.java | 18 ++++++++++++------ .../controllers/DatasetControllerTest.java | 19 +++++++++++++++++++ 6 files changed, 39 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d3023d1..06759f8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Changed - Jersey to version 3.1.7 [#250](https://github.com/IN-CORE/incore-services/issues/250) +- OpenCSV to version 5.9 [#299](https://github.com/IN-CORE/incore-services/issues/299) ## [1.26.1] - 2024-04-08 diff --git a/server/data-service/build.gradle b/server/data-service/build.gradle index a5aae506..401f479b 100644 --- a/server/data-service/build.gradle +++ b/server/data-service/build.gradle @@ -13,7 +13,7 @@ repositories { dependencies { implementation("com.google.inject:guice:7.0.0") - implementation("com.opencsv:opencsv:4.6") + implementation("com.opencsv:opencsv:5.9") implementation("org.jsoup:jsoup:1.16.1") implementation("com.github.lookfirst:sardine:5.12") diff --git a/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetController.java b/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetController.java index 0633e701..33924d67 100644 --- a/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetController.java +++ b/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetController.java @@ -12,6 +12,7 @@ package edu.illinois.ncsa.incore.service.data.controllers; +import com.opencsv.exceptions.CsvValidationException; import edu.illinois.ncsa.incore.common.HazardConstants; import edu.illinois.ncsa.incore.common.auth.Authorizer; import edu.illinois.ncsa.incore.common.auth.IAuthorizer; @@ -974,6 +975,9 @@ public Dataset uploadFiles(@Parameter(name = "Dataset Id from data service", req } catch (IncoreHTTPException e) { logger.error(e.getMessage()); throw new IncoreHTTPException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } catch (CsvValidationException e) { + logger.error(e.getMessage()); + throw new RuntimeException(e); } // clean up FileUtils.deleteTmpDir(geoPkgFile); diff --git a/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/FileUtils.java b/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/FileUtils.java index cc82cefe..735a2f1d 100644 --- a/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/FileUtils.java +++ b/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/FileUtils.java @@ -12,6 +12,7 @@ import com.github.sardine.DavResource; import com.github.sardine.Sardine; import com.github.sardine.SardineFactory; +import com.opencsv.exceptions.CsvValidationException; import edu.illinois.ncsa.incore.common.exceptions.IncoreHTTPException; import edu.illinois.ncsa.incore.service.data.dao.HttpDownloader; import edu.illinois.ncsa.incore.service.data.dao.IRepository; @@ -694,7 +695,7 @@ public static File loadZipdataFromRepository(String inId) throws IOException { * @throws URISyntaxException * @throws IOException */ - public static File joinShpTable(Dataset dataset, IRepository repository, boolean isRename) throws IncoreHTTPException, IOException { + public static File joinShpTable(Dataset dataset, IRepository repository, boolean isRename) throws IncoreHTTPException, IOException, CsvValidationException { List csvFDs = dataset.getFileDescriptors(); File csvFile = null; File shapeFile = null; diff --git a/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java b/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java index 6486eb9e..730b4161 100644 --- a/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java +++ b/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java @@ -10,7 +10,12 @@ package edu.illinois.ncsa.incore.service.data.utils; +import com.opencsv.CSVParser; +import com.opencsv.CSVParserBuilder; import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.exceptions.CsvException; +import com.opencsv.exceptions.CsvValidationException; import edu.illinois.ncsa.incore.common.exceptions.IncoreHTTPException; import edu.illinois.ncsa.incore.common.utils.GeoUtils; import edu.illinois.ncsa.incore.service.data.models.Dataset; @@ -263,7 +268,7 @@ public static SimpleFeatureCollection getSimpleFeatureCollectionFromGeopackage(F * @return * @throws IOException */ - public static File joinTableShapefile(Dataset dataset, List shpfiles, File csvFile, boolean isRename) throws IOException { + public static File joinTableShapefile(Dataset dataset, List shpfiles, File csvFile, boolean isRename) throws IOException, CsvValidationException { String outFileName = FilenameUtils.getBaseName(csvFile.getName()) + "." + FileUtils.EXTENSION_SHP; // create temp dir and copy files to temp dir @@ -354,7 +359,7 @@ public static SimpleFeatureSource createShapefileSource(String shapefilePath) th } public static SimpleFeatureCollection createCsvFeatureFromCsvType(String csvFilePath, SimpleFeatureType featureType) - throws IOException { + throws IOException, CsvValidationException { // create SimpleFeatureCollection from csv file List csvFeatures = new ArrayList<>(); @@ -393,7 +398,7 @@ public static Object convertStringToType(String value, Class outType) { } } - public static SimpleFeatureType createCsvFeatureType(String csvFilePath, String sourceName) throws IOException { + public static SimpleFeatureType createCsvFeatureType(String csvFilePath, String sourceName) throws IOException, CsvValidationException { // read CSV file to get column names and types try (CSVReader reader = new CSVReader(new FileReader(csvFilePath))) { String[] header = reader.readNext(); // assuming the first row is the header @@ -520,7 +525,7 @@ public static void addAttributesWithoutDuplicate(SimpleFeatureTypeBuilder builde } } - public static File joinTableGeopackage(Dataset dataset, String gpkgFileName, File csvFile, boolean isRename) throws IOException { + public static File joinTableGeopackage(Dataset dataset, String gpkgFileName, File csvFile, boolean isRename) throws IOException, CsvException { // set geometry factory GeometryFactory geometryFactory = JTSFactoryFinder.getGeometryFactory(); String outFileName = FilenameUtils.getBaseName(csvFile.getName()) + "." + FileUtils.EXTENSION_SHP; @@ -927,8 +932,9 @@ public static File outToGpkgFile(File geoPkgPathFile, DefaultFeatureCollection c * @throws IOException */ @SuppressWarnings("resource") - public static List readCsvFile(File inCsv) throws IOException { - CSVReader reader = new CSVReader(new FileReader(inCsv), ',', '"', 1); + public static List readCsvFile(File inCsv) throws IOException, CsvException { + CSVParser parser = new CSVParserBuilder().withSeparator(',').withQuoteChar('"').build(); + CSVReader reader = new CSVReaderBuilder(new FileReader(inCsv)).withSkipLines(1).withCSVParser(parser).build(); List rows = reader.readAll(); return rows; } diff --git a/server/data-service/src/test/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetControllerTest.java b/server/data-service/src/test/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetControllerTest.java index 456e7722..e428832b 100644 --- a/server/data-service/src/test/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetControllerTest.java +++ b/server/data-service/src/test/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetControllerTest.java @@ -10,6 +10,10 @@ *******************************************************************************/ package edu.illinois.ncsa.incore.service.data.controllers; +import com.opencsv.CSVParser; +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; import edu.illinois.ncsa.incore.common.HazardConstants; import edu.illinois.ncsa.incore.service.data.models.Dataset; import mocks.MockApplication; @@ -29,7 +33,9 @@ import jakarta.ws.rs.core.Response; import java.io.IOException; import java.io.InputStream; +import java.io.StringReader; import java.net.URL; +import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -185,4 +191,17 @@ public void testFindHazardDatasetTypeWithBadDataType() throws IOException { assertFalse(isHazardDataset); } + @Test + public void testReadCSV() throws Exception { + // Verifies the CSV reader is parsing correctly and ignoring the header + StringBuilder csv = new StringBuilder(); + csv.append("\"guid\",\"LS_0\", \"LS_1\"").append("\n"); + csv.append("\"some-guid-1\",\"0.0\"\"0.0\"").append("\n"); + csv.append("\"some-guid-2\",\"0.0\",\"0.0\""); + CSVParser parser = new CSVParserBuilder().withSeparator(',').withQuoteChar('"').build(); + CSVReader reader = new CSVReaderBuilder(new StringReader(csv.toString())).withSkipLines(1).withCSVParser(parser).build(); + List rows = reader.readAll(); + assertEquals(2, rows.size()); + } + } From ea6a6cf3def4c2ae5d97a8b3242faf7a81563a57 Mon Sep 17 00:00:00 2001 From: Chris Navarro Date: Wed, 10 Jul 2024 11:37:52 -0500 Subject: [PATCH 2/3] Update opencsv to 5.9 for semantics-service --- server/semantics-service/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/semantics-service/build.gradle b/server/semantics-service/build.gradle index 0db207b9..a6028439 100644 --- a/server/semantics-service/build.gradle +++ b/server/semantics-service/build.gradle @@ -7,6 +7,6 @@ dependencies { implementation group: 'org.freemarker', name: 'freemarker', version: '2.3.32' implementation group: 'org.apache.jena', name: 'jena-core', version: '3.1.1' implementation project(':semantic-core') - implementation("com.opencsv:opencsv:4.6") + implementation("com.opencsv:opencsv:5.9") } From 3bbef44ccd508cd3ab23fb9d8a06db3eb602c030 Mon Sep 17 00:00:00 2001 From: Chris Navarro Date: Wed, 10 Jul 2024 11:41:42 -0500 Subject: [PATCH 3/3] Upgraded dom4j to 2.1.3 --- server/build.gradle | 2 +- server/tools-common/build.gradle | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/build.gradle b/server/build.gradle index cfa520ff..2e6ca812 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -157,7 +157,7 @@ test { implementation("com.sun.activation:javax.activation:1.2.0") implementation("de.grundid.opendatalab:geojson-jackson:1.8") implementation("org.jamel.dbf:dbf-reader:0.2.0") - implementation("org.dom4j:dom4j:2.1.1") + implementation("org.dom4j:dom4j:2.1.3") //base jersey and jackson implementation("jakarta.ws.rs:jakarta.ws.rs-api:3.1.0") diff --git a/server/tools-common/build.gradle b/server/tools-common/build.gradle index f4f648c5..e599f2ef 100644 --- a/server/tools-common/build.gradle +++ b/server/tools-common/build.gradle @@ -2,6 +2,6 @@ apply plugin: 'java' dependencies { implementation 'commons-lang:commons-lang:2.6' - implementation 'org.dom4j:dom4j:2.0.0' + implementation 'org.dom4j:dom4j:2.1.3' implementation 'log4j:log4j:1.2.17.norce' -} \ No newline at end of file +}