Skip to content

Commit

Permalink
fix: TopoJSON encoding issue
Browse files Browse the repository at this point in the history 
Use Jackson's ObjectMapper to parse the JSON string into a JsonNode, then recursively traverses the JsonNode to replace "null" values with null. Finally, it converts the modified JsonNode back to a JSON string.

ING-3979
  • Loading branch information
@emanuelaepure10
emanuelaepure10 committed Nov 22, 2023
1 parent a3f4614 commit b0e5c2a
Show file tree
Hide file tree
Showing 4 changed files with 183 additions and 9 deletions.
4 changes: 3 additions & 1 deletion io/plugins/eu.esdihumboldt.hale.io.topojson.test/META-INF/MANIFEST.MF
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ Require-Bundle: eu.esdihumboldt.hale.io.topojson;bundle-version="4.2.0",
eu.esdihumboldt.hale.common.schema.groovy;bundle-version="4.2.0",
eu.esdihumboldt.hale.common.instance.groovy;bundle-version="4.2.0",
org.opengis;bundle-version="21.0.0",
org.eclipse.core.runtime;bundle-version="3.17.100"
org.eclipse.core.runtime;bundle-version="3.17.100",
com.fasterxml.jackson.core.jackson-core;bundle-version="2.13.4",
com.fasterxml.jackson.core.jackson-databind;bundle-version="2.13.4"
Import-Package: de.fhg.igd.slf4jplus,
org.slf4j;version="1.7.2"
39 changes: 34 additions & 5 deletions ...topojson.test/src/eu/esdihumboldt/hale/io/topojson/test/TopoJsonInstanceWriterTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,14 @@
package eu.esdihumboldt.hale.io.topojson.test

import static org.junit.Assert.assertEquals
import static org.junit.Assert.assertNull
import static org.junit.Assert.assertTrue

import java.nio.file.Files
import java.nio.file.Path
import java.time.Instant
import java.time.LocalDate
import java.time.format.DateTimeFormatter
import java.util.function.Consumer

import org.junit.Test
Expand Down Expand Up @@ -56,6 +60,8 @@ class TopoJsonInstanceWriterTest {
SimpleType {
id(String)
name(String)
label(String)
date(LocalDate)
geometry(GeometryProperty)
}
}
Expand Down Expand Up @@ -103,16 +109,33 @@ class TopoJsonInstanceWriterTest {

@Test
public void testWriteTopoJson() {
def aDate = LocalDate.of(2017, 10, 3)
def aTimestamp = Instant.parse('2023-04-02T00:00:00Z')
// Create a DateTimeFormatter with the desired pattern
def formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'");

InstanceCollection instances = new InstanceBuilder(types: schema).createCollection {
SimpleType {
id '1'
name 'Area 1'
label 'Label 1'
date(aDate)
geometry(createGeometry('POLYGON ((10 10, 20 10, 20 20, 10 20, 10 10))', 4326))
}

SimpleType {
id '2'
name 'Area 2'
name '\u0000\u0000\u0000'
label null
date null
geometry(createGeometry('POLYGON ((20 10, 40 10, 40 20, 20 20, 20 10))', 4326))
}

SimpleType {
id '3'
name 'Area 3\u0000\u0000\u0000'
label '\u0000\u0000\u0000'
date null
geometry(createGeometry('POLYGON ((20 10, 40 10, 40 20, 20 20, 20 10))', 4326))
}
}
Expand All @@ -127,25 +150,31 @@ class TopoJsonInstanceWriterTest {
assertEquals(1, json.objects.size())
assertEquals(4, json.arcs.size())

assertEquals(2, json.objects.Topology.geometries.size())
assertEquals(3, json.objects.Topology.geometries.size())

assertEquals(0, json.objects.Topology.geometries[0].id)
assertEquals('Polygon', json.objects.Topology.geometries[0].type)
assertEquals('Area 1', json.objects.Topology.geometries[0].'properties'.name)
assertEquals('1', json.objects.Topology.geometries[0].'properties'.id)
assertEquals('Label 1', json.objects.Topology.geometries[0].'properties'.label)
assertEquals(aDate.toString(), json.objects.Topology.geometries[0].'properties'.date)
assertEquals(1, json.objects.Topology.geometries[0].arcs.size())
assertEquals(2, json.objects.Topology.geometries[0].arcs[0].size())
assertEquals([0, 1], json.objects.Topology.geometries[0].arcs[0])

assertEquals(1, json.objects.Topology.geometries[1].id)
assertEquals('Polygon', json.objects.Topology.geometries[1].type)
assertEquals('Area 2', json.objects.Topology.geometries[1].'properties'.name)
assertNull(json.objects.Topology.geometries[1].'properties'.name)
assertEquals('2', json.objects.Topology.geometries[1].'properties'.id)
assertNull(json.objects.Topology.geometries[1].'properties'.label)
assertNull(json.objects.Topology.geometries[1].'properties'.date)
assertEquals(1, json.objects.Topology.geometries[1].arcs.size())
assertEquals(3, json.objects.Topology.geometries[1].arcs[0].size())
assertEquals([-1, 2, 3], json.objects.Topology.geometries[1].arcs[0])
}

assertEquals('Area 3', json.objects.Topology.geometries[2].'properties'.name)
assertNull(json.objects.Topology.geometries[2].'properties'.label)
assertNull(json.objects.Topology.geometries[2].'properties'.date)
}
}

}
7 changes: 6 additions & 1 deletion io/plugins/eu.esdihumboldt.hale.io.topojson/META-INF/MANIFEST.MF
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,10 @@ Require-Bundle: eu.esdihumboldt.hale.io.shp;bundle-version="4.2.0",
org.eclipse.core.runtime;bundle-version="3.17.100",
org.opengis;bundle-version="21.0.0",
eu.esdihumboldt.util;bundle-version="4.2.0",
eu.esdihumboldt.hale.io.json;bundle-version="4.2.0"
eu.esdihumboldt.hale.io.json;bundle-version="4.2.0",
com.fasterxml.jackson.core.jackson-core;bundle-version="2.13.4",
com.fasterxml.jackson.core.jackson-databind;bundle-version="2.13.4"
Export-Package: eu.esdihumboldt.hale.io.topojson
Import-Package: de.fhg.igd.slf4jplus,
com.fasterxml.jackson.databind;version="2.13.4",
org.slf4j;version="1.5.11"
142 changes: 140 additions & 2 deletions ...umboldt.hale.io.topojson/src/eu/esdihumboldt/hale/io/topojson/TopoJsonInstanceWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,26 @@
import java.io.OutputStream;
import java.net.URI;
import java.nio.file.Paths;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.TimeZone;

import org.opengis.referencing.crs.CoordinateReferenceSystem;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;

import de.fhg.igd.slf4jplus.ALogger;
import de.fhg.igd.slf4jplus.ALoggerFactory;
import eu.esdihumboldt.hale.common.core.io.IOProviderConfigurationException;
import eu.esdihumboldt.hale.common.core.io.ProgressIndicator;
import eu.esdihumboldt.hale.common.core.io.report.IOReport;
Expand All @@ -38,6 +53,8 @@
import eu.esdihumboldt.hale.common.instance.model.InstanceCollection;
import eu.esdihumboldt.hale.common.schema.geometry.CRSDefinition;
import eu.esdihumboldt.hale.io.shp.writer.ShapefileInstanceWriter;
import json.tools.Compress;
import json.tools.Toolbox;
import json.topojson.api.TopojsonApi;

/**
Expand Down Expand Up @@ -86,6 +103,7 @@ public class TopoJsonInstanceWriter extends AbstractInstanceWriter {
private static final String TARGET_CRS_CODE = "EPSG:4326";

private final CRSDefinition targetCrs = new CodeDefinition(TARGET_CRS_CODE, true);
private static final ALogger log = ALoggerFactory.getLogger(TopoJsonInstanceWriter.class);

@Override
protected String getDefaultTypeName() {
Expand Down Expand Up @@ -169,10 +187,130 @@ private List<URI> writeIntermediateShapefiles(InstanceCollection instances,

private void convertShapefileToTopoJson(String shapefilePath,
CoordinateReferenceSystem sourceCrs, String targetPath, String topologyName, int kink,
int quantizeDigit, boolean compressOutput) throws IOException {
int quantizeDigit, boolean compressOutput) throws Exception {

TopojsonApi.shpToTopojsonFile(shapefilePath, sourceCrs, targetPath, topologyName, kink,
// DBFExtractor is using DBFReader, which converts all the dates from
// hh:mm:ss into 00:00:00
String aJson = TopojsonApi.shpToTopojson(shapefilePath, sourceCrs, topologyName, kink,
quantizeDigit, compressOutput);
try {
// Parse JSON string
ObjectMapper mapper = new ObjectMapper();
JsonNode jsonNode = mapper.readTree(aJson);

// Replace "\u0000" values with null recursively
replaceUnwantedCharacters(jsonNode);

// Convert modified JSON node back to string
String modifiedJsonString = mapper.writeValueAsString(jsonNode);

if (compressOutput) {
Toolbox.writeFile(targetPath, Compress.compressB64(modifiedJsonString));
}
else {
Toolbox.writeFile(targetPath, modifiedJsonString);
}

} catch (Exception e) {
throw new Exception(e);
}
}

/**
* @param node to replace the unwanted characters
* @return the clean node
*/
public static JsonNode replaceUnwantedCharacters(JsonNode node) {
if (node.isObject()) {
ObjectNode objectNode = (ObjectNode) node;
objectNode.fields().forEachRemaining(entry -> {
JsonNode value = entry.getValue();
if (value.isTextual()) {

String textValue = value.textValue();
if (textValue.contains("\u0000")) {
if (textValue.replaceAll("\u0000", "").isEmpty()) {
objectNode.put(entry.getKey(), (String) null);
}
else {
objectNode.put(entry.getKey(), textValue.replaceAll("\u0000", ""));
}
}
else {
// Step 1: Parse the string into java.util.Date
try {
Date wrongDateFormat = parseStringToDate(textValue,
"EEE MMM dd HH:mm:ss z yyyy", true);

if (wrongDateFormat != null) {
// Step 2: Convert java.util.Date to
// java.time.Instant
Instant instant = wrongDateFormat.toInstant();

// Step 3: Convert java.time.Instant to
// java.time.LocalDateTime
LocalDateTime localDateTime;
if (wrongDateFormat.toString().contains("CET")) {
// Convert the Date object to LocalDateTime
localDateTime = LocalDateTime
.ofInstant(wrongDateFormat.toInstant(), ZoneOffset.UTC);

}
else {
localDateTime = instant.atZone(ZoneId.systemDefault())
.toLocalDateTime();
}

// Get the year from LocalDateTime
int year = localDateTime.getYear();
if (year == 2) {
objectNode.put(entry.getKey(), (String) null);
}
else {
// Create a DateTimeFormatter object with
// the desired format
DateTimeFormatter formatter = DateTimeFormatter
.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'");

objectNode.put(entry.getKey(), localDateTime.format(formatter));
}

}
else {
objectNode.put(entry.getKey(), textValue);
}

} catch (Exception e) {
log.error("Failed to parse the string " + textValue, e);
objectNode.put(entry.getKey(), (String) null);
}

}
}
else {
replaceUnwantedCharacters(value);
}
});
}
else if (node.isArray()) {
for (JsonNode arrayElement : node) {
replaceUnwantedCharacters(arrayElement);
}
}

return node;
}

private static Date parseStringToDate(String dateString, String pattern, boolean setGMT) {
SimpleDateFormat dateFormat = new SimpleDateFormat(pattern);
if (setGMT) {
dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
}
try {
return dateFormat.parse(dateString);
} catch (ParseException e) {
return null;
}
}

/**
Expand Down

0 comments on commit b0e5c2a

Please sign in to comment.