From cf410c05441cdf97f79dd9ada97732527566fb54 Mon Sep 17 00:00:00 2001 From: Simon Templer Date: Wed, 4 Sep 2024 16:37:30 +0200 Subject: [PATCH] feat(shp): support writing date fields with other bindings than Date - LocalDate - LocalDateTime - Instant Please note that the time portion gets lost because Geotools by default uses the date only type in DBF. --- .../resources/main/plugin.xml | 3 + .../core/LocalDateTimeToDateConverter.java | 36 ++++++++ .../eu.esdihumboldt.hale.io.shp/build.gradle | 2 + .../shp/writer/ShapefileInstanceWriter.java | 89 ++++++++++++++++--- .../io/shp/ShapefileInstanceWriterTest.groovy | 84 ++++++++++++++++- .../build.gradle | 2 + .../io/topojson/TopoJsonInstanceWriter.java | 38 +++----- .../test/TopoJsonInstanceWriterTest.groovy | 3 +- 8 files changed, 218 insertions(+), 39 deletions(-) create mode 100644 common/plugins/eu.esdihumboldt.hale.common.convert.core/src/eu/esdihumboldt/hale/common/convert/core/LocalDateTimeToDateConverter.java diff --git a/common/plugins/eu.esdihumboldt.hale.common.convert.core/resources/main/plugin.xml b/common/plugins/eu.esdihumboldt.hale.common.convert.core/resources/main/plugin.xml index c6b0753cd1..79315cc1cf 100644 --- a/common/plugins/eu.esdihumboldt.hale.common.convert.core/resources/main/plugin.xml +++ b/common/plugins/eu.esdihumboldt.hale.common.convert.core/resources/main/plugin.xml @@ -48,6 +48,9 @@ + + diff --git a/common/plugins/eu.esdihumboldt.hale.common.convert.core/src/eu/esdihumboldt/hale/common/convert/core/LocalDateTimeToDateConverter.java b/common/plugins/eu.esdihumboldt.hale.common.convert.core/src/eu/esdihumboldt/hale/common/convert/core/LocalDateTimeToDateConverter.java new file mode 100644 index 0000000000..6fc4f761e0 --- /dev/null +++ b/common/plugins/eu.esdihumboldt.hale.common.convert.core/src/eu/esdihumboldt/hale/common/convert/core/LocalDateTimeToDateConverter.java @@ -0,0 +1,36 @@ + +/* + * Copyright (c) 2024 wetransform GmbH + * + * All rights reserved. This program and the accompanying materials are made + * available under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the License, + * or (at your option) any later version. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this distribution. If not, see . + */ +package eu.esdihumboldt.hale.common.convert.core; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Date; + +import org.springframework.core.convert.converter.Converter; + +/** + * Convert a {@link LocalDateTime} to a {@link Date}. + * + * @author Simon Templer + */ +public class LocalDateTimeToDateConverter implements Converter { + + @Override + public Date convert(LocalDateTime source) { + if (source == null) { + return null; + } + return Date.from(source.atZone(ZoneId.systemDefault()).toInstant()); + } + +} diff --git a/io/plugins/eu.esdihumboldt.hale.io.shp/build.gradle b/io/plugins/eu.esdihumboldt.hale.io.shp/build.gradle index dd4c4efb59..3b065e9e08 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.shp/build.gradle +++ b/io/plugins/eu.esdihumboldt.hale.io.shp/build.gradle @@ -31,6 +31,8 @@ dependencies { implementation project(':util:plugins:eu.esdihumboldt.util') implementation project(':util:plugins:eu.esdihumboldt.util.groovy.sandbox') + runtimeOnly project(':common:plugins:eu.esdihumboldt.hale.common.convert.core') + testImplementation testLibs.junit4 testImplementation testLibs.assertj diff --git a/io/plugins/eu.esdihumboldt.hale.io.shp/src/eu/esdihumboldt/hale/io/shp/writer/ShapefileInstanceWriter.java b/io/plugins/eu.esdihumboldt.hale.io.shp/src/eu/esdihumboldt/hale/io/shp/writer/ShapefileInstanceWriter.java index 20c7ac2be3..1df293242d 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.shp/src/eu/esdihumboldt/hale/io/shp/writer/ShapefileInstanceWriter.java +++ b/io/plugins/eu.esdihumboldt.hale.io.shp/src/eu/esdihumboldt/hale/io/shp/writer/ShapefileInstanceWriter.java @@ -19,15 +19,11 @@ import java.net.URI; import java.nio.file.FileSystems; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.*; import java.util.Map.Entry; -import java.util.Set; import java.util.stream.Collectors; import org.geotools.data.DefaultTransaction; @@ -44,13 +40,18 @@ import org.opengis.feature.simple.SimpleFeature; import org.opengis.feature.simple.SimpleFeatureType; import org.opengis.feature.type.Name; +import org.springframework.core.convert.ConversionService; +import com.google.common.collect.ImmutableMap; + +import eu.esdihumboldt.hale.common.core.HalePlatform; import eu.esdihumboldt.hale.common.core.io.IOProviderConfigurationException; import eu.esdihumboldt.hale.common.core.io.ProgressIndicator; import eu.esdihumboldt.hale.common.core.io.report.IOReport; import eu.esdihumboldt.hale.common.core.io.report.IOReporter; import eu.esdihumboldt.hale.common.core.io.report.impl.IOMessageImpl; import eu.esdihumboldt.hale.common.core.io.supplier.MultiLocationOutputSupplier; +import eu.esdihumboldt.hale.common.core.report.SimpleLog; import eu.esdihumboldt.hale.common.instance.geometry.GeometryFinder; import eu.esdihumboldt.hale.common.instance.groovy.InstanceAccessor; import eu.esdihumboldt.hale.common.instance.helper.DepthFirstInstanceTraverser; @@ -82,6 +83,22 @@ public class ShapefileInstanceWriter extends AbstractGeoInstanceWriter { */ public static final String ID = "eu.esdihumboldt.hale.io.shp.instance.writer"; + /** + * Map for different bindings to use for feature type fields for encountered + * types. + */ + public static final Map, Class> VALID_BINDING_MAP = ImmutableMap + ., Class> builder() // + // Java 8 date + time + // + // Please note that regardless if there is a time component, Geotools will by + // default only use the date part when writing the DBF file, see + // https://github.com/geotools/geotools/blob/f802eb83131e1f7f346007791ce3a8bdde165ede/modules/plugin/shapefile/src/main/java/org/geotools/data/shapefile/ShapefileDataStore.java#L379 + .put(LocalDate.class, Date.class) // + .put(LocalDateTime.class, Date.class) // + .put(Instant.class, Date.class) // + .build(); // + /** * Regular expression to split the camelCase, the snake_case, or the * alphanumeric string. @@ -299,6 +316,8 @@ private void writePropertiesSchema(Instance instance, TypeDefinition type, type); for (PropertyDefinition prop : allNonComplexProperties) { Class binding = prop.getPropertyType().getConstraint(Binding.class).getBinding(); + binding = toValidBinding(binding); + // ignore geometry and filename properties. if (!prop.getPropertyType().getConstraint(GeometryType.class).isGeometry() && !prop.getName().getNamespaceURI() @@ -317,6 +336,34 @@ private void writePropertiesSchema(Instance instance, TypeDefinition type, } } + /** + * Not all kind of bindings that might be used are supported to be written. Map + * to a valid binding where possible / known. + * + * @param binding the original binding + * @return the binding to use for the original binding, or the original binding + * if that is supported or no mapping known + */ + private Class toValidBinding(Class binding) { + /* + * Supported binding can be found here: + * + * spotless:off + * https://github.com/geotools/geotools/blob/f802eb83131e1f7f346007791ce3a8bdde165ede/modules/plugin/shapefile/src/main/java/org/geotools/data/shapefile/ShapefileDataStore.java#L344 + * spotless:on + */ + if (binding == null) { + return null; + } + + Class mappedBinding = VALID_BINDING_MAP.get(binding); + + if (mappedBinding != null) { + return mappedBinding; + } + return binding; + } + /** * Method to truncate the property names up to 10 characters by splitting them * from camelCase, snake_case, or alphanumeric characters. E.g.
@@ -564,7 +611,7 @@ private Map>> createFeatures( if (schemaFtMap.containsKey(localPart)) { writeGeometryInstanceData(reporter, schemaFbMap, instance, localPart); // add data for the rest of the properties. - writePropertiesInstanceData(schemaFbMap, instance, type, localPart); + writePropertiesInstanceData(schemaFbMap, instance, type, localPart, reporter); // create list of simple features. // fix in case geometries have multiple geometry types but @@ -618,7 +665,7 @@ private void writeGeometryInstanceData(IOReporter reporter, */ private void writePropertiesInstanceData( Map> schemaFbMap, Instance instance, - TypeDefinition type, String localPart) { + TypeDefinition type, String localPart, SimpleLog log) { Collection allNonComplexProperties = getNonComplexProperties( type); for (PropertyDefinition prop : allNonComplexProperties) { @@ -628,6 +675,28 @@ private void writePropertiesInstanceData( && prop.getName().getLocalPart() != null) { Object value = new InstanceAccessor(instance) .findChildren(prop.getName().getLocalPart()).value(); + + if (value != null) { + Class binding = toValidBinding( + prop.getPropertyType().getConstraint(Binding.class).getBinding()); + if (!binding.isAssignableFrom(value.getClass())) { + // convert value + ConversionService cs = HalePlatform.getService(ConversionService.class); + if (cs != null) { + try { + value = cs.convert(value, binding); + } catch (Exception e) { + log.error("Could not convert value to binding {0}", binding, e); + } + } + else { + log.error( + "Could not access conversion service to convert to binding {0}", + binding); + } + } + } + List> geoms = traverseInstanceForGeometries(instance); // add value by traversing geometryType from instance for (GeometryProperty geoProp : geoms) { diff --git a/io/plugins/eu.esdihumboldt.hale.io.shp/test/eu/esdihumboldt/hale/io/shp/ShapefileInstanceWriterTest.groovy b/io/plugins/eu.esdihumboldt.hale.io.shp/test/eu/esdihumboldt/hale/io/shp/ShapefileInstanceWriterTest.groovy index 60501b77cc..d5eb53e8fb 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.shp/test/eu/esdihumboldt/hale/io/shp/ShapefileInstanceWriterTest.groovy +++ b/io/plugins/eu.esdihumboldt.hale.io.shp/test/eu/esdihumboldt/hale/io/shp/ShapefileInstanceWriterTest.groovy @@ -12,6 +12,7 @@ */ package eu.esdihumboldt.hale.io.shp +import static org.assertj.core.api.Assertions.* import static org.junit.Assert.assertEquals import static org.junit.Assert.assertFalse import static org.junit.Assert.assertTrue @@ -23,6 +24,8 @@ import java.nio.file.Path import java.nio.file.Paths import java.time.Instant import java.time.LocalDate +import java.time.LocalDateTime +import java.time.ZoneId import java.util.function.Consumer import org.geotools.data.shapefile.ShapefileDataStore @@ -1207,8 +1210,8 @@ class ShapefileInstanceWriterTest extends AbstractPlatformTest { assert name def a_date = inst.p.a_date.value() - assert a_date instanceof String - assert a_date == aDate.toString() + assert a_date instanceof Date + assert a_date == Date.from(aDate.atStartOfDay(ZoneId.systemDefault()).toInstant()) def a_timestamp = inst.p.a_timestam.value() assert a_timestamp == null def ati = inst.p.ati.value() @@ -1549,4 +1552,81 @@ class ShapefileInstanceWriterTest extends AbstractPlatformTest { assertEquals(2, num) } } + + @Test + void testWriteDates() { + Schema schema = new SchemaBuilder().schema { + abc { + a(Date) + b(LocalDate) + c(Instant) + d(LocalDateTime) + location(GeometryProperty) + } + } + + def now = Instant.now() + def localDate = now.atZone(ZoneId.systemDefault()).toLocalDate() + def localDateTime = now.atZone(ZoneId.systemDefault()).toLocalDateTime() + def date = Date.from(now) + + def expLocalDate = Date.from(localDate.atStartOfDay(ZoneId.systemDefault()).toInstant()) + + InstanceCollection instances = new InstanceBuilder(types: schema).createCollection { + abc { + a(date) + b(localDate) + c(now) + d(localDateTime) + location( createGeometry('POINT(49.872833 8.651222)', 4326)) + } + } + + withNewShapefile(schema, instances) { file -> + // load instances again and test + def loaded = loadInstances(file) + int num = 0 + loaded.iterator().withCloseable { + while (it.hasNext()) { + Instance inst = it.next() + num++ + + // test instance + + /* + * Note: Geotools will for Date binding always use a D field in the DBF which represents a date only, not a time. + * To store data as date and time (@ field), a system property would need to be set, but this would then apply for all fields: + * https://github.com/geotools/geotools/blob/f802eb83131e1f7f346007791ce3a8bdde165ede/modules/plugin/shapefile/src/main/java/org/geotools/data/shapefile/ShapefileDataStore.java#L379 + */ + + assertThat(inst.p.a.value() as Object) + .isNotNull() + .isInstanceOf(Date) + .isEqualTo(expLocalDate) + + assertThat(inst.p.b.value() as Object) + .isNotNull() + .isInstanceOf(Date) + .isEqualTo(expLocalDate) + + assertThat(inst.p.c.value() as Object) + .isNotNull() + .isInstanceOf(Date) + .isEqualTo(expLocalDate) + + assertThat(inst.p.d.value() as Object) + .isNotNull() + .isInstanceOf(Date) + .isEqualTo(expLocalDate) + + def the_geom = inst.p.the_geom.value() + assert the_geom + assert the_geom instanceof GeometryProperty + } + } + + // two instances were loaded + assertEquals(1, num) + } + } } diff --git a/io/plugins/eu.esdihumboldt.hale.io.topojson/build.gradle b/io/plugins/eu.esdihumboldt.hale.io.topojson/build.gradle index 479c5cdb61..9671f45816 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.topojson/build.gradle +++ b/io/plugins/eu.esdihumboldt.hale.io.topojson/build.gradle @@ -25,6 +25,8 @@ dependencies { implementation project(':io:plugins:eu.esdihumboldt.hale.io.shp') implementation project(':io:plugins:eu.esdihumboldt.hale.io.json') + runtimeOnly project(':common:plugins:eu.esdihumboldt.hale.common.convert.core') + testImplementation testLibs.junit4 testImplementation libs.groovy.json diff --git a/io/plugins/eu.esdihumboldt.hale.io.topojson/src/eu/esdihumboldt/hale/io/topojson/TopoJsonInstanceWriter.java b/io/plugins/eu.esdihumboldt.hale.io.topojson/src/eu/esdihumboldt/hale/io/topojson/TopoJsonInstanceWriter.java index 583a0ad82f..8efaf85353 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.topojson/src/eu/esdihumboldt/hale/io/topojson/TopoJsonInstanceWriter.java +++ b/io/plugins/eu.esdihumboldt.hale.io.topojson/src/eu/esdihumboldt/hale/io/topojson/TopoJsonInstanceWriter.java @@ -19,10 +19,7 @@ import java.nio.file.Paths; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZoneOffset; +import java.time.*; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -249,9 +246,11 @@ else if (textValue.equals("null")) { } else { // Step 1: Parse the string into java.util.Date + // Note: The Geotools based Shapefile writer by default only creates dates + // w/o time component try { Date wrongDateFormat = parseStringToDate(textValue, - "EEE MMM dd HH:mm:ss z yyyy", true); + "EEE MMM dd HH:mm:ss z yyyy", false); if (wrongDateFormat == null) { wrongDateFormat = parseStringToDate(textValue, "yyyy-mm-dd", true); @@ -260,33 +259,20 @@ else if (textValue.equals("null")) { // Step 2: Convert java.util.Date to // java.time.Instant Instant instant = wrongDateFormat.toInstant(); - - // Step 3: Convert java.time.Instant to - // java.time.LocalDateTime - LocalDateTime localDateTime; - if (wrongDateFormat.toString().contains("CET")) { - // Convert the Date object to LocalDateTime - localDateTime = LocalDateTime - .ofInstant(wrongDateFormat.toInstant(), ZoneOffset.UTC); - - } - else { - localDateTime = instant.atZone(ZoneId.systemDefault()) - .toLocalDateTime(); - } + // assume only date because of default behavior of Shapefile writer + LocalDate localDate = instant.atZone(ZoneId.systemDefault()) + .toLocalDate(); // Get the year from LocalDateTime - int year = localDateTime.getYear(); + int year = localDate.getYear(); if (year == 2) { + // assume problem parsing date + // XXX seems to have occurred at some point, but reproduction + // case is unclear objectNode.put(entry.getKey(), (String) null); } else { - // Create a DateTimeFormatter object with -// // the desired format -// DateTimeFormatter formatter = DateTimeFormatter -// .ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); - - objectNode.put(entry.getKey(), textValue); + objectNode.put(entry.getKey(), localDate.toString()); } } diff --git a/io/plugins/eu.esdihumboldt.hale.io.topojson/test/eu/esdihumboldt/hale/io/topojson/test/TopoJsonInstanceWriterTest.groovy b/io/plugins/eu.esdihumboldt.hale.io.topojson/test/eu/esdihumboldt/hale/io/topojson/test/TopoJsonInstanceWriterTest.groovy index 37b15b780d..6c24d3fd4c 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.topojson/test/eu/esdihumboldt/hale/io/topojson/test/TopoJsonInstanceWriterTest.groovy +++ b/io/plugins/eu.esdihumboldt.hale.io.topojson/test/eu/esdihumboldt/hale/io/topojson/test/TopoJsonInstanceWriterTest.groovy @@ -22,6 +22,7 @@ import groovy.transform.CompileStatic import java.nio.file.Files import java.nio.file.Path import java.time.LocalDate +import java.time.ZoneId import java.util.function.Consumer import org.junit.Test @@ -176,7 +177,7 @@ class TopoJsonInstanceWriterTest extends AbstractPlatformTest { assertEquals('Area 1', json.objects.Topology.geometries[0].'properties'.name) assertEquals(1, json.objects.Topology.geometries[0].'properties'.id) assertEquals('Label 1', json.objects.Topology.geometries[0].'properties'.label) - assertEquals("2023-11-28", json.objects.Topology.geometries[0].'properties'.date) + assertEquals('2023-11-28', json.objects.Topology.geometries[0].'properties'.date) assertEquals(1, json.objects.Topology.geometries[0].'properties'.fiin) assertEquals(1.2, json.objects.Topology.geometries[0].'properties'.fido) assertEquals(true, json.objects.Topology.geometries[0].'properties'.fibo)