Skip to content

Commit

Permalink
feat/ING-3911 (#1033)
Browse files Browse the repository at this point in the history 
* Update test and writer

From the test class the whenReadWithBufferedReader_thenCorrect() and testCPGFileCreation() have been removed and a new method testCpgFileAgainstRelatedSpapefile() called from all the Test methods.
The writer has been updated taking the charset from the ShapefileDataStore of the related shapefile or adding the default charset in case the above one is null

feat: extend shapefile writer to create .cpg file

Create .cpg file/files when one/more .shp files are created.
The .cpg file contain only the encoding used to write the shapefile.
Create test for creating .cpg file

* feat: extend shapefile writer to create .cpg file

Create .cpg file/files when one/more .shp files are created.
The .cpg file contain only the encoding used to write the shapefile.
Create test for checking .cpg file content against related .shp file

* Set the charset to the writer as well as is done in the reader

Update test containing text with special characters

Decode the encoded text containing special characters
  • Loading branch information
@emanuelaepure10
emanuelaepure10 committed Jul 7, 2023
1 parent ed9c173 commit 6083214
Show file tree
Hide file tree
Showing 3 changed files with 91 additions and 19 deletions.
70 changes: 51 additions & 19 deletions ...boldt.hale.io.shp.test/src/eu/esdihumboldt/hale/io/shp/ShapefileInstanceWriterTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import java.time.Instant
import java.time.LocalDate
import java.util.function.Consumer

import org.geotools.data.shapefile.ShapefileDataStore
import org.junit.Test
import org.locationtech.jts.geom.Coordinate
import org.locationtech.jts.geom.Geometry
Expand Down Expand Up @@ -100,7 +101,7 @@ class ShapefileInstanceWriterTest {
String filenameOnly = Paths.get(location).getFileName().toString();

filenameOnly = filenameOnly.substring(0, filenameOnly.lastIndexOf("."));
String filename = filePath + "/" + filenameOnly + "_" + additionalName + ".shp";
String filename = filePath + "/" + filenameOnly + "_" + additionalName + ShapefileConstants.SHP_EXTENSION;
file = new File(filename)
Schema schema = loadSchema(file)

Expand All @@ -114,10 +115,11 @@ class ShapefileInstanceWriterTest {
assertTrue(report.isSuccess())
assertTrue(report.getErrors().isEmpty())

testCpgFileAgainstRelatedShapefile(file)

return reader.getInstances();
}


/**
* Write an instance collection to a Shapefile.
*/
Expand Down Expand Up @@ -183,12 +185,26 @@ class ShapefileInstanceWriterTest {
try {
println "Temporary file is $tmpFile"
writeInstances(tmpFile.toFile(), schema, instances, configurator)
testCpgFileAgainstRelatedShapefile(tmpFile.toFile())

handler.accept(tmpFile.toFile())
} finally {
tmpDir.deleteDir()
}
}

private static testCpgFileAgainstRelatedShapefile(File shpFile) {
ShapefileInstanceWriter shapefileInstanceWriter = new ShapefileInstanceWriter()
String cpgFilePath = shpFile.getAbsolutePath().replace(ShapefileConstants.SHP_EXTENSION, ShapefileConstants.CPG_EXTENSION)

def cpgFile = new File(cpgFilePath)
if (cpgFile.exists()) {
assertTrue(shapefileInstanceWriter.getCharset().toString().equals(cpgFile.text))
} else {
println("File not found.")
}
}

//@CompileStatic
static void withNewShapefileWithReporterErrors(Schema schema, InstanceCollection instances, Consumer<File> handler,
Consumer<ShapefileInstanceWriter> configurator = null) {
Expand All @@ -198,6 +214,8 @@ class ShapefileInstanceWriterTest {
try {
println "Temporary file is $tmpFile"
writeInstancesWithReporterErrors(tmpFile.toFile(), schema, instances, configurator)
testCpgFileAgainstRelatedShapefile(tmpFile.toFile())

handler.accept(tmpFile.toFile())
} finally {
tmpDir.deleteDir()
Expand Down Expand Up @@ -240,6 +258,7 @@ class ShapefileInstanceWriterTest {
num++
}
}

// 593 instances were loaded
assertEquals(593, num)
}
Expand Down Expand Up @@ -276,6 +295,7 @@ class ShapefileInstanceWriterTest {
// load instances again and test

def loaded = loadInstances(file)
ShapefileDataStore shapeFileDataStore = new ShapefileDataStore(file.toURL())

int num = 0
loaded.iterator().withCloseable {
Expand All @@ -293,16 +313,18 @@ class ShapefileInstanceWriterTest {
def jts = the_geom.geometry
assert jts instanceof Point
def name = inst.p.name.value()
assert name
switch (name) {

String decodedName = new String(name.getBytes(shapeFileDataStore.getCharset()));
assert decodedName
switch (decodedName) {
case 'Darmstadt':
assert inst.p.population.value() == 158254
break
case 'München':
assert inst.p.population.value() == 1471508
break
default:
throw new IllegalStateException("Unexpected type $typeName")
throw new IllegalStateException("Unexpected type $decodedName")
}
}
}
Expand Down Expand Up @@ -349,6 +371,7 @@ class ShapefileInstanceWriterTest {
// load instances again and test

def loaded = loadInstances(file)
ShapefileDataStore shapeFileDataStore = new ShapefileDataStore(file.toURL())

int num = 0
loaded.iterator().withCloseable {
Expand All @@ -368,16 +391,17 @@ class ShapefileInstanceWriterTest {

assert jts instanceof MultiPolygon
def name = inst.p.name.value()
assert name
switch (name) {
String decodedName = new String(name.getBytes(shapeFileDataStore.getCharset()));
assert decodedName
switch (decodedName) {
case 'Darmstadt':
assert inst.p.population.value() == 158254
break
case 'München':
assert inst.p.population.value() == 1471508
break
default:
throw new IllegalStateException("Unexpected type $typeName")
throw new IllegalStateException("Unexpected type $decodedName")
}
}
}
Expand Down Expand Up @@ -422,6 +446,7 @@ class ShapefileInstanceWriterTest {
int num = 0
for (geom in geomNames) {
def loaded = loadInstances(file, geom)
ShapefileDataStore shapeFileDataStore = new ShapefileDataStore(file.toURL())

loaded.iterator().withCloseable {
while (it.hasNext()) {
Expand All @@ -439,16 +464,17 @@ class ShapefileInstanceWriterTest {
def jts = the_geom.geometry
assert jts
def name = inst.p.name.value()
assert name
switch (name) {
String decodedName = new String(name.getBytes(shapeFileDataStore.getCharset()));
assert decodedName
switch (decodedName) {
case 'Darmstadt':
assert inst.p.population.value() == 158254
break
case 'München':
assert inst.p.population.value() == 1471508
break
default:
throw new IllegalStateException("Unexpected type $typeName")
throw new IllegalStateException("Unexpected type $decodedName")
}
}
}
Expand Down Expand Up @@ -1158,6 +1184,7 @@ class ShapefileInstanceWriterTest {
withNewShapefile(schema, instances) { file ->
// load instances again and test
def loaded = loadInstances(file)
ShapefileDataStore shapeFileDataStore = new ShapefileDataStore(file.toURL())

int num = 0
loaded.iterator().withCloseable {
Expand All @@ -1178,16 +1205,17 @@ class ShapefileInstanceWriterTest {
def jts = geom.geometry
assert jts instanceof Point
def name = inst.p.name.value()
assert name
switch (name) {
String decodedName = new String(name.getBytes(shapeFileDataStore.getCharset()));
assert decodedName
switch (decodedName) {
case 'Darmstadt':
assert inst.p.population.value() == 158254
break
case 'München':
assert inst.p.population.value() == 1471508
break
default:
throw new IllegalStateException("Unexpected type $typeName")
throw new IllegalStateException("Unexpected type $decodedName")
}
}
}
Expand Down Expand Up @@ -1339,6 +1367,7 @@ class ShapefileInstanceWriterTest {
// load instances again and test

def loaded = loadInstances(file)
ShapefileDataStore shapeFileDataStore = new ShapefileDataStore(file.toURL())

int num = 0
loaded.iterator().withCloseable {
Expand All @@ -1358,16 +1387,17 @@ class ShapefileInstanceWriterTest {

assert jts instanceof MultiPolygon
def name = inst.p.name.value()
assert name
switch (name) {
String decodedName = new String(name.getBytes(shapeFileDataStore.getCharset()));
assert decodedName
switch (decodedName) {
case 'Darmstadt':
assert inst.p.po12.value() == 158254
break
case 'München':
assert inst.p.po12.value() == 1471508
break
default:
throw new IllegalStateException("Unexpected type $typeName")
throw new IllegalStateException("Unexpected type $decodedName")
}
}
}
Expand Down Expand Up @@ -1413,6 +1443,7 @@ class ShapefileInstanceWriterTest {
// load instances again and test

def loaded = loadInstances(file)
ShapefileDataStore shapeFileDataStore = new ShapefileDataStore(file.toURL())

int num = 0
loaded.iterator().withCloseable {
Expand All @@ -1432,8 +1463,9 @@ class ShapefileInstanceWriterTest {

assert jts instanceof MultiPolygon
def name = inst.p.name.value()
assert name
switch (name) {
String decodedName = new String(name.getBytes(shapeFileDataStore.getCharset()));
assert decodedName
switch (decodedName) {
case 'München':
assert inst.p.po12.value() == 1471508
break
Expand Down
6 changes: 6 additions & 0 deletions ...ugins/eu.esdihumboldt.hale.io.shp/src/eu/esdihumboldt/hale/io/shp/ShapefileConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,12 @@ public interface ShapefileConstants {
* Constant for the shape file extension.
*/
public static final String SHP_EXTENSION = ".shp";

/**
* Constant for the CPG file extension.
*/
public static final String CPG_EXTENSION = ".cpg";

/**
* Constant for underscore.
*/
Expand Down
34 changes: 34 additions & 0 deletions ...ihumboldt.hale.io.shp/src/eu/esdihumboldt/hale/io/shp/writer/ShapefileInstanceWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package eu.esdihumboldt.hale.io.shp.writer;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Serializable;
import java.net.URI;
Expand Down Expand Up @@ -122,6 +123,11 @@ protected IOReport execute(ProgressIndicator progress, IOReporter reporter)
setTarget(new MultiLocationOutputSupplier(uris));
}

for (String f : filesWritten) {
String cpgFileName = filePath + "/" + f + ShapefileConstants.CPG_EXTENSION;
writeCodePageFile(cpgFileName);
}

reporter.setSuccess(true);
} catch (Exception e) {
reporter.error(new IOMessageImpl(e.getMessage(), e));
Expand Down Expand Up @@ -455,6 +461,7 @@ private Map<String, Map<String, ShapefileDataStore>> createSchema(URI location,
ShapefileDataStore newDataStore;

newDataStore = (ShapefileDataStore) dataStoreFactory.createNewDataStore(params);
newDataStore.setCharset(getCharset());
newDataStore.createSchema(geometryEntry.getValue());
schemaDataStoreMap
.computeIfAbsent(schemaEntry.getKey(),
Expand Down Expand Up @@ -724,4 +731,31 @@ private List<String> writeToFile(
return filesWritten;
}

/**
* Create the CPG file starting from the Shapefile
*
* @param cpgFilePath Path of the file to be written with just one line of
* the encoding
* @throws IOException exception in any.
*/
public void writeCodePageFile(String cpgFilePath) throws IOException {
File cpgFile = new File(cpgFilePath);
FileWriter fileWriter = new FileWriter(cpgFile);

try {
fileWriter.write(getCharset() != null ? getCharset().toString()
: getDefaultCharset().toString());
} catch (IOException e) {
throw new IOException("An error occurred while writing the CPG file: " + cpgFilePath
+ " " + e.getMessage());
} finally {
try {
fileWriter.close();
} catch (IOException e) {
throw new IOException("An error occurred while trying to close the CPG file: "
+ cpgFilePath + " " + e.getMessage());
}
}
}

}

0 comments on commit 6083214

Please sign in to comment.