Skip to content

Commit

Permalink
Fix the cumulative non-spatial attributes issue (#223)
Browse files Browse the repository at this point in the history 
* Fix the issue that UserData field of a Geometry has cumulative non-spatial attributes. Refine the internal logic of FormatMapper and remove redundant code.

* Fix the error in core scala test

* Reduce the unit test data to make the test faster
  • Loading branch information
@jiayuasu
jiayuasu committed Apr 26, 2018
1 parent 19ffb21 commit c56b615
Show file tree
Hide file tree
Showing 23 changed files with 3,129 additions and 47,906 deletions.
3 changes: 2 additions & 1 deletion core/src/main/java/org/datasyslab/geospark/enums/GeometryType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ public enum GeometryType
MULTIPOLYGON,
MULTILINESTRING,
GEOMETRYCOLLECTION,
CIRCLE;
CIRCLE,
RECTANGLE;

/**
* Gets the GeometryType.
Expand Down
80 changes: 54 additions & 26 deletions core/src/main/java/org/datasyslab/geospark/formatMapper/FormatMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import com.vividsolutions.jts.geom.Geometry;
import com.vividsolutions.jts.geom.GeometryCollection;
import com.vividsolutions.jts.geom.GeometryFactory;
import com.vividsolutions.jts.geom.Polygon;
import com.vividsolutions.jts.io.ParseException;
import com.vividsolutions.jts.io.WKTReader;
import com.vividsolutions.jts.io.WKBReader;
Expand Down Expand Up @@ -93,12 +94,18 @@ public class FormatMapper
* @param splitter the splitter
* @param carryInputData the carry input data
*/
public FormatMapper(int startOffset, int endOffset, FileDataSplitter splitter, boolean carryInputData)
public FormatMapper(int startOffset, int endOffset, FileDataSplitter splitter, boolean carryInputData, GeometryType geometryType)
{
this.startOffset = startOffset;
this.endOffset = endOffset;
this.splitter = splitter;
this.carryInputData = carryInputData;
this.geometryType = geometryType;
// Only the following formats are allowed to use this format mapper because each input has the geometry type definition
if (geometryType == null)
{
assert splitter == FileDataSplitter.WKB || splitter == FileDataSplitter.WKT || splitter == FileDataSplitter.GEOJSON;
}
}

/**
Expand All @@ -108,14 +115,9 @@ public FormatMapper(int startOffset, int endOffset, FileDataSplitter splitter, b
*/
public FormatMapper(FileDataSplitter splitter, boolean carryInputData)
{
this(0,-1,splitter,carryInputData);
this(0,-1,splitter,carryInputData, null);
}

public FormatMapper(int startOffset, int endOffset, FileDataSplitter splitter, boolean carryInputData, GeometryType geometryType)
{
this(startOffset, endOffset, splitter, carryInputData);
this.geometryType = geometryType;
}

/**
* This format mapper is used in GeoSparkSQL.
Expand All @@ -125,8 +127,7 @@ public FormatMapper(int startOffset, int endOffset, FileDataSplitter splitter, b
*/
public FormatMapper(FileDataSplitter splitter, boolean carryInputData, GeometryType geometryType)
{
this(0, -1, splitter, carryInputData);
this.geometryType = geometryType;
this(0, -1, splitter, carryInputData, geometryType);
}

private void readObject(ObjectInputStream inputStream)
Expand All @@ -146,6 +147,7 @@ public Geometry readGeoJSON(String geoJson)
geometry = geoJSONReader.read(feature.getGeometry());
if (carryInputData) {
boolean hasIdFlag = false;
otherAttributes = "";
if (feature.getId()!=null)
{
this.otherAttributes += feature.getId();
Expand Down Expand Up @@ -176,6 +178,7 @@ public Geometry readWkt(String line)
final Geometry geometry = wktReader.read(columns[this.startOffset]);
if (carryInputData) {
boolean firstColumnFlag = true;
otherAttributes = "";
for (int i=0; i<columns.length;i++)
{
if (i!=this.startOffset )
Expand Down Expand Up @@ -206,6 +209,7 @@ public Geometry readWkb(String line)
final Geometry geometry = wkbReader.read(aux);
if (carryInputData) {
boolean firstColumnFlag = true;
otherAttributes = "";
for (int i=0; i<columns.length;i++)
{
if (i!=this.startOffset )
Expand Down Expand Up @@ -234,24 +238,28 @@ public Coordinate[] readCoordinates(String line)
for (int i = this.startOffset; i <= actualEndOffset; i += 2) {
coordinates[i / 2] = new Coordinate(Double.parseDouble(columns[i]), Double.parseDouble(columns[i + 1]));
}
boolean firstColumnFlag = true;
for (int i= 0;i<this.startOffset;i++)
if (carryInputData)
{
if (firstColumnFlag)
boolean firstColumnFlag = true;
otherAttributes = "";
for (int i= 0;i<this.startOffset;i++)
{
otherAttributes += columns[i];
firstColumnFlag = false;
if (firstColumnFlag)
{
otherAttributes += columns[i];
firstColumnFlag = false;
}
else otherAttributes += "\t" + columns[i];
}
else otherAttributes += "\t" + columns[i];
}
for (int i=actualEndOffset+1;i<columns.length;i++)
{
if (firstColumnFlag)
for (int i=actualEndOffset+1;i<columns.length;i++)
{
otherAttributes += columns[i];
firstColumnFlag = false;
if (firstColumnFlag)
{
otherAttributes += columns[i];
firstColumnFlag = false;
}
else otherAttributes += "\t" + columns[i];
}
else otherAttributes += "\t" + columns[i];
}
return coordinates;
}
Expand Down Expand Up @@ -289,16 +297,36 @@ public Geometry readGeometry(String line)
private Geometry createGeometry(Coordinate[] coordinates, GeometryType geometryType)
{
GeometryFactory geometryFactory = new GeometryFactory();
Geometry geometry = null;
switch (geometryType) {
case POINT:
return geometryFactory.createPoint(coordinates[0]);
geometry = geometryFactory.createPoint(coordinates[0]);
break;
case POLYGON:
return geometryFactory.createPolygon(coordinates);
geometry = geometryFactory.createPolygon(coordinates);
break;
case LINESTRING:
return geometryFactory.createLineString(coordinates);
geometry = geometryFactory.createLineString(coordinates);
break;
case RECTANGLE:
// The rectangle mapper reads two coordinates from the input line. The two coordinates are the two on the diagonal.
assert coordinates.length == 2;
Coordinate[] polyCoordinates = new Coordinate[5];
polyCoordinates[0] = coordinates[0];
polyCoordinates[1] = new Coordinate(coordinates[0].x, coordinates[1].y);
polyCoordinates[2] = coordinates[1];
polyCoordinates[3] = new Coordinate(coordinates[1].x, coordinates[0].y);
polyCoordinates[4] = polyCoordinates[0];
geometry = factory.createPolygon(polyCoordinates);
break;
// Read string to point if no geometry type specified but GeoSpark should never reach here
default:
return geometryFactory.createPoint(coordinates[0]);
geometry = geometryFactory.createPoint(coordinates[0]);
}
if (carryInputData)
{
geometry.setUserData(otherAttributes);
}
return geometry;
}
}
32 changes: 4 additions & 28 deletions core/src/main/java/org/datasyslab/geospark/formatMapper/LineStringFormatMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import com.vividsolutions.jts.geom.MultiLineString;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.datasyslab.geospark.enums.FileDataSplitter;
import org.datasyslab.geospark.enums.GeometryType;

import java.util.ArrayList;
import java.util.Iterator;
Expand All @@ -49,7 +50,7 @@ public class LineStringFormatMapper
*/
public LineStringFormatMapper(FileDataSplitter Splitter, boolean carryInputData)
{
super(0, -1, Splitter, carryInputData);
super(0, -1, Splitter, carryInputData, GeometryType.LINESTRING);
}

/**
Expand All @@ -63,7 +64,7 @@ public LineStringFormatMapper(FileDataSplitter Splitter, boolean carryInputData)
public LineStringFormatMapper(Integer startOffset, Integer endOffset, FileDataSplitter Splitter,
boolean carryInputData)
{
super(startOffset, endOffset, Splitter, carryInputData);
super(startOffset, endOffset, Splitter, carryInputData, GeometryType.LINESTRING);
}

@Override
Expand All @@ -73,32 +74,7 @@ public Iterator<LineString> call(Iterator<String> stringIterator)
List<LineString> result = new ArrayList<>();
while (stringIterator.hasNext()) {
String line = stringIterator.next();
switch (splitter) {
case GEOJSON: {
Geometry geometry = readGeoJSON(line);
addGeometry(geometry, result);
break;
}
case WKT: {
Geometry geometry = readWkt(line);
addGeometry(geometry, result);
break;
}
case WKB: {
Geometry geometry = readWkb(line);
addGeometry(geometry, result);
break;
}
default: {
Coordinate[] coordinates = readCoordinates(line);
LineString lineString = factory.createLineString(coordinates);
if (this.carryInputData) {
lineString.setUserData(otherAttributes);
}
result.add(lineString);
break;
}
}
addGeometry(readGeometry(line), result);
}
return result.iterator();
}
Expand Down
32 changes: 4 additions & 28 deletions core/src/main/java/org/datasyslab/geospark/formatMapper/PointFormatMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import com.vividsolutions.jts.geom.Point;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.datasyslab.geospark.enums.FileDataSplitter;
import org.datasyslab.geospark.enums.GeometryType;

import java.util.ArrayList;
import java.util.Iterator;
Expand All @@ -49,7 +50,7 @@ public class PointFormatMapper
*/
public PointFormatMapper(FileDataSplitter Splitter, boolean carryInputData)
{
super(0, 1, Splitter, carryInputData);
super(0, 1, Splitter, carryInputData, GeometryType.POINT);
}

/**
Expand All @@ -62,7 +63,7 @@ public PointFormatMapper(FileDataSplitter Splitter, boolean carryInputData)
public PointFormatMapper(Integer startOffset, FileDataSplitter Splitter,
boolean carryInputData)
{
super(startOffset, startOffset+1, Splitter, carryInputData);
super(startOffset, startOffset+1, Splitter, carryInputData, GeometryType.POINT);
}

@Override
Expand All @@ -72,32 +73,7 @@ public Iterator<Point> call(final Iterator<String> stringIterator)
List<Point> result = new ArrayList<>();
while (stringIterator.hasNext()) {
String line = stringIterator.next();
switch (splitter) {
case GEOJSON: {
Geometry geometry = readGeoJSON(line);
addGeometry(geometry, result);
break;
}
case WKT: {
Geometry geometry = readWkt(line);
addGeometry(geometry, result);
break;
}
case WKB: {
Geometry geometry = readWkb(line);
addGeometry(geometry, result);
break;
}
default:
Coordinate[] coordinates = readCoordinates(line);
assert coordinates.length == 1;
Point point = factory.createPoint(coordinates[0]);
if (this.carryInputData) {
point.setUserData(otherAttributes);
}
result.add(point);
break;
}
addGeometry(readGeometry(line), result);
}
return result.iterator();
}
Expand Down
33 changes: 4 additions & 29 deletions core/src/main/java/org/datasyslab/geospark/formatMapper/PolygonFormatMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import com.vividsolutions.jts.geom.Polygon;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.datasyslab.geospark.enums.FileDataSplitter;
import org.datasyslab.geospark.enums.GeometryType;

import java.util.ArrayList;
import java.util.Iterator;
Expand All @@ -50,7 +51,7 @@ public class PolygonFormatMapper
*/
public PolygonFormatMapper(FileDataSplitter Splitter, boolean carryInputData)
{
super(0, -1, Splitter, carryInputData);
super(0, -1, Splitter, carryInputData, GeometryType.POLYGON);
}

/**
Expand All @@ -64,7 +65,7 @@ public PolygonFormatMapper(FileDataSplitter Splitter, boolean carryInputData)
public PolygonFormatMapper(Integer startOffset, Integer endOffset, FileDataSplitter Splitter,
boolean carryInputData)
{
super(startOffset, endOffset, Splitter, carryInputData);
super(startOffset, endOffset, Splitter, carryInputData, GeometryType.POLYGON);
}

@Override
Expand All @@ -74,33 +75,7 @@ public Iterator<Polygon> call(Iterator<String> stringIterator)
List<Polygon> result = new ArrayList<>();
while (stringIterator.hasNext()) {
String line = stringIterator.next();
switch (splitter) {
case GEOJSON: {
Geometry geometry = readGeoJSON(line);
addGeometry(geometry, result);
break;
}
case WKT: {
Geometry geometry = readWkt(line);
addGeometry(geometry, result);
break;
}
case WKB: {
Geometry geometry = readWkb(line);
addGeometry(geometry, result);
break;
}
default: {
Coordinate[] coordinates = readCoordinates(line);
LinearRing linearRing = factory.createLinearRing(coordinates);
Polygon polygon = factory.createPolygon(linearRing);
if (this.carryInputData) {
polygon.setUserData(otherAttributes);
}
result.add(polygon);
break;
}
}
addGeometry(readGeometry(line), result);
}
return result.iterator();
}
Expand Down
Loading

0 comments on commit c56b615

Please sign in to comment.